Add new module for virtual clusters

This module will take care of managing paths for virtual clusters.
Unittests are included (100% coverage).
Signed-off-by: Michael Hanselmann <hansmi@google.com>
Reviewed-by: René Nussbaumer <rn@google.com>

Makefile.am

@@ -240,6 +240,7 @@ pkgpython_PYTHON = \
 	lib/ssh.py \
 	lib/storage.py \
 	lib/uidpool.py \
+	lib/vcluster.py \
 	lib/workerpool.py
 
 client_PYTHON = \
@@ -913,6 +914,7 @@ python_tests = \
 	test/ganeti.utils.wrapper_unittest.py \
 	test/ganeti.utils.x509_unittest.py \
 	test/ganeti.utils_unittest.py \
+	test/ganeti.vcluster_unittest.py \
 	test/ganeti.workerpool_unittest.py \
 	test/qa.qa_config_unittest.py \
 	test/cfgupgrade_unittest.py \

lib/vcluster.py

+#
+#
+
+# Copyright (C) 2012 Google Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+# 02110-1301, USA.
+
+
+"""Module containing utilities for virtual clusters.
+
+
+"""
+
+import os
+
+from ganeti import compat
+
+
+_VIRT_PATH_PREFIX = "/###-VIRTUAL-PATH-###,"
+_ROOTDIR_ENVNAME = "GANETI_ROOTDIR"
+_HOSTNAME_ENVNAME = "GANETI_HOSTNAME"
+
+
+def _GetRootDirectory(envname):
+  """Retrieves root directory from an environment variable.
+
+  @type envname: string
+  @param envname: Environment variable name
+  @rtype: string
+  @return: Root directory (can be empty)
+
+  """
+  path = os.getenv(envname)
+
+  if path:
+    if not os.path.isabs(path):
+      raise RuntimeError("Root directory in '%s' must be absolute: %s" %
+                         (envname, path))
+    return os.path.normpath(path)
+
+  return ""
+
+
+def _GetHostname(envname):
+  """Retrieves virtual hostname from an environment variable.
+
+  @type envname: string
+  @param envname: Environment variable name
+  @rtype: string
+  @return: Host name (can be empty)
+
+  """
+  return os.getenv(envname, default="")
+
+
+def _CheckHostname(hostname):
+  """Very basic check for hostnames.
+
+  @type hostname: string
+  @param hostname: Hostname
+
+  """
+  if os.path.basename(hostname) != hostname:
+    raise RuntimeError("Hostname '%s' can not be used for a file system"
+                       " path" % hostname)
+
+
+def _PreparePaths(rootdir, hostname):
+  """Checks if the root directory and hostname are acceptable.
+
+  @type rootdir: string
+  @param rootdir: Root directory (from environment)
+  @type hostname: string
+  @param hostname: Hostname (from environment)
+  @rtype: tuple; (string, string, string or None)
+  @return: Tuple containing cluster-global root directory, node root directory
+    and virtual hostname
+
+  """
+  if bool(rootdir) ^ bool(hostname):
+    raise RuntimeError("Both root directory and hostname must be specified"
+                       " using the environment variables %s and %s" %
+                       (_ROOTDIR_ENVNAME, _HOSTNAME_ENVNAME))
+
+  if rootdir:
+    assert rootdir == os.path.normpath(rootdir)
+
+    _CheckHostname(hostname)
+
+    if os.path.basename(rootdir) != hostname:
+      raise RuntimeError("Last component of root directory ('%s') must match"
+                         " hostname ('%s')" % (rootdir, hostname))
+
+    return (os.path.dirname(rootdir), rootdir, hostname)
+  else:
+    return ("", "", None)
+
+
+(_VIRT_BASEDIR, _VIRT_NODEROOT, _VIRT_HOSTNAME) = \
+  _PreparePaths(_GetRootDirectory(_ROOTDIR_ENVNAME),
+                _GetHostname(_HOSTNAME_ENVNAME))
+
+
+assert (compat.all([_VIRT_BASEDIR, _VIRT_NODEROOT, _VIRT_HOSTNAME]) or
+        not compat.any([_VIRT_BASEDIR, _VIRT_NODEROOT, _VIRT_HOSTNAME]))
+
+
+def GetVirtualHostname():
+  """Returns the virtual hostname.
+
+  @rtype: string or L{None}
+
+  """
+  return _VIRT_HOSTNAME
+
+
+def _MakeNodeRoot(base, node_name):
+  """Appends a node name to the base directory.
+
+  """
+  _CheckHostname(node_name)
+  return os.path.normpath("%s/%s" % (base, node_name))
+
+
+def ExchangeNodeRoot(node_name, filename,
+                     _basedir=_VIRT_BASEDIR, _noderoot=_VIRT_NODEROOT):
+  """Replaces the node-specific root directory in a path.
+
+  Replaces it with the root directory for another node.
+
+  """
+  if _basedir:
+    pure = _RemoveNodePrefix(filename, _noderoot=_noderoot)
+    result = "%s/%s" % (_MakeNodeRoot(_basedir, node_name), pure)
+  else:
+    result = filename
+
+  return os.path.normpath(result)
+
+
+def EnvironmentForHost(hostname, _basedir=_VIRT_BASEDIR):
+  """Returns the environment variables for a host.
+
+  """
+  if _basedir:
+    return {
+      _ROOTDIR_ENVNAME: _MakeNodeRoot(_basedir, hostname),
+      _HOSTNAME_ENVNAME: hostname,
+      }
+  else:
+    return {}
+
+
+def AddNodePrefix(path, _noderoot=_VIRT_NODEROOT):
+  """Adds a node-specific prefix to a path in a virtual cluster.
+
+  Returned path includes user-specified root directory if specified in
+  environment.
+
+  """
+  assert os.path.isabs(path)
+
+  if _noderoot:
+    result = "%s/%s" % (_noderoot, path)
+  else:
+    result = path
+
+  assert os.path.isabs(result)
+
+  return os.path.normpath(result)
+
+
+def _RemoveNodePrefix(path, _noderoot=_VIRT_NODEROOT):
+  """Removes the node-specific prefix from a path.
+
+  """
+  assert os.path.isabs(path)
+
+  norm_path = os.path.normpath(path)
+
+  if _noderoot:
+    # Make sure path is actually below node root
+    norm_root = os.path.normpath(_noderoot)
+    root_with_sep = "%s%s" % (norm_root, os.sep)
+    prefix = os.path.commonprefix([root_with_sep, norm_path])
+
+    if prefix == root_with_sep:
+      result = norm_path[len(norm_root):]
+    else:
+      raise RuntimeError("Path '%s' is not below node root '%s'" %
+                         (path, _noderoot))
+  else:
+    result = norm_path
+
+  assert os.path.isabs(result)
+
+  return result
+
+
+def MakeVirtualPath(path, _noderoot=_VIRT_NODEROOT):
+  """Virtualizes a path.
+
+  A path is "virtualized" by stripping it of its node-specific directory and
+  prepending a prefix (L{_VIRT_PATH_PREFIX}). Use L{LocalizeVirtualPath} to
+  undo the process.
+
+  """
+  assert os.path.isabs(path)
+
+  if _noderoot:
+    return _VIRT_PATH_PREFIX + _RemoveNodePrefix(path, _noderoot=_noderoot)
+  else:
+    return path
+
+
+def LocalizeVirtualPath(path, _noderoot=_VIRT_NODEROOT):
+  """Localizes a virtual path.
+
+  A "virtualized" path consists of a prefix (L{LocalizeVirtualPath}) and a
+  local path. This function adds the node-specific directory to the local path.
+
+  """
+  assert os.path.isabs(path)
+
+  if _noderoot:
+    if path.startswith(_VIRT_PATH_PREFIX):
+      return AddNodePrefix(path[len(_VIRT_PATH_PREFIX):], _noderoot=_noderoot)
+    else:
+      raise RuntimeError("Path '%s' is not a virtual path" % path)
+  else:
+    return path

test/ganeti.vcluster_unittest.py

+#!/usr/bin/python
+#
+
+# Copyright (C) 2012 Google Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+# 02110-1301, USA.
+
+
+"""Script for testing ganeti.vcluster"""
+
+import os
+import unittest
+
+from ganeti import utils
+from ganeti import compat
+from ganeti import vcluster
+
+import testutils
+
+
+_ENV_DOES_NOT_EXIST = "GANETI_TEST_DOES_NOT_EXIST"
+_ENV_TEST = "GANETI_TESTVAR"
+
+
+class _EnvVarTest(testutils.GanetiTestCase):
+  def setUp(self):
+    testutils.GanetiTestCase.setUp(self)
+
+    os.environ.pop(_ENV_DOES_NOT_EXIST, None)
+    os.environ.pop(_ENV_TEST, None)
+
+
+class TestGetRootDirectory(_EnvVarTest):
+  def test(self):
+    assert os.getenv(_ENV_TEST) is None
+
+    self.assertEqual(vcluster._GetRootDirectory(_ENV_DOES_NOT_EXIST), "")
+    self.assertEqual(vcluster._GetRootDirectory(_ENV_TEST), "")
+
+    # Absolute path
+    os.environ[_ENV_TEST] = "/tmp/xy11"
+    self.assertEqual(vcluster._GetRootDirectory(_ENV_TEST), "/tmp/xy11")
+
+    # Relative path
+    os.environ[_ENV_TEST] = "foobar"
+    self.assertRaises(RuntimeError, vcluster._GetRootDirectory, _ENV_TEST)
+
+
+
+class TestGetHostname(_EnvVarTest):
+  def test(self):
+    assert os.getenv(_ENV_TEST) is None
+
+    self.assertEqual(vcluster._GetRootDirectory(_ENV_DOES_NOT_EXIST), "")
+    self.assertEqual(vcluster._GetRootDirectory(_ENV_TEST), "")
+
+    os.environ[_ENV_TEST] = "some.host.example.com"
+    self.assertEqual(vcluster._GetHostname(_ENV_TEST), "some.host.example.com")
+
+
+class TestCheckHostname(_EnvVarTest):
+  def test(self):
+    for i in ["/", "/tmp"]:
+      self.assertRaises(RuntimeError, vcluster._CheckHostname, i)
+
+
+class TestPreparePaths(_EnvVarTest):
+  def testInvalidParameters(self):
+    self.assertRaises(RuntimeError, vcluster._PreparePaths,
+                      None, "host.example.com")
+    self.assertRaises(RuntimeError, vcluster._PreparePaths,
+                      "/tmp/", "")
+
+  def testNonNormalizedRootDir(self):
+    self.assertRaises(AssertionError, vcluster._PreparePaths,
+                      "/tmp////xyz//", "host.example.com")
+
+  def testInvalidHostname(self):
+    self.assertRaises(RuntimeError, vcluster._PreparePaths, "/tmp", "/")
+
+  def testPathHostnameMismatch(self):
+    self.assertRaises(RuntimeError, vcluster._PreparePaths,
+                      "/tmp/host.example.com", "server.example.com")
+
+  def testNoVirtCluster(self):
+    for i in ["", None]:
+      self.assertEqual(vcluster._PreparePaths(i, i), ("", "", None))
+
+  def testVirtCluster(self):
+    self.assertEqual(vcluster._PreparePaths("/tmp/host.example.com",
+                                            "host.example.com"),
+                     ("/tmp", "/tmp/host.example.com", "host.example.com"))
+
+
+class TestMakeNodeRoot(unittest.TestCase):
+  def test(self):
+    self.assertRaises(RuntimeError, vcluster._MakeNodeRoot, "/tmp", "/")
+
+    for i in ["/tmp", "/tmp/", "/tmp///"]:
+      self.assertEqual(vcluster._MakeNodeRoot(i, "other.example.com"),
+                       "/tmp/other.example.com")
+
+
+class TestEnvironmentForHost(unittest.TestCase):
+  def test(self):
+    self.assertEqual(vcluster.EnvironmentForHost("host.example.com",
+                                                 _basedir=None),
+                     {})
+    for i in ["host.example.com", "other.example.com"]:
+      self.assertEqual(vcluster.EnvironmentForHost(i, _basedir="/tmp"), {
+        vcluster._ROOTDIR_ENVNAME: "/tmp/%s" % i,
+        vcluster._HOSTNAME_ENVNAME: i,
+        })
+
+
+class TestExchangeNodeRoot(unittest.TestCase):
+  def test(self):
+    result = vcluster.ExchangeNodeRoot("node1.example.com", "/tmp/file",
+                                       _basedir=None, _noderoot=None)
+    self.assertEqual(result, "/tmp/file")
+
+    self.assertRaises(RuntimeError, vcluster.ExchangeNodeRoot,
+                      "node1.example.com", "/tmp/node1.example.com",
+                      _basedir="/tmp",
+                      _noderoot="/tmp/nodeZZ.example.com")
+
+    result = vcluster.ExchangeNodeRoot("node2.example.com",
+                                       "/tmp/node1.example.com/file",
+                                       _basedir="/tmp",
+                                       _noderoot="/tmp/node1.example.com")
+    self.assertEqual(result, "/tmp/node2.example.com/file")
+
+
+class TestAddNodePrefix(unittest.TestCase):
+  def testRelativePath(self):
+    self.assertRaises(AssertionError, vcluster.AddNodePrefix,
+                      "foobar", _noderoot=None)
+
+  def testRelativeNodeRoot(self):
+    self.assertRaises(AssertionError, vcluster.AddNodePrefix,
+                      "/tmp", _noderoot="foobar")
+
+  def test(self):
+    path = vcluster.AddNodePrefix("/file/path",
+                                  _noderoot="/tmp/node1.example.com/")
+    self.assertEqual(path, "/tmp/node1.example.com/file/path")
+
+    self.assertEqual(vcluster.AddNodePrefix("/file/path", _noderoot=""),
+                     "/file/path")
+
+
+class TestRemoveNodePrefix(unittest.TestCase):
+  def testRelativePath(self):
+    self.assertRaises(AssertionError, vcluster._RemoveNodePrefix,
+                      "foobar", _noderoot=None)
+
+  def testOutsideNodeRoot(self):
+    self.assertRaises(RuntimeError, vcluster._RemoveNodePrefix,
+                      "/file/path", _noderoot="/tmp/node1.example.com")
+    self.assertRaises(RuntimeError, vcluster._RemoveNodePrefix,
+                      "/tmp/xyzfile", _noderoot="/tmp/xyz")
+
+  def test(self):
+    path = vcluster._RemoveNodePrefix("/tmp/node1.example.com/file/path",
+                                      _noderoot="/tmp/node1.example.com")
+    self.assertEqual(path, "/file/path")
+
+    path = vcluster._RemoveNodePrefix("/file/path", _noderoot=None)
+    self.assertEqual(path, "/file/path")
+
+
+class TestMakeVirtualPath(unittest.TestCase):
+  def testRelativePath(self):
+    self.assertRaises(AssertionError, vcluster.MakeVirtualPath,
+                      "foobar", _noderoot=None)
+
+  def testOutsideNodeRoot(self):
+    self.assertRaises(RuntimeError, vcluster.MakeVirtualPath,
+                      "/file/path", _noderoot="/tmp/node1.example.com")
+
+  def testWithNodeRoot(self):
+    path = vcluster.MakeVirtualPath("/tmp/node1.example.com/tmp/file",
+                                    _noderoot="/tmp/node1.example.com")
+    self.assertEqual(path, "%s/tmp/file" % vcluster._VIRT_PATH_PREFIX)
+
+  def testNormal(self):
+    self.assertEqual(vcluster.MakeVirtualPath("/tmp/file", _noderoot=None),
+                     "/tmp/file")
+
+
+class TestLocalizeVirtualPath(unittest.TestCase):
+  def testWrongPrefix(self):
+    self.assertRaises(RuntimeError, vcluster.LocalizeVirtualPath,
+                      "/tmp/some/path", _noderoot="/tmp/node1.example.com")
+
+  def testCorrectPrefixRelativePath(self):
+    self.assertRaises(AssertionError, vcluster.LocalizeVirtualPath,
+                      vcluster._VIRT_PATH_PREFIX + "foobar",
+                      _noderoot="/tmp/node1.example.com")
+
+  def testWithNodeRoot(self):
+    lvp = vcluster.LocalizeVirtualPath
+
+    virtpath1 = "%s/tmp/file" % vcluster._VIRT_PATH_PREFIX
+    virtpath2 = "%s////tmp////file" % vcluster._VIRT_PATH_PREFIX
+
+    for i in [virtpath1, virtpath2]:
+      result = lvp(i, _noderoot="/tmp/node1.example.com")
+      self.assertEqual(result, "/tmp/node1.example.com/tmp/file")
+
+  def testNormal(self):
+    self.assertEqual(vcluster.LocalizeVirtualPath("/tmp/file", _noderoot=None),
+                     "/tmp/file")
+
+
+class TestVirtualPathPrefix(unittest.TestCase):
+  def test(self):
+    self.assertTrue(os.path.isabs(vcluster._VIRT_PATH_PREFIX))
+    self.assertEqual(os.path.normcase(vcluster._VIRT_PATH_PREFIX),
+                     vcluster._VIRT_PATH_PREFIX)
+
+
+if __name__ == "__main__":
+  testutils.GanetiTestProgram()