Rewrite of ensure-dirs in python

I provided unittest to test the important pieces of the infrastructure.
The one remaining function (ResuriveEnsure) is not easy to unittest
but also not critical if it fails to operate correctly.
Signed-off-by: René Nussbaumer <rn@google.com>
Reviewed-by: Iustin Pop <iustin@google.com>

.gitignore

@@ -82,6 +82,7 @@
 
 # tools
 /tools/kvm-ifup
+/tools/ensure-dirs
 
 # scripts
 /scripts/gnt-backup

Makefile.am

@@ -36,6 +36,7 @@ impexpddir = $(pkgpythondir)/impexpd
 utilsdir = $(pkgpythondir)/utils
 toolsdir = $(pkglibdir)/tools
 iallocatorsdir = $(pkglibdir)/iallocators
+pytoolsdir = $(pkgpythondir)/tools
 docdir = $(datadir)/doc/$(PACKAGE)
 
 # Delete output file if an error occurred while building it
@@ -65,6 +66,7 @@ DIRS = \
 	lib/masterd \
 	lib/rapi \
 	lib/server \
+	lib/tools \
 	lib/utils \
 	lib/watcher \
 	man \
@@ -247,6 +249,10 @@ server_PYTHON = \
 	lib/server/noded.py \
 	lib/server/rapi.py
 
+pytools_PYTHON = \
+	lib/tools/__init__.py \
+	lib/tools/ensure_dirs.py
+
 utils_PYTHON = \
 	lib/utils/__init__.py \
 	lib/utils/algo.py \
@@ -399,7 +405,7 @@ gnt_scripts = \
 	scripts/gnt-node \
 	scripts/gnt-os
 
-PYTHON_BOOTSTRAP = \
+PYTHON_BOOTSTRAP_SBIN = \
 	daemons/ganeti-confd \
 	daemons/ganeti-masterd \
 	daemons/ganeti-noded \
@@ -414,6 +420,10 @@ PYTHON_BOOTSTRAP = \
 	scripts/gnt-node \
 	scripts/gnt-os
 
+PYTHON_BOOTSTRAP = \
+	$(PYTHON_BOOTSTRAP_SBIN) \
+	tools/ensure-dirs
+
 qa_scripts = \
 	qa/ganeti-qa.py \
 	qa/qa_cluster.py \
@@ -451,7 +461,7 @@ dist_sbin_SCRIPTS = \
 	tools/ganeti-listrunner
 
 nodist_sbin_SCRIPTS = \
-	$(PYTHON_BOOTSTRAP) \
+	$(PYTHON_BOOTSTRAP_SBIN) \
 	daemons/ganeti-cleaner
 
 dist_tools_SCRIPTS = \
@@ -469,12 +479,18 @@ pkglib_python_scripts = \
 	daemons/import-export \
 	tools/check-cert-expired
 
+nodist_pkglib_python_scripts = \
+	tools/ensure-dirs
+
 pkglib_SCRIPTS = \
 	daemons/daemon-util \
 	daemons/ensure-dirs \
 	tools/kvm-ifup \
 	$(pkglib_python_scripts)
 
+nodist_pkglib_SCRIPTS = \
+	$(nodist_pkglib_python_scripts)
+
 EXTRA_DIST = \
 	NEWS \
 	UPGRADE \
@@ -610,6 +626,7 @@ python_tests = \
 	test/ganeti.runtime_unittest.py \
 	test/ganeti.serializer_unittest.py \
 	test/ganeti.ssh_unittest.py \
+	test/ganeti.tools.ensure_dirs_unittest.py \
 	test/ganeti.uidpool_unittest.py \
 	test/ganeti.utils.algo_unittest.py \
 	test/ganeti.utils.filelock_unittest.py \
@@ -657,12 +674,14 @@ all_python_code = \
 	$(dist_sbin_SCRIPTS) \
 	$(dist_tools_SCRIPTS) \
 	$(pkglib_python_scripts) \
+	$(nodist_pkglib_python_scripts) \
 	$(python_tests) \
 	$(pkgpython_PYTHON) \
 	$(client_PYTHON) \
 	$(hypervisor_PYTHON) \
 	$(rapi_PYTHON) \
 	$(server_PYTHON) \
+	$(pytools_PYTHON) \
 	$(http_PYTHON) \
 	$(confd_PYTHON) \
 	$(masterd_PYTHON) \
@@ -881,6 +900,7 @@ $(REPLACE_VARS_SED): Makefile
 daemons/ganeti-%: MODULE = ganeti.server.$(patsubst ganeti-%,%,$(notdir $@))
 daemons/ganeti-watcher: MODULE = ganeti.watcher
 scripts/%: MODULE = ganeti.client.$(subst -,_,$(notdir $@))
+tools/ensure-dirs: MODULE = ganeti.tools.ensure_dirs
 
 $(PYTHON_BOOTSTRAP): Makefile | $(all_dirfiles)
 	test -n "$(MODULE)" || { echo Missing module; exit 1; }

lib/tools/__init__.py

+# Copyright (C) 2011 Google Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+# 02110-1301, USA.
+
+"""Common tools modules.
+
+"""

lib/tools/ensure_dirs.py

+# Copyright (C) 2011 Google Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+# 02110-1301, USA.
+
+"""Script to ensure permissions on files/dirs are accurate.
+
+"""
+
+import errno
+import os
+import os.path
+import optparse
+import sys
+import stat
+
+from ganeti import constants
+from ganeti import errors
+from ganeti import runtime
+from ganeti import ssconf
+
+
+(DIR, FILE) = range(2)
+ALL_TYPES = frozenset([DIR, FILE])
+
+
+class EnsureError(errors.GenericError):
+  """Top-level error class related to this script.
+
+  """
+
+
+def EnsurePermission(path, mode, uid=-1, gid=-1, must_exist=True,
+                     _chmod_fn=os.chmod, _chown_fn=os.chown):
+  """Ensures that given path has given mode.
+
+  @param path: The path to the file
+  @param mode: The mode of the file
+  @param uid: The uid of the owner of this file
+  @param gid: The gid of the owner of this file
+  @param must_exist: Specifies if non-existance of path will be an error
+  @param _chmod_fn: chmod function to use (unittest only)
+  @param _chown_fn: chown function to use (unittest only)
+
+  """
+  try:
+    _chmod_fn(path, mode)
+
+    if max(uid, gid) > -1:
+      _chown_fn(path, uid, gid)
+  except EnvironmentError, err:
+    if err.errno == errno.ENOENT:
+      if must_exist:
+        raise EnsureError("Path %s does not exists, but should" % path)
+    else:
+      raise EnsureError("Error while changing permission on %s: %s" %
+                        (path, err))
+
+
+def EnsureDir(path, mode, uid, gid, _stat_fn=os.lstat, _mkdir_fn=os.mkdir,
+              _ensure_fn=EnsurePermission):
+  """Ensures that given path is a dir and has given mode, uid and gid set.
+
+  @param path: The path to the file
+  @param mode: The mode of the file
+  @param uid: The uid of the owner of this file
+  @param gid: The gid of the owner of this file
+  @param _stat_fn: Stat function to use (unittest only)
+  @param _mkdir_fn: mkdir function to use (unittest only)
+  @param _ensure_fn: ensure function to use (unittest only)
+
+  """
+  try:
+    # We don't want to follow symlinks
+    st_mode = _stat_fn(path)[stat.ST_MODE]
+
+    if not stat.S_ISDIR(st_mode):
+      raise EnsureError("Path %s is expected to be a directory, but it's not" %
+                        path)
+  except EnvironmentError, err:
+    if err.errno == errno.ENOENT:
+      _mkdir_fn(path)
+    else:
+      raise EnsureError("Error while do a stat() on %s: %s" % (path, err))
+
+  _ensure_fn(path, mode, uid=uid, gid=gid)
+
+
+def RecursiveEnsure(path, uid, gid, dir_perm, file_perm):
+  """Ensures permissions recursively down a directory.
+
+  This functions walks the path and sets permissions accordingly.
+
+  @param path: The absolute path to walk
+  @param uid: The uid used as owner
+  @param gid: The gid used as group
+  @param dir_perm: The permission bits set for directories
+  @param file_perm: The permission bits set for files
+
+  """
+  assert os.path.isabs(path), "Path %s is not absolute" % path
+  assert os.path.isdir(path), "Path %s is not a dir" % path
+
+  for root, dirs, files in os.walk(path):
+    for subdir in dirs:
+      EnsurePermission(os.path.join(root, subdir), dir_perm, uid=uid, gid=gid)
+
+    for filename in files:
+      EnsurePermission(os.path.join(root, filename), file_perm, uid=uid,
+                       gid=gid)
+
+
+def ProcessPath(path):
+  """Processes a path component.
+
+  @param path: A tuple of the path component to process
+
+  """
+  (pathname, pathtype, mode, uid, gid) = path[0:5]
+
+  assert pathtype in ALL_TYPES
+
+  if pathtype == DIR:
+    # No additional parameters
+    assert len(path[5:]) == 0
+    EnsureDir(pathname, mode, uid, gid)
+  elif pathtype == FILE:
+    (must_exist, ) = path[5:]
+    EnsurePermission(pathname, mode, uid=uid, gid=gid, must_exist=must_exist)
+
+
+def GetPaths():
+  """Returns a tuple of path objects to process.
+
+  """
+  getent = runtime.GetEnts()
+  masterd_log = constants.DAEMONS_LOGFILES[constants.MASTERD]
+  noded_log = constants.DAEMONS_LOGFILES[constants.NODED]
+  confd_log = constants.DAEMONS_LOGFILES[constants.CONFD]
+  rapi_log = constants.DAEMONS_LOGFILES[constants.RAPI]
+
+  rapi_dir = os.path.join(constants.DATA_DIR, "rapi")
+
+  paths = [
+    (constants.DATA_DIR, DIR, 0755, getent.masterd_uid,
+     getent.masterd_gid),
+    (constants.CLUSTER_DOMAIN_SECRET_FILE, FILE, 0640,
+     getent.masterd_uid, getent.masterd_gid, False),
+    (constants.CLUSTER_CONF_FILE, FILE, 0640, getent.masterd_uid,
+     getent.confd_gid, False),
+    (constants.CONFD_HMAC_KEY, FILE, 0440, getent.confd_uid,
+     getent.masterd_gid, False),
+    (constants.SSH_KNOWN_HOSTS_FILE, FILE, 0644, getent.masterd_uid,
+     getent.masterd_gid, False),
+    (constants.RAPI_CERT_FILE, FILE, 0440, getent.rapi_uid,
+     getent.masterd_gid, False),
+    (constants.NODED_CERT_FILE, FILE, 0440, getent.masterd_uid,
+     getent.masterd_gid, False),
+    ]
+
+  ss = ssconf.SimpleStore()
+  for ss_path in ss.GetFileList():
+    paths.append((ss_path, FILE, 0400, getent.noded_uid, 0, False))
+
+  paths.extend([
+    (constants.QUEUE_DIR, DIR, 0700, getent.masterd_uid,
+     getent.masterd_gid),
+    (constants.JOB_QUEUE_SERIAL_FILE, FILE, 0600,
+     getent.masterd_uid, getent.masterd_gid, False),
+    (constants.JOB_QUEUE_ARCHIVE_DIR, DIR, 0700,
+     getent.masterd_uid, getent.masterd_gid),
+    (rapi_dir, DIR, 0750, getent.rapi_uid, getent.masterd_gid),
+    (constants.RAPI_USERS_FILE, FILE, 0640, getent.rapi_uid,
+     getent.masterd_gid, False),
+    (constants.RUN_GANETI_DIR, DIR, 0775, getent.masterd_uid,
+     getent.daemons_gid),
+    (constants.SOCKET_DIR, DIR, 0750, getent.masterd_uid,
+     getent.daemons_gid),
+    (constants.MASTER_SOCKET, FILE, 0770, getent.masterd_uid,
+     getent.daemons_gid, False),
+    (constants.BDEV_CACHE_DIR, DIR, 0755, getent.noded_uid,
+     getent.masterd_gid),
+    (constants.UIDPOOL_LOCKDIR, DIR, 0750, getent.noded_uid,
+     getent.masterd_gid),
+    (constants.DISK_LINKS_DIR, DIR, 0755, getent.noded_uid,
+     getent.masterd_gid),
+    (constants.CRYPTO_KEYS_DIR, DIR, 0700, getent.noded_uid,
+     getent.masterd_gid),
+    (constants.IMPORT_EXPORT_DIR, DIR, 0755, getent.noded_uid,
+     getent.masterd_gid),
+    (constants.LOG_DIR, DIR, 0770, getent.masterd_uid,
+     getent.daemons_gid),
+    (masterd_log, FILE, 0600, getent.masterd_uid, getent.masterd_gid,
+     False),
+    (confd_log, FILE, 0600, getent.confd_uid, getent.masterd_gid, False),
+    (noded_log, FILE, 0600, getent.noded_uid, getent.masterd_gid, False),
+    (rapi_log, FILE, 0600, getent.rapi_uid, getent.masterd_gid, False),
+    (constants.LOG_OS_DIR, DIR, 0750, getent.masterd_uid,
+     getent.daemons_gid),
+    ])
+
+  return tuple(paths)
+
+
+def ParseOptions():
+  """Parses the options passed to the program.
+
+  @return: Options and arguments
+
+  """
+  program = os.path.basename(sys.argv[0])
+
+  parser = optparse.OptionParser(usage="%%prog [--full-run]",
+                                 prog=program)
+  parser.add_option("--full-run", "-f", dest="full_run", action="store_true",
+                    default=False, help=("Make a full run and collect"
+                                         " additional files (time consuming)"))
+
+  return parser.parse_args()
+
+
+def Main():
+  """Main routine.
+
+  """
+  getent = runtime.GetEnts()
+  (opts, _) = ParseOptions()
+
+  try:
+    for path in GetPaths():
+      ProcessPath(path)
+
+    if opts.full_run:
+      RecursiveEnsure(constants.JOB_QUEUE_ARCHIVE_DIR, getent.masterd_uid,
+                      getent.masterd_gid, 0700, 0600)
+  except EnsureError, err:
+    print >> sys.stderr, "An error occurred while ensure permissions:", err
+    return constants.EXIT_FAILURE
+
+  return constants.EXIT_SUCCESS

test/ganeti.tools.ensure_dirs_unittest.py

+#!/usr/bin/python
+#
+
+# Copyright (C) 2011 Google Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+# 02110-1301, USA.
+
+
+"""Script for testing ganeti.tools.ensure_dirs"""
+
+import errno
+import stat
+import unittest
+import os.path
+
+from ganeti.tools import ensure_dirs
+
+import testutils
+
+
+class TestEnsureDirsFunctions(unittest.TestCase):
+  def _NoopMkdir(self, _):
+    self.mkdir_called = True
+
+  @staticmethod
+  def _MockStatResult(mode, pre_fn=lambda: 0):
+    def _fn(path):
+      pre_fn()
+      return {stat.ST_MODE: mode}
+    return _fn
+
+  def _VerifyEnsure(self, path, mode, uid=-1, gid=-1):
+    self.assertEqual(path, "/ganeti-qa-non-test")
+    self.assertEqual(mode, 0700)
+    self.assertEqual(uid, 0)
+    self.assertEqual(gid, 0)
+
+  @staticmethod
+  def _RaiseNoEntError():
+    noent_error = EnvironmentError()
+    noent_error.errno = errno.ENOENT
+    raise noent_error
+
+  @staticmethod
+  def _OtherStatRaise():
+    raise EnvironmentError()
+
+  def _ChmodWrapper(self, pre_fn=lambda: 0):
+    def _fn(path, mode):
+      self.chmod_called = True
+      pre_fn()
+    return _fn
+
+  def _NoopChown(self, path, uid, gid):
+    self.chown_called = True
+
+  def testEnsureDir(self):
+    is_dir_stat = self._MockStatResult(stat.S_IFDIR)
+    not_dir_stat = self._MockStatResult(0)
+    non_exist_stat = self._MockStatResult(stat.S_IFDIR,
+                                          pre_fn=self._RaiseNoEntError)
+    other_stat_raise = self._MockStatResult(stat.S_IFDIR,
+                                            pre_fn=self._OtherStatRaise)
+
+    self.assertRaises(ensure_dirs.EnsureError, ensure_dirs.EnsureDir,
+                      "/ganeti-qa-non-test", 0700, 0, 0,
+                      _stat_fn=not_dir_stat)
+    self.assertRaises(ensure_dirs.EnsureError, ensure_dirs.EnsureDir,
+                      "/ganeti-qa-non-test", 0700, 0, 0,
+                      _stat_fn=other_stat_raise)
+    self.mkdir_called = False
+    ensure_dirs.EnsureDir("/ganeti-qa-non-test", 0700, 0, 0,
+                          _stat_fn=non_exist_stat, _mkdir_fn=self._NoopMkdir,
+                          _ensure_fn=self._VerifyEnsure)
+    self.assertTrue(self.mkdir_called)
+    self.mkdir_called = False
+    ensure_dirs.EnsureDir("/ganeti-qa-non-test", 0700, 0, 0,
+                          _stat_fn=is_dir_stat, _ensure_fn=self._VerifyEnsure)
+    self.assertFalse(self.mkdir_called)
+
+  def testEnsurePermission(self):
+    noent_chmod_fn = self._ChmodWrapper(pre_fn=self._RaiseNoEntError)
+    self.assertRaises(ensure_dirs.EnsureError, ensure_dirs.EnsurePermission,
+                      "/ganeti-qa-non-test", 0600,
+                      _chmod_fn=noent_chmod_fn)
+    self.chmod_called = False
+    ensure_dirs.EnsurePermission("/ganeti-qa-non-test", 0600, must_exist=False,
+                                 _chmod_fn=noent_chmod_fn)
+    self.assertTrue(self.chmod_called)
+    self.assertRaises(ensure_dirs.EnsureError, ensure_dirs.EnsurePermission,
+                      "/ganeti-qa-non-test", 0600, must_exist=False,
+                      _chmod_fn=self._ChmodWrapper(pre_fn=self._OtherStatRaise))
+    self.chmod_called = False
+    self.chown_called = False
+    ensure_dirs.EnsurePermission("/ganeti-qa-non-test", 0600,
+                                 _chmod_fn=self._ChmodWrapper(),
+                                 _chown_fn=self._NoopChown)
+    self.assertTrue(self.chmod_called)
+    self.assertFalse(self.chown_called)
+    self.chmod_called = False
+    ensure_dirs.EnsurePermission("/ganeti-qa-non-test", 0600, uid=1, gid=1,
+                                 _chmod_fn=self._ChmodWrapper(),
+                                 _chown_fn=self._NoopChown)
+    self.assertTrue(self.chmod_called)
+    self.assertTrue(self.chown_called)
+
+  def testPaths(self):
+    paths = [(path[0], path[1]) for path in ensure_dirs.GetPaths()]
+
+    seen = []
+    last_dirname = ""
+    for path, pathtype in paths:
+      self.assertTrue(pathtype in ensure_dirs.ALL_TYPES)
+      dirname = os.path.dirname(path)
+      if dirname != last_dirname or pathtype == ensure_dirs.DIR:
+        if pathtype == ensure_dirs.FILE:
+          self.assertFalse(dirname in seen,
+                           msg="path %s; dirname %s seen in %s" % (path,
+                                                                   dirname,
+                                                                   seen))
+          last_dirname = dirname
+          seen.append(dirname)
+        elif pathtype == ensure_dirs.DIR:
+          self.assertFalse(path in seen)
+          last_dirname = path
+          seen.append(path)
+
+
+if __name__ == "__main__":
+  testutils.GanetiTestProgram()