From 0d2bf835ce60fed0003db60bbe52e30190bbb79e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ren=C3=A9=20Nussbaumer?= <rn@google.com>
Date: Fri, 18 Mar 2011 10:30:09 +0100
Subject: [PATCH] Rewrite of ensure-dirs in python
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
I provided unittest to test the important pieces of the infrastructure.
The one remaining function (ResuriveEnsure) is not easy to unittest
but also not critical if it fails to operate correctly.
Signed-off-by: RenΓ© Nussbaumer <rn@google.com>
Reviewed-by: Iustin Pop <iustin@google.com>
---
.gitignore | 1 +
Makefile.am | 24 ++-
lib/tools/__init__.py | 20 ++
lib/tools/ensure_dirs.py | 252 ++++++++++++++++++++++
test/ganeti.tools.ensure_dirs_unittest.py | 143 ++++++++++++
5 files changed, 438 insertions(+), 2 deletions(-)
create mode 100644 lib/tools/__init__.py
create mode 100644 lib/tools/ensure_dirs.py
create mode 100755 test/ganeti.tools.ensure_dirs_unittest.py
diff --git a/.gitignore b/.gitignore
index 453f8fab3..ab52962fb 100644
--- a/.gitignore
+++ b/.gitignore
@@ -82,6 +82,7 @@
# tools
/tools/kvm-ifup
+/tools/ensure-dirs
# scripts
/scripts/gnt-backup
diff --git a/Makefile.am b/Makefile.am
index 6c1fe1d4e..c9f3aec82 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -36,6 +36,7 @@ impexpddir = $(pkgpythondir)/impexpd
utilsdir = $(pkgpythondir)/utils
toolsdir = $(pkglibdir)/tools
iallocatorsdir = $(pkglibdir)/iallocators
+pytoolsdir = $(pkgpythondir)/tools
docdir = $(datadir)/doc/$(PACKAGE)
# Delete output file if an error occurred while building it
@@ -65,6 +66,7 @@ DIRS = \
lib/masterd \
lib/rapi \
lib/server \
+ lib/tools \
lib/utils \
lib/watcher \
man \
@@ -247,6 +249,10 @@ server_PYTHON = \
lib/server/noded.py \
lib/server/rapi.py
+pytools_PYTHON = \
+ lib/tools/__init__.py \
+ lib/tools/ensure_dirs.py
+
utils_PYTHON = \
lib/utils/__init__.py \
lib/utils/algo.py \
@@ -399,7 +405,7 @@ gnt_scripts = \
scripts/gnt-node \
scripts/gnt-os
-PYTHON_BOOTSTRAP = \
+PYTHON_BOOTSTRAP_SBIN = \
daemons/ganeti-confd \
daemons/ganeti-masterd \
daemons/ganeti-noded \
@@ -414,6 +420,10 @@ PYTHON_BOOTSTRAP = \
scripts/gnt-node \
scripts/gnt-os
+PYTHON_BOOTSTRAP = \
+ $(PYTHON_BOOTSTRAP_SBIN) \
+ tools/ensure-dirs
+
qa_scripts = \
qa/ganeti-qa.py \
qa/qa_cluster.py \
@@ -451,7 +461,7 @@ dist_sbin_SCRIPTS = \
tools/ganeti-listrunner
nodist_sbin_SCRIPTS = \
- $(PYTHON_BOOTSTRAP) \
+ $(PYTHON_BOOTSTRAP_SBIN) \
daemons/ganeti-cleaner
dist_tools_SCRIPTS = \
@@ -469,12 +479,18 @@ pkglib_python_scripts = \
daemons/import-export \
tools/check-cert-expired
+nodist_pkglib_python_scripts = \
+ tools/ensure-dirs
+
pkglib_SCRIPTS = \
daemons/daemon-util \
daemons/ensure-dirs \
tools/kvm-ifup \
$(pkglib_python_scripts)
+nodist_pkglib_SCRIPTS = \
+ $(nodist_pkglib_python_scripts)
+
EXTRA_DIST = \
NEWS \
UPGRADE \
@@ -610,6 +626,7 @@ python_tests = \
test/ganeti.runtime_unittest.py \
test/ganeti.serializer_unittest.py \
test/ganeti.ssh_unittest.py \
+ test/ganeti.tools.ensure_dirs_unittest.py \
test/ganeti.uidpool_unittest.py \
test/ganeti.utils.algo_unittest.py \
test/ganeti.utils.filelock_unittest.py \
@@ -657,12 +674,14 @@ all_python_code = \
$(dist_sbin_SCRIPTS) \
$(dist_tools_SCRIPTS) \
$(pkglib_python_scripts) \
+ $(nodist_pkglib_python_scripts) \
$(python_tests) \
$(pkgpython_PYTHON) \
$(client_PYTHON) \
$(hypervisor_PYTHON) \
$(rapi_PYTHON) \
$(server_PYTHON) \
+ $(pytools_PYTHON) \
$(http_PYTHON) \
$(confd_PYTHON) \
$(masterd_PYTHON) \
@@ -881,6 +900,7 @@ $(REPLACE_VARS_SED): Makefile
daemons/ganeti-%: MODULE = ganeti.server.$(patsubst ganeti-%,%,$(notdir $@))
daemons/ganeti-watcher: MODULE = ganeti.watcher
scripts/%: MODULE = ganeti.client.$(subst -,_,$(notdir $@))
+tools/ensure-dirs: MODULE = ganeti.tools.ensure_dirs
$(PYTHON_BOOTSTRAP): Makefile | $(all_dirfiles)
test -n "$(MODULE)" || { echo Missing module; exit 1; }
diff --git a/lib/tools/__init__.py b/lib/tools/__init__.py
new file mode 100644
index 000000000..99df0ed0d
--- /dev/null
+++ b/lib/tools/__init__.py
@@ -0,0 +1,20 @@
+# Copyright (C) 2011 Google Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+# 02110-1301, USA.
+
+"""Common tools modules.
+
+"""
diff --git a/lib/tools/ensure_dirs.py b/lib/tools/ensure_dirs.py
new file mode 100644
index 000000000..8de0cbf4d
--- /dev/null
+++ b/lib/tools/ensure_dirs.py
@@ -0,0 +1,252 @@
+# Copyright (C) 2011 Google Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+# 02110-1301, USA.
+
+"""Script to ensure permissions on files/dirs are accurate.
+
+"""
+
+import errno
+import os
+import os.path
+import optparse
+import sys
+import stat
+
+from ganeti import constants
+from ganeti import errors
+from ganeti import runtime
+from ganeti import ssconf
+
+
+(DIR, FILE) = range(2)
+ALL_TYPES = frozenset([DIR, FILE])
+
+
+class EnsureError(errors.GenericError):
+ """Top-level error class related to this script.
+
+ """
+
+
+def EnsurePermission(path, mode, uid=-1, gid=-1, must_exist=True,
+ _chmod_fn=os.chmod, _chown_fn=os.chown):
+ """Ensures that given path has given mode.
+
+ @param path: The path to the file
+ @param mode: The mode of the file
+ @param uid: The uid of the owner of this file
+ @param gid: The gid of the owner of this file
+ @param must_exist: Specifies if non-existance of path will be an error
+ @param _chmod_fn: chmod function to use (unittest only)
+ @param _chown_fn: chown function to use (unittest only)
+
+ """
+ try:
+ _chmod_fn(path, mode)
+
+ if max(uid, gid) > -1:
+ _chown_fn(path, uid, gid)
+ except EnvironmentError, err:
+ if err.errno == errno.ENOENT:
+ if must_exist:
+ raise EnsureError("Path %s does not exists, but should" % path)
+ else:
+ raise EnsureError("Error while changing permission on %s: %s" %
+ (path, err))
+
+
+def EnsureDir(path, mode, uid, gid, _stat_fn=os.lstat, _mkdir_fn=os.mkdir,
+ _ensure_fn=EnsurePermission):
+ """Ensures that given path is a dir and has given mode, uid and gid set.
+
+ @param path: The path to the file
+ @param mode: The mode of the file
+ @param uid: The uid of the owner of this file
+ @param gid: The gid of the owner of this file
+ @param _stat_fn: Stat function to use (unittest only)
+ @param _mkdir_fn: mkdir function to use (unittest only)
+ @param _ensure_fn: ensure function to use (unittest only)
+
+ """
+ try:
+ # We don't want to follow symlinks
+ st_mode = _stat_fn(path)[stat.ST_MODE]
+
+ if not stat.S_ISDIR(st_mode):
+ raise EnsureError("Path %s is expected to be a directory, but it's not" %
+ path)
+ except EnvironmentError, err:
+ if err.errno == errno.ENOENT:
+ _mkdir_fn(path)
+ else:
+ raise EnsureError("Error while do a stat() on %s: %s" % (path, err))
+
+ _ensure_fn(path, mode, uid=uid, gid=gid)
+
+
+def RecursiveEnsure(path, uid, gid, dir_perm, file_perm):
+ """Ensures permissions recursively down a directory.
+
+ This functions walks the path and sets permissions accordingly.
+
+ @param path: The absolute path to walk
+ @param uid: The uid used as owner
+ @param gid: The gid used as group
+ @param dir_perm: The permission bits set for directories
+ @param file_perm: The permission bits set for files
+
+ """
+ assert os.path.isabs(path), "Path %s is not absolute" % path
+ assert os.path.isdir(path), "Path %s is not a dir" % path
+
+ for root, dirs, files in os.walk(path):
+ for subdir in dirs:
+ EnsurePermission(os.path.join(root, subdir), dir_perm, uid=uid, gid=gid)
+
+ for filename in files:
+ EnsurePermission(os.path.join(root, filename), file_perm, uid=uid,
+ gid=gid)
+
+
+def ProcessPath(path):
+ """Processes a path component.
+
+ @param path: A tuple of the path component to process
+
+ """
+ (pathname, pathtype, mode, uid, gid) = path[0:5]
+
+ assert pathtype in ALL_TYPES
+
+ if pathtype == DIR:
+ # No additional parameters
+ assert len(path[5:]) == 0
+ EnsureDir(pathname, mode, uid, gid)
+ elif pathtype == FILE:
+ (must_exist, ) = path[5:]
+ EnsurePermission(pathname, mode, uid=uid, gid=gid, must_exist=must_exist)
+
+
+def GetPaths():
+ """Returns a tuple of path objects to process.
+
+ """
+ getent = runtime.GetEnts()
+ masterd_log = constants.DAEMONS_LOGFILES[constants.MASTERD]
+ noded_log = constants.DAEMONS_LOGFILES[constants.NODED]
+ confd_log = constants.DAEMONS_LOGFILES[constants.CONFD]
+ rapi_log = constants.DAEMONS_LOGFILES[constants.RAPI]
+
+ rapi_dir = os.path.join(constants.DATA_DIR, "rapi")
+
+ paths = [
+ (constants.DATA_DIR, DIR, 0755, getent.masterd_uid,
+ getent.masterd_gid),
+ (constants.CLUSTER_DOMAIN_SECRET_FILE, FILE, 0640,
+ getent.masterd_uid, getent.masterd_gid, False),
+ (constants.CLUSTER_CONF_FILE, FILE, 0640, getent.masterd_uid,
+ getent.confd_gid, False),
+ (constants.CONFD_HMAC_KEY, FILE, 0440, getent.confd_uid,
+ getent.masterd_gid, False),
+ (constants.SSH_KNOWN_HOSTS_FILE, FILE, 0644, getent.masterd_uid,
+ getent.masterd_gid, False),
+ (constants.RAPI_CERT_FILE, FILE, 0440, getent.rapi_uid,
+ getent.masterd_gid, False),
+ (constants.NODED_CERT_FILE, FILE, 0440, getent.masterd_uid,
+ getent.masterd_gid, False),
+ ]
+
+ ss = ssconf.SimpleStore()
+ for ss_path in ss.GetFileList():
+ paths.append((ss_path, FILE, 0400, getent.noded_uid, 0, False))
+
+ paths.extend([
+ (constants.QUEUE_DIR, DIR, 0700, getent.masterd_uid,
+ getent.masterd_gid),
+ (constants.JOB_QUEUE_SERIAL_FILE, FILE, 0600,
+ getent.masterd_uid, getent.masterd_gid, False),
+ (constants.JOB_QUEUE_ARCHIVE_DIR, DIR, 0700,
+ getent.masterd_uid, getent.masterd_gid),
+ (rapi_dir, DIR, 0750, getent.rapi_uid, getent.masterd_gid),
+ (constants.RAPI_USERS_FILE, FILE, 0640, getent.rapi_uid,
+ getent.masterd_gid, False),
+ (constants.RUN_GANETI_DIR, DIR, 0775, getent.masterd_uid,
+ getent.daemons_gid),
+ (constants.SOCKET_DIR, DIR, 0750, getent.masterd_uid,
+ getent.daemons_gid),
+ (constants.MASTER_SOCKET, FILE, 0770, getent.masterd_uid,
+ getent.daemons_gid, False),
+ (constants.BDEV_CACHE_DIR, DIR, 0755, getent.noded_uid,
+ getent.masterd_gid),
+ (constants.UIDPOOL_LOCKDIR, DIR, 0750, getent.noded_uid,
+ getent.masterd_gid),
+ (constants.DISK_LINKS_DIR, DIR, 0755, getent.noded_uid,
+ getent.masterd_gid),
+ (constants.CRYPTO_KEYS_DIR, DIR, 0700, getent.noded_uid,
+ getent.masterd_gid),
+ (constants.IMPORT_EXPORT_DIR, DIR, 0755, getent.noded_uid,
+ getent.masterd_gid),
+ (constants.LOG_DIR, DIR, 0770, getent.masterd_uid,
+ getent.daemons_gid),
+ (masterd_log, FILE, 0600, getent.masterd_uid, getent.masterd_gid,
+ False),
+ (confd_log, FILE, 0600, getent.confd_uid, getent.masterd_gid, False),
+ (noded_log, FILE, 0600, getent.noded_uid, getent.masterd_gid, False),
+ (rapi_log, FILE, 0600, getent.rapi_uid, getent.masterd_gid, False),
+ (constants.LOG_OS_DIR, DIR, 0750, getent.masterd_uid,
+ getent.daemons_gid),
+ ])
+
+ return tuple(paths)
+
+
+def ParseOptions():
+ """Parses the options passed to the program.
+
+ @return: Options and arguments
+
+ """
+ program = os.path.basename(sys.argv[0])
+
+ parser = optparse.OptionParser(usage="%%prog [--full-run]",
+ prog=program)
+ parser.add_option("--full-run", "-f", dest="full_run", action="store_true",
+ default=False, help=("Make a full run and collect"
+ " additional files (time consuming)"))
+
+ return parser.parse_args()
+
+
+def Main():
+ """Main routine.
+
+ """
+ getent = runtime.GetEnts()
+ (opts, _) = ParseOptions()
+
+ try:
+ for path in GetPaths():
+ ProcessPath(path)
+
+ if opts.full_run:
+ RecursiveEnsure(constants.JOB_QUEUE_ARCHIVE_DIR, getent.masterd_uid,
+ getent.masterd_gid, 0700, 0600)
+ except EnsureError, err:
+ print >> sys.stderr, "An error occurred while ensure permissions:", err
+ return constants.EXIT_FAILURE
+
+ return constants.EXIT_SUCCESS
diff --git a/test/ganeti.tools.ensure_dirs_unittest.py b/test/ganeti.tools.ensure_dirs_unittest.py
new file mode 100755
index 000000000..6e0dff3c7
--- /dev/null
+++ b/test/ganeti.tools.ensure_dirs_unittest.py
@@ -0,0 +1,143 @@
+#!/usr/bin/python
+#
+
+# Copyright (C) 2011 Google Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+# 02110-1301, USA.
+
+
+"""Script for testing ganeti.tools.ensure_dirs"""
+
+import errno
+import stat
+import unittest
+import os.path
+
+from ganeti.tools import ensure_dirs
+
+import testutils
+
+
+class TestEnsureDirsFunctions(unittest.TestCase):
+ def _NoopMkdir(self, _):
+ self.mkdir_called = True
+
+ @staticmethod
+ def _MockStatResult(mode, pre_fn=lambda: 0):
+ def _fn(path):
+ pre_fn()
+ return {stat.ST_MODE: mode}
+ return _fn
+
+ def _VerifyEnsure(self, path, mode, uid=-1, gid=-1):
+ self.assertEqual(path, "/ganeti-qa-non-test")
+ self.assertEqual(mode, 0700)
+ self.assertEqual(uid, 0)
+ self.assertEqual(gid, 0)
+
+ @staticmethod
+ def _RaiseNoEntError():
+ noent_error = EnvironmentError()
+ noent_error.errno = errno.ENOENT
+ raise noent_error
+
+ @staticmethod
+ def _OtherStatRaise():
+ raise EnvironmentError()
+
+ def _ChmodWrapper(self, pre_fn=lambda: 0):
+ def _fn(path, mode):
+ self.chmod_called = True
+ pre_fn()
+ return _fn
+
+ def _NoopChown(self, path, uid, gid):
+ self.chown_called = True
+
+ def testEnsureDir(self):
+ is_dir_stat = self._MockStatResult(stat.S_IFDIR)
+ not_dir_stat = self._MockStatResult(0)
+ non_exist_stat = self._MockStatResult(stat.S_IFDIR,
+ pre_fn=self._RaiseNoEntError)
+ other_stat_raise = self._MockStatResult(stat.S_IFDIR,
+ pre_fn=self._OtherStatRaise)
+
+ self.assertRaises(ensure_dirs.EnsureError, ensure_dirs.EnsureDir,
+ "/ganeti-qa-non-test", 0700, 0, 0,
+ _stat_fn=not_dir_stat)
+ self.assertRaises(ensure_dirs.EnsureError, ensure_dirs.EnsureDir,
+ "/ganeti-qa-non-test", 0700, 0, 0,
+ _stat_fn=other_stat_raise)
+ self.mkdir_called = False
+ ensure_dirs.EnsureDir("/ganeti-qa-non-test", 0700, 0, 0,
+ _stat_fn=non_exist_stat, _mkdir_fn=self._NoopMkdir,
+ _ensure_fn=self._VerifyEnsure)
+ self.assertTrue(self.mkdir_called)
+ self.mkdir_called = False
+ ensure_dirs.EnsureDir("/ganeti-qa-non-test", 0700, 0, 0,
+ _stat_fn=is_dir_stat, _ensure_fn=self._VerifyEnsure)
+ self.assertFalse(self.mkdir_called)
+
+ def testEnsurePermission(self):
+ noent_chmod_fn = self._ChmodWrapper(pre_fn=self._RaiseNoEntError)
+ self.assertRaises(ensure_dirs.EnsureError, ensure_dirs.EnsurePermission,
+ "/ganeti-qa-non-test", 0600,
+ _chmod_fn=noent_chmod_fn)
+ self.chmod_called = False
+ ensure_dirs.EnsurePermission("/ganeti-qa-non-test", 0600, must_exist=False,
+ _chmod_fn=noent_chmod_fn)
+ self.assertTrue(self.chmod_called)
+ self.assertRaises(ensure_dirs.EnsureError, ensure_dirs.EnsurePermission,
+ "/ganeti-qa-non-test", 0600, must_exist=False,
+ _chmod_fn=self._ChmodWrapper(pre_fn=self._OtherStatRaise))
+ self.chmod_called = False
+ self.chown_called = False
+ ensure_dirs.EnsurePermission("/ganeti-qa-non-test", 0600,
+ _chmod_fn=self._ChmodWrapper(),
+ _chown_fn=self._NoopChown)
+ self.assertTrue(self.chmod_called)
+ self.assertFalse(self.chown_called)
+ self.chmod_called = False
+ ensure_dirs.EnsurePermission("/ganeti-qa-non-test", 0600, uid=1, gid=1,
+ _chmod_fn=self._ChmodWrapper(),
+ _chown_fn=self._NoopChown)
+ self.assertTrue(self.chmod_called)
+ self.assertTrue(self.chown_called)
+
+ def testPaths(self):
+ paths = [(path[0], path[1]) for path in ensure_dirs.GetPaths()]
+
+ seen = []
+ last_dirname = ""
+ for path, pathtype in paths:
+ self.assertTrue(pathtype in ensure_dirs.ALL_TYPES)
+ dirname = os.path.dirname(path)
+ if dirname != last_dirname or pathtype == ensure_dirs.DIR:
+ if pathtype == ensure_dirs.FILE:
+ self.assertFalse(dirname in seen,
+ msg="path %s; dirname %s seen in %s" % (path,
+ dirname,
+ seen))
+ last_dirname = dirname
+ seen.append(dirname)
+ elif pathtype == ensure_dirs.DIR:
+ self.assertFalse(path in seen)
+ last_dirname = path
+ seen.append(path)
+
+
+if __name__ == "__main__":
+ testutils.GanetiTestProgram()
--
GitLab