Commit 0d2bf835 authored by René Nussbaumer's avatar René Nussbaumer

Rewrite of ensure-dirs in python

I provided unittest to test the important pieces of the infrastructure.
The one remaining function (ResuriveEnsure) is not easy to unittest
but also not critical if it fails to operate correctly.
Signed-off-by: default avatarRené Nussbaumer <rn@google.com>
Reviewed-by: default avatarIustin Pop <iustin@google.com>
parent 07dde0c1
......@@ -82,6 +82,7 @@
# tools
/tools/kvm-ifup
/tools/ensure-dirs
# scripts
/scripts/gnt-backup
......
......@@ -36,6 +36,7 @@ impexpddir = $(pkgpythondir)/impexpd
utilsdir = $(pkgpythondir)/utils
toolsdir = $(pkglibdir)/tools
iallocatorsdir = $(pkglibdir)/iallocators
pytoolsdir = $(pkgpythondir)/tools
docdir = $(datadir)/doc/$(PACKAGE)
# Delete output file if an error occurred while building it
......@@ -65,6 +66,7 @@ DIRS = \
lib/masterd \
lib/rapi \
lib/server \
lib/tools \
lib/utils \
lib/watcher \
man \
......@@ -247,6 +249,10 @@ server_PYTHON = \
lib/server/noded.py \
lib/server/rapi.py
pytools_PYTHON = \
lib/tools/__init__.py \
lib/tools/ensure_dirs.py
utils_PYTHON = \
lib/utils/__init__.py \
lib/utils/algo.py \
......@@ -399,7 +405,7 @@ gnt_scripts = \
scripts/gnt-node \
scripts/gnt-os
PYTHON_BOOTSTRAP = \
PYTHON_BOOTSTRAP_SBIN = \
daemons/ganeti-confd \
daemons/ganeti-masterd \
daemons/ganeti-noded \
......@@ -414,6 +420,10 @@ PYTHON_BOOTSTRAP = \
scripts/gnt-node \
scripts/gnt-os
PYTHON_BOOTSTRAP = \
$(PYTHON_BOOTSTRAP_SBIN) \
tools/ensure-dirs
qa_scripts = \
qa/ganeti-qa.py \
qa/qa_cluster.py \
......@@ -451,7 +461,7 @@ dist_sbin_SCRIPTS = \
tools/ganeti-listrunner
nodist_sbin_SCRIPTS = \
$(PYTHON_BOOTSTRAP) \
$(PYTHON_BOOTSTRAP_SBIN) \
daemons/ganeti-cleaner
dist_tools_SCRIPTS = \
......@@ -469,12 +479,18 @@ pkglib_python_scripts = \
daemons/import-export \
tools/check-cert-expired
nodist_pkglib_python_scripts = \
tools/ensure-dirs
pkglib_SCRIPTS = \
daemons/daemon-util \
daemons/ensure-dirs \
tools/kvm-ifup \
$(pkglib_python_scripts)
nodist_pkglib_SCRIPTS = \
$(nodist_pkglib_python_scripts)
EXTRA_DIST = \
NEWS \
UPGRADE \
......@@ -610,6 +626,7 @@ python_tests = \
test/ganeti.runtime_unittest.py \
test/ganeti.serializer_unittest.py \
test/ganeti.ssh_unittest.py \
test/ganeti.tools.ensure_dirs_unittest.py \
test/ganeti.uidpool_unittest.py \
test/ganeti.utils.algo_unittest.py \
test/ganeti.utils.filelock_unittest.py \
......@@ -657,12 +674,14 @@ all_python_code = \
$(dist_sbin_SCRIPTS) \
$(dist_tools_SCRIPTS) \
$(pkglib_python_scripts) \
$(nodist_pkglib_python_scripts) \
$(python_tests) \
$(pkgpython_PYTHON) \
$(client_PYTHON) \
$(hypervisor_PYTHON) \
$(rapi_PYTHON) \
$(server_PYTHON) \
$(pytools_PYTHON) \
$(http_PYTHON) \
$(confd_PYTHON) \
$(masterd_PYTHON) \
......@@ -881,6 +900,7 @@ $(REPLACE_VARS_SED): Makefile
daemons/ganeti-%: MODULE = ganeti.server.$(patsubst ganeti-%,%,$(notdir $@))
daemons/ganeti-watcher: MODULE = ganeti.watcher
scripts/%: MODULE = ganeti.client.$(subst -,_,$(notdir $@))
tools/ensure-dirs: MODULE = ganeti.tools.ensure_dirs
$(PYTHON_BOOTSTRAP): Makefile | $(all_dirfiles)
test -n "$(MODULE)" || { echo Missing module; exit 1; }
......
# Copyright (C) 2011 Google Inc.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
# 02110-1301, USA.
"""Common tools modules.
"""
# Copyright (C) 2011 Google Inc.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
# 02110-1301, USA.
"""Script to ensure permissions on files/dirs are accurate.
"""
import errno
import os
import os.path
import optparse
import sys
import stat
from ganeti import constants
from ganeti import errors
from ganeti import runtime
from ganeti import ssconf
(DIR, FILE) = range(2)
ALL_TYPES = frozenset([DIR, FILE])
class EnsureError(errors.GenericError):
"""Top-level error class related to this script.
"""
def EnsurePermission(path, mode, uid=-1, gid=-1, must_exist=True,
_chmod_fn=os.chmod, _chown_fn=os.chown):
"""Ensures that given path has given mode.
@param path: The path to the file
@param mode: The mode of the file
@param uid: The uid of the owner of this file
@param gid: The gid of the owner of this file
@param must_exist: Specifies if non-existance of path will be an error
@param _chmod_fn: chmod function to use (unittest only)
@param _chown_fn: chown function to use (unittest only)
"""
try:
_chmod_fn(path, mode)
if max(uid, gid) > -1:
_chown_fn(path, uid, gid)
except EnvironmentError, err:
if err.errno == errno.ENOENT:
if must_exist:
raise EnsureError("Path %s does not exists, but should" % path)
else:
raise EnsureError("Error while changing permission on %s: %s" %
(path, err))
def EnsureDir(path, mode, uid, gid, _stat_fn=os.lstat, _mkdir_fn=os.mkdir,
_ensure_fn=EnsurePermission):
"""Ensures that given path is a dir and has given mode, uid and gid set.
@param path: The path to the file
@param mode: The mode of the file
@param uid: The uid of the owner of this file
@param gid: The gid of the owner of this file
@param _stat_fn: Stat function to use (unittest only)
@param _mkdir_fn: mkdir function to use (unittest only)
@param _ensure_fn: ensure function to use (unittest only)
"""
try:
# We don't want to follow symlinks
st_mode = _stat_fn(path)[stat.ST_MODE]
if not stat.S_ISDIR(st_mode):
raise EnsureError("Path %s is expected to be a directory, but it's not" %
path)
except EnvironmentError, err:
if err.errno == errno.ENOENT:
_mkdir_fn(path)
else:
raise EnsureError("Error while do a stat() on %s: %s" % (path, err))
_ensure_fn(path, mode, uid=uid, gid=gid)
def RecursiveEnsure(path, uid, gid, dir_perm, file_perm):
"""Ensures permissions recursively down a directory.
This functions walks the path and sets permissions accordingly.
@param path: The absolute path to walk
@param uid: The uid used as owner
@param gid: The gid used as group
@param dir_perm: The permission bits set for directories
@param file_perm: The permission bits set for files
"""
assert os.path.isabs(path), "Path %s is not absolute" % path
assert os.path.isdir(path), "Path %s is not a dir" % path
for root, dirs, files in os.walk(path):
for subdir in dirs:
EnsurePermission(os.path.join(root, subdir), dir_perm, uid=uid, gid=gid)
for filename in files:
EnsurePermission(os.path.join(root, filename), file_perm, uid=uid,
gid=gid)
def ProcessPath(path):
"""Processes a path component.
@param path: A tuple of the path component to process
"""
(pathname, pathtype, mode, uid, gid) = path[0:5]
assert pathtype in ALL_TYPES
if pathtype == DIR:
# No additional parameters
assert len(path[5:]) == 0
EnsureDir(pathname, mode, uid, gid)
elif pathtype == FILE:
(must_exist, ) = path[5:]
EnsurePermission(pathname, mode, uid=uid, gid=gid, must_exist=must_exist)
def GetPaths():
"""Returns a tuple of path objects to process.
"""
getent = runtime.GetEnts()
masterd_log = constants.DAEMONS_LOGFILES[constants.MASTERD]
noded_log = constants.DAEMONS_LOGFILES[constants.NODED]
confd_log = constants.DAEMONS_LOGFILES[constants.CONFD]
rapi_log = constants.DAEMONS_LOGFILES[constants.RAPI]
rapi_dir = os.path.join(constants.DATA_DIR, "rapi")
paths = [
(constants.DATA_DIR, DIR, 0755, getent.masterd_uid,
getent.masterd_gid),
(constants.CLUSTER_DOMAIN_SECRET_FILE, FILE, 0640,
getent.masterd_uid, getent.masterd_gid, False),
(constants.CLUSTER_CONF_FILE, FILE, 0640, getent.masterd_uid,
getent.confd_gid, False),
(constants.CONFD_HMAC_KEY, FILE, 0440, getent.confd_uid,
getent.masterd_gid, False),
(constants.SSH_KNOWN_HOSTS_FILE, FILE, 0644, getent.masterd_uid,
getent.masterd_gid, False),
(constants.RAPI_CERT_FILE, FILE, 0440, getent.rapi_uid,
getent.masterd_gid, False),
(constants.NODED_CERT_FILE, FILE, 0440, getent.masterd_uid,
getent.masterd_gid, False),
]
ss = ssconf.SimpleStore()
for ss_path in ss.GetFileList():
paths.append((ss_path, FILE, 0400, getent.noded_uid, 0, False))
paths.extend([
(constants.QUEUE_DIR, DIR, 0700, getent.masterd_uid,
getent.masterd_gid),
(constants.JOB_QUEUE_SERIAL_FILE, FILE, 0600,
getent.masterd_uid, getent.masterd_gid, False),
(constants.JOB_QUEUE_ARCHIVE_DIR, DIR, 0700,
getent.masterd_uid, getent.masterd_gid),
(rapi_dir, DIR, 0750, getent.rapi_uid, getent.masterd_gid),
(constants.RAPI_USERS_FILE, FILE, 0640, getent.rapi_uid,
getent.masterd_gid, False),
(constants.RUN_GANETI_DIR, DIR, 0775, getent.masterd_uid,
getent.daemons_gid),
(constants.SOCKET_DIR, DIR, 0750, getent.masterd_uid,
getent.daemons_gid),
(constants.MASTER_SOCKET, FILE, 0770, getent.masterd_uid,
getent.daemons_gid, False),
(constants.BDEV_CACHE_DIR, DIR, 0755, getent.noded_uid,
getent.masterd_gid),
(constants.UIDPOOL_LOCKDIR, DIR, 0750, getent.noded_uid,
getent.masterd_gid),
(constants.DISK_LINKS_DIR, DIR, 0755, getent.noded_uid,
getent.masterd_gid),
(constants.CRYPTO_KEYS_DIR, DIR, 0700, getent.noded_uid,
getent.masterd_gid),
(constants.IMPORT_EXPORT_DIR, DIR, 0755, getent.noded_uid,
getent.masterd_gid),
(constants.LOG_DIR, DIR, 0770, getent.masterd_uid,
getent.daemons_gid),
(masterd_log, FILE, 0600, getent.masterd_uid, getent.masterd_gid,
False),
(confd_log, FILE, 0600, getent.confd_uid, getent.masterd_gid, False),
(noded_log, FILE, 0600, getent.noded_uid, getent.masterd_gid, False),
(rapi_log, FILE, 0600, getent.rapi_uid, getent.masterd_gid, False),
(constants.LOG_OS_DIR, DIR, 0750, getent.masterd_uid,
getent.daemons_gid),
])
return tuple(paths)
def ParseOptions():
"""Parses the options passed to the program.
@return: Options and arguments
"""
program = os.path.basename(sys.argv[0])
parser = optparse.OptionParser(usage="%%prog [--full-run]",
prog=program)
parser.add_option("--full-run", "-f", dest="full_run", action="store_true",
default=False, help=("Make a full run and collect"
" additional files (time consuming)"))
return parser.parse_args()
def Main():
"""Main routine.
"""
getent = runtime.GetEnts()
(opts, _) = ParseOptions()
try:
for path in GetPaths():
ProcessPath(path)
if opts.full_run:
RecursiveEnsure(constants.JOB_QUEUE_ARCHIVE_DIR, getent.masterd_uid,
getent.masterd_gid, 0700, 0600)
except EnsureError, err:
print >> sys.stderr, "An error occurred while ensure permissions:", err
return constants.EXIT_FAILURE
return constants.EXIT_SUCCESS
#!/usr/bin/python
#
# Copyright (C) 2011 Google Inc.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
# 02110-1301, USA.
"""Script for testing ganeti.tools.ensure_dirs"""
import errno
import stat
import unittest
import os.path
from ganeti.tools import ensure_dirs
import testutils
class TestEnsureDirsFunctions(unittest.TestCase):
def _NoopMkdir(self, _):
self.mkdir_called = True
@staticmethod
def _MockStatResult(mode, pre_fn=lambda: 0):
def _fn(path):
pre_fn()
return {stat.ST_MODE: mode}
return _fn
def _VerifyEnsure(self, path, mode, uid=-1, gid=-1):
self.assertEqual(path, "/ganeti-qa-non-test")
self.assertEqual(mode, 0700)
self.assertEqual(uid, 0)
self.assertEqual(gid, 0)
@staticmethod
def _RaiseNoEntError():
noent_error = EnvironmentError()
noent_error.errno = errno.ENOENT
raise noent_error
@staticmethod
def _OtherStatRaise():
raise EnvironmentError()
def _ChmodWrapper(self, pre_fn=lambda: 0):
def _fn(path, mode):
self.chmod_called = True
pre_fn()
return _fn
def _NoopChown(self, path, uid, gid):
self.chown_called = True
def testEnsureDir(self):
is_dir_stat = self._MockStatResult(stat.S_IFDIR)
not_dir_stat = self._MockStatResult(0)
non_exist_stat = self._MockStatResult(stat.S_IFDIR,
pre_fn=self._RaiseNoEntError)
other_stat_raise = self._MockStatResult(stat.S_IFDIR,
pre_fn=self._OtherStatRaise)
self.assertRaises(ensure_dirs.EnsureError, ensure_dirs.EnsureDir,
"/ganeti-qa-non-test", 0700, 0, 0,
_stat_fn=not_dir_stat)
self.assertRaises(ensure_dirs.EnsureError, ensure_dirs.EnsureDir,
"/ganeti-qa-non-test", 0700, 0, 0,
_stat_fn=other_stat_raise)
self.mkdir_called = False
ensure_dirs.EnsureDir("/ganeti-qa-non-test", 0700, 0, 0,
_stat_fn=non_exist_stat, _mkdir_fn=self._NoopMkdir,
_ensure_fn=self._VerifyEnsure)
self.assertTrue(self.mkdir_called)
self.mkdir_called = False
ensure_dirs.EnsureDir("/ganeti-qa-non-test", 0700, 0, 0,
_stat_fn=is_dir_stat, _ensure_fn=self._VerifyEnsure)
self.assertFalse(self.mkdir_called)
def testEnsurePermission(self):
noent_chmod_fn = self._ChmodWrapper(pre_fn=self._RaiseNoEntError)
self.assertRaises(ensure_dirs.EnsureError, ensure_dirs.EnsurePermission,
"/ganeti-qa-non-test", 0600,
_chmod_fn=noent_chmod_fn)
self.chmod_called = False
ensure_dirs.EnsurePermission("/ganeti-qa-non-test", 0600, must_exist=False,
_chmod_fn=noent_chmod_fn)
self.assertTrue(self.chmod_called)
self.assertRaises(ensure_dirs.EnsureError, ensure_dirs.EnsurePermission,
"/ganeti-qa-non-test", 0600, must_exist=False,
_chmod_fn=self._ChmodWrapper(pre_fn=self._OtherStatRaise))
self.chmod_called = False
self.chown_called = False
ensure_dirs.EnsurePermission("/ganeti-qa-non-test", 0600,
_chmod_fn=self._ChmodWrapper(),
_chown_fn=self._NoopChown)
self.assertTrue(self.chmod_called)
self.assertFalse(self.chown_called)
self.chmod_called = False
ensure_dirs.EnsurePermission("/ganeti-qa-non-test", 0600, uid=1, gid=1,
_chmod_fn=self._ChmodWrapper(),
_chown_fn=self._NoopChown)
self.assertTrue(self.chmod_called)
self.assertTrue(self.chown_called)
def testPaths(self):
paths = [(path[0], path[1]) for path in ensure_dirs.GetPaths()]
seen = []
last_dirname = ""
for path, pathtype in paths:
self.assertTrue(pathtype in ensure_dirs.ALL_TYPES)
dirname = os.path.dirname(path)
if dirname != last_dirname or pathtype == ensure_dirs.DIR:
if pathtype == ensure_dirs.FILE:
self.assertFalse(dirname in seen,
msg="path %s; dirname %s seen in %s" % (path,
dirname,
seen))
last_dirname = dirname
seen.append(dirname)
elif pathtype == ensure_dirs.DIR:
self.assertFalse(path in seen)
last_dirname = path
seen.append(path)
if __name__ == "__main__":
testutils.GanetiTestProgram()
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment