Commit bb3776b4 authored by Michael Hanselmann's avatar Michael Hanselmann
Browse files

Merge remote branch 'origin/devel-2.1'



* origin/devel-2.1: (116 commits)
  Implement replacing cluster certs and keys via “gnt-cluster renew-crypto”
  cli: Add helper function to stop and start whole cluster
  cfgupgrade: Use new bootstrap function for certs and keys
  bootstrap: Add new function to create cluster certs and keys
  utils.CreateBackup: Use human-readable instead of seconds since Epoch
  Add unittest for daemon-util
  Add support for non-Python unittests
  daemon-util: Generate daemon path in separate function
  daemon-util: Use “return” instead of “exit” in all functions
  daemon-util: Add function to start and stop all daemons
  ganeti.initd: Move all daemon names from init script to daemon-util
  ganeti.initd: Move code checking daemon exit code to daemon-util
  ganeti.initd: Move code checking config to daemon-util
  daemon-util: Require dashes in commands
  Improve ganeti.serializer unittests
  Add unittests for ganeti.errors
  Verify cluster certificates in LUVerifyCluster
  utils: Add function to extract X509 cert validity
  Add constant with cluster X509 certificates
  Release version 2.1.1
  ...

Conflicts:
	lib/backend.py: Trivial
	lib/bootstrap.py: Trivial
	lib/constants.py: Trivial
	lib/http/server.py: Trivial
	lib/utils.py: RunCmd parameter “reset_env”
	test/ganeti.utils_unittest.py: Trivial
	tools/cfgupgrade: Trivial
Signed-off-by: default avatarMichael Hanselmann <hansmi@google.com>
Reviewed-by: default avatarGuido Trotter <ultrotter@google.com>
parents f0476905 6d4a1656
......@@ -41,6 +41,7 @@
# doc
/doc/api
/doc/build
/doc/coverage
/doc/html
/doc/install-quick.rst
/doc/news.rst
......
......@@ -54,7 +54,7 @@ MAINTAINERCLEANFILES = \
doc/news.rst
maintainer-clean-local:
rm -rf doc/api doc/html
rm -rf doc/api doc/html doc/coverage
CLEANFILES = \
autotools/replace_vars.sed \
......@@ -235,6 +235,7 @@ dist_tools_SCRIPTS = \
tools/burnin \
tools/cfgshell \
tools/cfgupgrade \
tools/cluster-merge \
tools/lvmstrap
pkglib_SCRIPTS = \
......@@ -247,6 +248,8 @@ EXTRA_DIST = \
autotools/check-python-code \
autotools/check-man \
autotools/docbook-wrapper \
autotools/gen-coverage \
autotools/testrunner \
$(RUN_IN_TEMPDIR) \
daemons/daemon-util.in \
daemons/ganeti-cleaner.in \
......@@ -313,23 +316,26 @@ TEST_FILES = \
test/data/bdev-8.3-both.txt \
test/data/bdev-disk.txt \
test/data/bdev-net.txt \
test/data/cert1.pem \
test/data/proc_drbd8.txt \
test/data/proc_drbd80-emptyline.txt \
test/data/proc_drbd83.txt
dist_TESTS = \
python_tests = \
test/ganeti.bdev_unittest.py \
test/ganeti.cli_unittest.py \
test/ganeti.cmdlib_unittest.py \
test/ganeti.confd_client_unittest.py \
test/ganeti.config_unittest.py \
test/ganeti.constants_unittest.py \
test/ganeti.errors_unittest.py \
test/ganeti.hooks_unittest.py \
test/ganeti.http_unittest.py \
test/ganeti.locking_unittest.py \
test/ganeti.luxi_unittest.py \
test/ganeti.mcpu_unittest.py \
test/ganeti.objects_unittest.py \
test/ganeti.opcodes_unittest.py \
test/ganeti.rapi.resources_unittest.py \
test/ganeti.serializer_unittest.py \
test/ganeti.ssh_unittest.py \
......@@ -338,18 +344,26 @@ dist_TESTS = \
test/docs_unittest.py \
test/tempfile_fork_unittest.py
dist_TESTS = \
test/daemon-util_unittest.bash \
$(python_tests)
nodist_TESTS =
TESTS = $(dist_TESTS) $(nodist_TESTS)
# Environment for all tests
PLAIN_TESTS_ENVIRONMENT = \
PYTHONPATH=. TOP_SRCDIR=$(abs_top_srcdir) PYTHON=$(PYTHON) $(RUN_IN_TEMPDIR)
# Environment for tests run by automake
TESTS_ENVIRONMENT = \
PYTHONPATH=. TOP_SRCDIR=$(abs_top_srcdir) \
$(RUN_IN_TEMPDIR) $(PYTHON)
$(PLAIN_TESTS_ENVIRONMENT) $(abs_top_srcdir)/autotools/testrunner
all_python_code = \
$(dist_sbin_SCRIPTS) \
$(dist_tools_SCRIPTS) \
$(dist_TESTS) \
$(python_tests) \
$(pkgpython_PYTHON) \
$(hypervisor_PYTHON) \
$(rapi_PYTHON) \
......@@ -359,6 +373,7 @@ all_python_code = \
srclink_files = \
man/footer.sgml \
test/daemon-util_unittest.bash \
$(all_python_code)
check_python_code = \
......@@ -371,6 +386,8 @@ lint_python_code = \
$(dist_tools_SCRIPTS) \
$(BUILD_BASH_COMPLETION)
test/daemon-util_unittest.bash: daemons/daemon-util
devel/upload: devel/upload.in $(REPLACE_VARS_SED)
sed -f $(REPLACE_VARS_SED) < $< > $@
chmod u+x $@
......@@ -442,12 +459,14 @@ lib/_autoconf.py: Makefile stamp-directories
echo "VERSION_FULL = '$(VERSION_FULL)'"; \
echo "LOCALSTATEDIR = '$(localstatedir)'"; \
echo "SYSCONFDIR = '$(sysconfdir)'"; \
echo "SSH_CONFIG_DIR = '$(SSH_CONFIG_DIR)'"; \
echo "EXPORT_DIR = '$(EXPORT_DIR)'"; \
echo "OS_SEARCH_PATH = [$(OS_SEARCH_PATH)]"; \
echo "XEN_BOOTLOADER = '$(XEN_BOOTLOADER)'"; \
echo "XEN_KERNEL = '$(XEN_KERNEL)'"; \
echo "XEN_INITRD = '$(XEN_INITRD)'"; \
echo "FILE_STORAGE_DIR = '$(FILE_STORAGE_DIR)'"; \
echo "ENABLE_FILE_STORAGE = $(ENABLE_FILE_STORAGE)"; \
echo "IALLOCATOR_SEARCH_PATH = [$(IALLOCATOR_SEARCH_PATH)]"; \
echo "KVM_PATH = '$(KVM_PATH)'"; \
echo "SOCAT_PATH = '$(SOCAT)'"; \
......@@ -537,4 +556,14 @@ TAGS: $(BUILT_SOURCES)
-path 'daemons/ganeti-*' -o -path 'tools/*' | \
etags -
.PHONY: coverage
coverage: $(BUILT_SOURCES) $(python_tests)
set -e; \
mkdir -p doc/coverage; \
COVERAGE_FILE=$(CURDIR)/doc/coverage/data \
TEXT_COVERAGE=$(CURDIR)/doc/coverage/report.txt \
HTML_COVERAGE=$(CURDIR)/doc/coverage \
$(PLAIN_TESTS_ENVIRONMENT) $(abs_top_srcdir)/autotools/gen-coverage \
$(python_tests)
# vim: set noet :
......@@ -9,6 +9,120 @@ Version 2.2.0
RFC2616 (HTTP/1.1), section 7.2.1)
Version 2.1.1
-------------
During the 2.1.0 long release candidate cycle, a lot of improvements and
changes have accumulated with were released later as 2.1.1.
Major changes
~~~~~~~~~~~~~
The node evacuate command (``gnt-node evacuate``) was significantly
rewritten, and as such the IAllocator protocol was changed - a new
request type has been added. This unfortunate change during a stable
series is designed to improve performance of node evacuations; on
clusters with more than about five nodes and which are well-balanced,
evacuation should proceed in parallel for all instances of the node
being evacuated. As such, any existing IAllocator scripts need to be
updated, otherwise the above command will fail due to the unknown
request. The provided "dumb" allocator has not been updated; but the
ganeti-htools package supports the new protocol since version 0.2.4.
Another important change is increased validation of node and instance
names. This might create problems in special cases, if invalid host
names are being used.
Also, a new layer of hypervisor parameters has been added, that sits at
OS level between the cluster defaults and the instance ones. This allows
customisation of virtualization parameters depending on the installed
OS. For example instances with OS 'X' may have a different KVM kernel
(or any other parameter) than the cluster defaults. This is intended to
help managing a multiple OSes on the same cluster, without manual
modification of each instance's parameters.
A tool for merging clusters, ``cluster-merge``, has been added in the
tools sub-directory.
Bug fixes
~~~~~~~~~
- Improved the int/float conversions that should make the code more
robust in face of errors from the node daemons
- Fixed the remove node code in case of internal configuration errors
- Fixed the node daemon behaviour in face of inconsistent queue
directory (e.g. read-only file-system where we can't open the files
read-write, etc.)
- Fixed the behaviour of gnt-node modify for master candidate demotion;
now it either aborts cleanly or, if given the new “auto_promote”
parameter, will automatically promote other nodes as needed
- Fixed compatibility with (unreleased yet) Python 2.6.5 that would
completely prevent Ganeti from working
- Fixed bug for instance export when not all disks were successfully
exported
- Fixed behaviour of node add when the new node is slow in starting up
the node daemon
- Fixed handling of signals in the LUXI client, which should improve
behaviour of command-line scripts
- Added checks for invalid node/instance names in the configuration (now
flagged during cluster verify)
- Fixed watcher behaviour for disk activation errors
- Fixed two potentially endless loops in http library, which led to the
RAPI daemon hanging and consuming 100% CPU in some cases
- Fixed bug in RAPI daemon related to hashed passwords
- Fixed bug for unintended qemu-level bridging of multi-NIC KVM
instances
- Enhanced compatibility with non-Debian OSes, but not using absolute
path in some commands and allowing customisation of the ssh
configuration directory
- Fixed possible future issue with new Python versions by abiding to the
proper use of ``__slots__`` attribute on classes
- Added checks that should prevent directory traversal attacks
- Many documentation fixes based on feedback from users
New features
~~~~~~~~~~~~
- Added an “early_release” more for instance replace disks and node
evacuate, where we release locks earlier and thus allow higher
parallelism within the cluster
- Added watcher hooks, intended to allow the watcher to restart other
daemons (e.g. from the ganeti-nbma project), but they can be used of
course for any other purpose
- Added a compile-time disable for DRBD barriers, to increase
performance if the administrator trusts the power supply or the
storage system to not lose writes
- Added the option of using syslog for logging instead of, or in
addition to, Ganeti's own log files
- Removed boot restriction for paravirtual NICs for KVM, recent versions
can indeed boot from a paravirtual NIC
- Added a generic debug level for many operations; while this is not
used widely yet, it allows one to pass the debug value all the way to
the OS scripts
- Enhanced the hooks environment for instance moves (failovers,
migrations) where the primary/secondary nodes changed during the
operation, by adding {NEW,OLD}_{PRIMARY,SECONDARY} vars
- Enhanced data validations for many user-supplied values; one important
item is the restrictions imposed on instance and node names, which
might reject some (invalid) host names
- Add a configure-time option to disable file-based storage, if it's not
needed; this allows greater security separation between the master
node and the other nodes from the point of view of the inter-node RPC
protocol
- Added user notification in interactive tools if job is waiting in the
job queue or trying to acquire locks
- Added log messages when a job is waiting for locks
- Added filtering by node tags in instance operations which admit
multiple instances (start, stop, reboot, reinstall)
- Added a new tool for cluster mergers, ``cluster-merge``
- Parameters from command line which are of the form ``a=b,c=d`` can now
use backslash escapes to pass in values which contain commas,
e.g. ``a=b\\c,d=e`` where the 'a' parameter would get the value
``b,c``
- For KVM, the instance name is the first parameter passed to KVM, so
that it's more visible in the process list
Version 2.1.0
-------------
......@@ -108,6 +222,15 @@ Details
- Improved burnin
Version 2.0.6
-------------
- Fix cleaner behaviour on nodes not in a cluster (Debian bug 568105)
- Fix a string formatting bug
- Improve safety of the code in some error paths
- Improve data validation in the master of values returned from nodes
Version 2.0.5
-------------
......
......@@ -458,6 +458,8 @@ class CompletionWriter:
choices = "$(_ganeti_nodes)"
elif isinstance(arg, cli.ArgJobId):
choices = "$(_ganeti_jobs)"
elif isinstance(arg, cli.ArgOs):
choices = "$(_ganeti_os)"
elif isinstance(arg, cli.ArgFile):
choices = ""
compgenargs.append("-f")
......
#!/bin/bash
#
# Copyright (C) 2010 Google Inc.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
# 02110-1301, USA.
set -e
: ${COVERAGE:=coverage}
: ${PYTHON:=python}
: ${COVERAGE_FILE:?}
: ${TEXT_COVERAGE:?}
omit=$($PYTHON -c 'import sys;
print " ".join(["--omit=%s/" % i for i in
set([sys.prefix, sys.exec_prefix])])')
$COVERAGE erase
for script; do
$COVERAGE run --branch --append $script
done
echo "Writing text report to $TEXT_COVERAGE ..." >&2
$COVERAGE report $omit | tee "$TEXT_COVERAGE"
if [[ -n "$HTML_COVERAGE" ]]; then
echo "Generating HTML report in $HTML_COVERAGE ..." >&2
$COVERAGE html $omit -d "$HTML_COVERAGE"
fi
......@@ -5,7 +5,7 @@ set -e
tmpdir=$(mktemp -d -t gntbuild.XXXXXXXX)
trap "rm -rf $tmpdir" EXIT
cp -r scripts lib tools test $tmpdir
cp -r daemons scripts lib tools test $tmpdir
mv $tmpdir/lib $tmpdir/ganeti
cd $tmpdir && "$@"
#!/bin/bash
#
# Copyright (C) 2010 Google Inc.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
# 02110-1301, USA.
set -e
filename=$1
case "$filename" in
*.py) exec $PYTHON "$@" ;;
*) exec "$@" ;;
esac
# Configure script for Ganeti
m4_define([gnt_version_major], [2])
m4_define([gnt_version_minor], [1])
m4_define([gnt_version_revision], [0])
m4_define([gnt_version_suffix], [~rc5])
m4_define([gnt_version_revision], [1])
m4_define([gnt_version_suffix], [])
m4_define([gnt_version_full],
m4_format([%d.%d.%d%s],
gnt_version_major, gnt_version_minor,
......@@ -39,6 +39,16 @@ AC_ARG_WITH([export-dir],
[export_dir="/srv/ganeti/export"])
AC_SUBST(EXPORT_DIR, $export_dir)
# --with-ssh-config-dir=...
AC_ARG_WITH([ssh-config-dir],
[AS_HELP_STRING([--with-ssh-config-dir=DIR],
[ directory with ssh host keys ]
[ (default is /etc/ssh)]
)],
[ssh_config_dir="$withval"],
[ssh_config_dir="/etc/ssh"])
AC_SUBST(SSH_CONFIG_DIR, $ssh_config_dir)
# --with-os-search-path=...
# do a bit of black sed magic to for quoting of the strings in the list
AC_ARG_WITH([os-search-path],
......@@ -94,9 +104,16 @@ AC_ARG_WITH([file-storage-dir],
[directory to store files for file-based backend]
[ (default is /srv/ganeti/file-storage)]
)],
[file_storage_dir="$withval"],
[file_storage_dir="/srv/ganeti/file-storage"])
[[file_storage_dir="$withval";
if test "$withval" != no; then
enable_file_storage=True
else
enable_file_storage=False
fi
]],
[[file_storage_dir="/srv/ganeti/file-storage"; enable_file_storage="True"]])
AC_SUBST(FILE_STORAGE_DIR, $file_storage_dir)
AC_SUBST(ENABLE_FILE_STORAGE, $enable_file_storage)
# --with-kvm-path=...
AC_ARG_WITH([kvm-path],
......
......@@ -22,6 +22,16 @@ set -e
defaults_file=@SYSCONFDIR@/default/ganeti
# This is a list of all daemons and the order in which they're started. The
# order is important as there are dependencies between them. On shutdown,
# they're stopped in reverse order.
DAEMONS=(
ganeti-noded
ganeti-masterd
ganeti-rapi
ganeti-confd
)
NODED_ARGS=
MASTERD_ARGS=
CONFD_ARGS=
......@@ -36,11 +46,86 @@ _daemon_pidfile() {
echo "@LOCALSTATEDIR@/run/ganeti/$1.pid"
}
_daemon_executable() {
echo "@PREFIX@/sbin/$1"
}
# Checks whether the local machine is part of a cluster
check_config() {
local server_pem=@LOCALSTATEDIR@/lib/ganeti/server.pem
local fname
for fname in $server_pem; do
if [[ ! -f $fname ]]; then
echo "Missing configuration file $fname" >&2
return 1
fi
done
return 0
}
# Checks the exit code of a daemon
check_exitcode() {
if [[ "$#" -lt 1 ]]; then
echo 'Missing exit code.' >&2
return 1
fi
local rc="$1"; shift
case "$rc" in
0) ;;
11)
echo "not master"
;;
*)
echo "exit code $rc"
return 1
;;
esac
return 0
}
# Prints a list of all daemons in the order in which they should be started
list_start_daemons() {
local name
for name in "${DAEMONS[@]}"; do
echo "$name"
done
}
# Prints a list of all daemons in the order in which they should be stopped
list_stop_daemons() {
list_start_daemons | tac
}
# Checks whether a daemon name is known
is_daemon_name() {
if [[ "$#" -lt 1 ]]; then
echo 'Missing daemon name.' >&2
return 1
fi
local name="$1"; shift
for i in "${DAEMONS[@]}"; do
if [[ "$i" == "$name" ]]; then
return 0
fi
done
echo "Unknown daemon name '$name'" >&2
return 1
}
# Checks whether daemon is running
check() {
if [[ "$#" -lt 1 ]]; then
echo 'Missing daemon name.' >&2
exit 1
return 1
fi
local name="$1"; shift
......@@ -53,7 +138,7 @@ check() {
start() {
if [[ "$#" -lt 1 ]]; then
echo 'Missing daemon name.' >&2
exit 1
return 1
fi
local name="$1"; shift
......@@ -66,7 +151,7 @@ start() {
start-stop-daemon --start --quiet --oknodo \
--pidfile $(_daemon_pidfile $name) \
--startas "@PREFIX@/sbin/$name" \
--startas $(_daemon_executable $name) \
-- $args "$@"
}
......@@ -74,7 +159,7 @@ start() {
stop() {
if [[ "$#" -lt 1 ]]; then
echo 'Missing daemon name.' >&2
exit 1
return 1
fi
local name="$1"; shift
......@@ -104,6 +189,30 @@ stop_master() {
stop ganeti-masterd
}
# Start all daemons
start_all() {
for i in $(list_start_daemons); do
local rc=0
# Try to start daemon
start $i || rc=$?
if ! errmsg=$(check_exitcode $rc); then
echo "$errmsg" >&2
return 1
fi
done
return 0
}
# Stop all daemons
stop_all() {
for i in $(list_stop_daemons); do
stop $i
done
}
# Reloads the SSH keys
reload_ssh_keys() {
@RPL_SSH_INITD_SCRIPT@ restart
......@@ -116,6 +225,11 @@ fi
orig_action=$1; shift
if [[ "$orig_action" == *_* ]]; then
echo "Command must not contain underscores" >&2
exit 1
fi
# Replace all dashes (-) with underlines (_)
action=${orig_action//-/_}
......
......@@ -27,7 +27,7 @@ QUEUE_ARCHIVE_DIR=$DATA_DIR/queue/archive
REMOVE_AFTER=21
# Exit if machine is not part of a cluster
[[ -e $DATA_DIR/ssconf_master_node ]] || echo 0
[[ -e $DATA_DIR/ssconf_master_node ]] || exit 0
# Exit if queue archive directory doesn't exist
[[ -d $QUEUE_ARCHIVE_DIR ]] || exit 0
......
......@@ -102,6 +102,7 @@ class ConfdInotifyEventHandler(pyinotify.ProcessEvent):
@param filename: config file to watch
"""
# pylint: disable-msg=W0231
# no need to call the parent's constructor
self.watch_manager = watch_manager
self.callback = callback
......
......@@ -122,17 +122,18 @@ class RemoteApiHttpServer(http.auth.HttpServerRequestAuthentication,
req.private = ctx
# Check for expected attributes
assert req.private.handler
assert req.private.handler_fn
assert req.private.handler_access is not None
return req.private
def GetAuthRealm(self, req):
"""Override the auth realm for queries.
def AuthenticationRequired(self, req):
"""Determine whether authentication is required.
"""
ctx = self._GetRequestContext(req)
if ctx.handler_access:
return self.AUTH_REALM
else:
return None
return bool(self._GetRequestContext(req).handler_access)
def Authenticate(self, req, username, password):
"""Checks whether a user can access a resource.
......
......@@ -64,53 +64,61 @@ class NotMasterError(errors.GenericError):
"""Exception raised when this host is not the master."""
def Indent(s, prefix='| '):
"""Indent a piece of text with a given prefix before each line.
@param s: the string to indent
@param prefix: the string to prepend each line
def ShouldPause():
"""Check whether we should pause.
"""
return "%s%s\n" % (prefix, ('\n' + prefix).join(s.splitlines()))
return bool(utils.ReadWatcherPauseFile(constants.WATCHER_PAUSEFILE))