Commit dad226e3 authored by Thomas Thrainer's avatar Thomas Thrainer

Merge branch 'stable-2.8' into master

* stable-2.8:
  gnt-cluster info (py): add enabled disk templates
  Version bump to 2.8.0~beta1
  Change version numbers in documentation
  Fix issue with python coverage tests
  Merge branch 'stable-2.7' into stable-2.8
  Disable python test if required libraries are missing
  Better specify what packages to install
  Improve install guide
  Fix typo in the documentation index
  Fix typos in the documentation index
  Update security document wrt confd access to SSL cert
  Add tools for building deb packages to build_chroot
  Improve the final message of build_chroot
  Make build_chroot self-contained

Conflicts:
	configure.ac (trivial)
	lib/cmdlib/instance_storage.py (trivial)

The version numbers changed in 'Change version numbers in documentation'
and 'Version bump to 2.8.0~beta1' were adapted to 2.9.
Signed-off-by: default avatarThomas Thrainer <thomasth@google.com>
Reviewed-by: default avatarMichele Tartara <mtartara@google.com>
parents c632d3a5 e4b5f955
......@@ -166,9 +166,10 @@ If using a distribution which does not provide them, first install
the Haskell platform. You can also install ``cabal`` manually::
$ apt-get install cabal-install
$ cabal update
Then install the additional libraries via
``cabal``::
Then install the additional libraries (only the ones not available in your
distribution packages) via ``cabal``::
$ cabal install json network parallel utf8-string curl
......
......@@ -1277,8 +1277,11 @@ dist_TESTS = \
test/py/ganeti-cleaner_unittest.bash \
test/py/import-export_unittest.bash \
test/py/cli-test.bash \
test/py/bash_completion.bash \
$(python_tests)
test/py/bash_completion.bash
if !PY_NODEV
dist_TESTS += $(python_tests)
endif
nodist_TESTS =
check_SCRIPTS =
......@@ -1311,7 +1314,6 @@ all_python_code = \
$(pkglib_python_scripts) \
$(nodist_pkglib_python_scripts) \
$(nodist_tools_python_scripts) \
$(python_tests) \
$(pkgpython_PYTHON) \
$(client_PYTHON) \
$(cmdlib_PYTHON) \
......@@ -1329,6 +1331,10 @@ all_python_code = \
$(noinst_PYTHON) \
$(qa_scripts)
if !PY_NODEV
all_python_code += $(python_tests)
endif
srclink_files = \
man/footer.rst \
test/py/check-cert-expired_unittest.bash \
......@@ -1999,11 +2005,16 @@ TAGS: $(GENERATED_FILES)
etags -l python -a -
.PHONY: coverage
COVERAGE_TESTS=
if WANT_HTOOLS
coverage: py-coverage hs-coverage
else
coverage: py-coverage
COVERAGE_TESTS += hs-coverage
endif
if !PY_NODEV
COVERAGE_TESTS += py-coverage
endif
coverage: $(COVERAGE_TESTS)
.PHONY: py-coverage
py-coverage: $(GENERATED_FILES) $(python_tests)
......
......@@ -24,7 +24,7 @@ Version 2.9.0 beta1
Version 2.8.0 beta1
-------------------
*(unreleased)*
*(Released Mon, 24 Jun 2013)*
Incompatible/important changes
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
......@@ -93,10 +93,10 @@ For Python:
- ``yaml`` library (only for running the QA).
Version 2.7.0 rc2
Version 2.7.0 rc3
-----------------
*(Released Fri, 24 May 2013)*
*(Released Tue, 25 Jun 2013)*
Incompatible/important changes
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
......@@ -201,7 +201,22 @@ Misc changes
- The functionality for allocating multiple instances at once has been
overhauled and is now also available through :doc:`RAPI <rapi>`.
Since rc1:
Since rc2:
- Fix permissions on the confd query socket (Issue 477)
- Fix permissions on the job archive dir (Issue 498)
- Fix handling of an internal exception in replace-disks (Issue 472)
- Fix gnt-node info handling of shortened names (Issue 497)
- Fix gnt-instance grow-disk when wiping is enabled
- Documentation improvements, and support for newer pandoc
- Fix hspace honoring ipolicy for disks (Issue 484)
- Improve handling of the ``kvm_extra`` HV parameter
Version 2.7.0 rc2
-----------------
*(Released Fri, 24 May 2013)*
- ``devel/upload`` now works when ``/var/run`` on the target nodes is a
symlink.
......
Ganeti 2.7
Ganeti 2.9
==========
For installation instructions, read the INSTALL and the doc/install.rst
......
......@@ -767,11 +767,26 @@ AC_PYTHON_MODULE(pyinotify, t)
AC_PYTHON_MODULE(pycurl, t)
AC_PYTHON_MODULE(bitarray, t)
AC_PYTHON_MODULE(ipaddr, t)
AC_PYTHON_MODULE(yaml, t)
AC_PYTHON_MODULE(mock)
AC_PYTHON_MODULE(affinity)
AC_PYTHON_MODULE(paramiko)
# Development-only Python modules
PY_NODEV=
AC_PYTHON_MODULE(yaml)
if test $HAVE_PYMOD_YAML == "no"; then
PY_NODEV="$PY_NODEV yaml"
fi
if test -n "$PY_NODEV"; then
AC_MSG_WARN(m4_normalize([Required development modules ($PY_NODEV) were not
found, you won't be able to run Python unittests]))
else
AC_MSG_NOTICE([Python development modules found, unittests enabled])
fi
AC_SUBST(PY_NODEV)
AM_CONDITIONAL([PY_NODEV], [test -n $PY_NODEV])
AC_CONFIG_FILES([ Makefile ])
AC_OUTPUT
#!/bin/bash
#Configuration
: ${DATA_DIR=`dirname $0`/data}
: ${ARCH=amd64}
: ${DIST_RELEASE=squeeze}
: ${ARCH:=amd64}
: ${DIST_RELEASE:=squeeze}
: ${CONF_DIR:=/etc/schroot/chroot.d}
: ${CHROOT_DIR:=/srv/chroot}
: ${ALTERNATIVE_EDITOR:=/usr/bin/vim.basic}
# The value of DATA_DIR is read as well from the environment.
#Automatically generated variables
CHROOTNAME=$DIST_RELEASE-$ARCH
......@@ -17,6 +17,9 @@ CHDIR=$ROOT/$CHNAME
USER=`whoami`
COMP_FILENAME=$CHROOTNAME.tar.gz
COMP_FILEPATH=$ROOT/$COMP_FILENAME
TEMP_DATA_DIR=`mktemp -d`
ACTUAL_DATA_DIR=$DATA_DIR
ACTUAL_DATA_DIR=${ACTUAL_DATA_DIR:-$TEMP_DATA_DIR}
#Runnability checks
if [ $USER != 'root' ]
......@@ -34,16 +37,46 @@ then
exit
fi
if [ ! -d $DATA_DIR ]
#Create configuration dir and files if they do not exist
if [ ! -d $ACTUAL_DATA_DIR ]
then
mkdir $ACTUAL_DATA_DIR
echo "The data directory"
echo " $DATA_DIR"
echo "does not exist."
echo "Please, set the DATA_DIR environment variable so that it points to the"
echo "data directory."
exit
echo " $ACTUAL_DATA_DIR"
echo "has been created."
fi
if [ ! -f $ACTUAL_DATA_DIR/final.schroot.conf.in ]
then
cat <<END >$ACTUAL_DATA_DIR/final.schroot.conf.in
[${CHROOTNAME}]
description=Debian ${DIST_RELEASE} ${ARCH}
groups=src
source-root-groups=root
type=file
file=${CHROOT_DIR}/${COMP_FILENAME}
END
echo "The file"
echo " $ACTUAL_DATA_DIR/final.schroot.conf.in"
echo "has been created with default configurations."
fi
if [ ! -f $ACTUAL_DATA_DIR/temp.schroot.conf.in ]
then
cat <<END >$ACTUAL_DATA_DIR/temp.schroot.conf.in
[${CHNAME}]
description=Debian ${DIST_RELEASE} ${ARCH}
directory=${CHDIR}
groups=src
users=root
type=directory
END
echo "The file"
echo " $ACTUAL_DATA_DIR/temp.schroot.conf.in"
echo "has been created with default configurations."
fi
#Stop on errors
set -e
#Cleanup
......@@ -65,8 +98,8 @@ alias subst_variables='sed \
-e "s/\${DIST_RELEASE}/$DIST_RELEASE/"'
#Generate chroot configurations
cat $DATA_DIR/temp.schroot.conf.in | subst_variables > $TEMP_CHROOT_CONF
cat $DATA_DIR/final.schroot.conf.in | subst_variables > $FINAL_CHROOT_CONF
cat $ACTUAL_DATA_DIR/temp.schroot.conf.in | subst_variables > $TEMP_CHROOT_CONF
cat $ACTUAL_DATA_DIR/final.schroot.conf.in | subst_variables > $FINAL_CHROOT_CONF
#Install the base system
debootstrap --arch $ARCH $DIST_RELEASE $CHDIR
......@@ -172,6 +205,10 @@ in_chroot -- \
in_chroot -- \
cabal install --global shelltestrunner
#Tools for creating debian packages
in_chroot -- \
apt-get install python-docutils debhelper quilt
#Set default editor
in_chroot -- \
update-alternatives --set editor $ALTERNATIVE_EDITOR
......@@ -181,7 +218,13 @@ echo "Creating compressed schroot image..."
cd $CHDIR
tar czf $COMP_FILEPATH ./*
cd $ROOT
echo "Done"
rm -rf $CHDIR
rm -f $TEMP_CHROOT_CONF
rm -rf $TEMP_DATA_DIR
echo "Chroot created. In order to run it:"
echo " * Copy the file $FINAL_CHROOT_CONF to $CONF_DIR/$FINAL_CHROOT_CONF"
echo " * Copy the file $COMP_FILEPATH to $CHROOT_DIR/$COMP_FILENAME"
echo "Then run \"schroot -c $CHROOTNAME\""
[${CHROOTNAME}]
description=Debian ${DIST_RELEASE} ${ARCH}
groups=src
source-root-groups=root
type=file
file=${CHROOT_DIR}/${COMP_FILENAME}
[${CHNAME}]
description=Debian ${DIST_RELEASE} ${ARCH}
directory=${CHDIR}
groups=src
users=root
type=directory
......@@ -2,7 +2,7 @@
Design document drafts
======================
.. Last updated for Ganeti 2.7
.. Last updated for Ganeti 2.9
.. toctree::
:maxdepth: 2
......
Ganeti customisation using hooks
================================
Documents Ganeti version 2.7
Documents Ganeti version 2.9
.. contents::
......
Ganeti automatic instance allocation
====================================
Documents Ganeti version 2.7
Documents Ganeti version 2.9
.. contents::
......
......@@ -20,7 +20,7 @@ Installing Ganeti
In order to install Ganeti, follow the instructions contained in the
:doc:`install`.
If you are an experience user, the content of the :doc:`install-quick` should
If you are an experienced user, the content of the :doc:`install-quick` should
be enough.
Instructions for upgrading an existing installation to the latest version of
......@@ -38,15 +38,15 @@ A more example-oriended guide is available in :doc:`walkthrough`.
The various tool that are part of Ganeti are described one by one in the
:doc:`manpages`.
A description of the security model underlying a Ganeti cluster can be find in
A description of the security model underlying a Ganeti cluster can be found in
the :doc:`security` document.
Ganeti functionalities can be extended be hooking scripts automatically
Ganeti functionalities can be extended by hooking scripts automatically
activated when certain events happen. Information on this mechanism is provided
in the :doc:`hooks` document.
While using Ganeti, the allocation of instances can happen manually or
automatically, through some external tool making decisions about this. The API
automatically, through some external tools making decisions about this. The API
for such tools is described in :doc:`iallocator`.
Most of the functionalities of Ganeti can be programmatically accessed through
......
Security in Ganeti
==================
Documents Ganeti version 2.7
Documents Ganeti version 2.9
Ganeti was developed to run on internal, trusted systems. As such, the
security model is all-or-nothing.
......@@ -127,7 +127,7 @@ non-Linux system at the moment.
Conf daemon
-----------
In Ganeti 2.7, the ``confd`` daemon (if enabled at build time), serves
In Ganeti 2.8, the ``confd`` daemon (if enabled at build time), serves
both network-originated queries (about the static configuration) and
local (UNIX socket) queries (about the run-time configuration; answering
these means talking to other cluster nodes, which makes use of the
......@@ -138,10 +138,13 @@ RPC), so to harden security it's recommended to:
- disable confd at build time if it's not needed in your setup
- otherwise, configure Ganeti (at build time) to use separate users, so
that the confd daemon doesn't also have access to the server SSL/TLS
certificates
certificates.
It is planned to split the two functionalities (local/remote querying)
of confd into two separate daemons in a future Ganeti version.
NB: the second suggestion is not valid since Ganeti 2.8.0~beta1, because confd
needs access to the certificate in order to communicate on the network.
This will be fixed when the planned split of the two functionalities
(local/remote querying) of confd into two separate daemons will take place,
in a future Ganeti version.
Monitoring daemon
-----------------
......
Virtual cluster support
=======================
Documents Ganeti version 2.7
Documents Ganeti version 2.9
.. contents::
......
......@@ -324,6 +324,7 @@ class LUClusterQuery(NoHooksLU):
"prealloc_wipe_disks": cluster.prealloc_wipe_disks,
"hidden_os": cluster.hidden_os,
"blacklisted_os": cluster.blacklisted_os,
"enabled_disk_templates": cluster.enabled_disk_templates,
}
return result
......
......@@ -1465,6 +1465,7 @@ class LUInstanceGrowDisk(LogicalUnit):
if wipe_disks:
# Get disk size from primary node for wiping
self.cfg.SetDiskID(self.disk, self.instance.primary_node)
result = self.rpc.call_blockdev_getdimensions(self.instance.primary_node,
[self.disk])
result.Raise("Failed to retrieve disk size from node '%s'" %
......@@ -2240,9 +2241,12 @@ class TLReplaceDisks(Tasklet):
# we pass force_create=True to force the LVM creation
for new_lv in new_lvs:
_CreateBlockDevInner(self.lu, node_uuid, self.instance, new_lv, True,
GetInstanceInfoText(self.instance), False,
excl_stor)
try:
_CreateBlockDevInner(self.lu, node_uuid, self.instance, new_lv, True,
GetInstanceInfoText(self.instance), False,
excl_stor)
except errors.DeviceCreationError, e:
raise errors.OpExecError("Can't create block device: %s" % e.message)
return iv_names
......@@ -2462,9 +2466,12 @@ class TLReplaceDisks(Tasklet):
(self.cfg.GetNodeName(self.new_node_uuid), idx))
# we pass force_create=True to force LVM creation
for new_lv in dev.children:
_CreateBlockDevInner(self.lu, self.new_node_uuid, self.instance, new_lv,
True, GetInstanceInfoText(self.instance), False,
excl_stor)
try:
_CreateBlockDevInner(self.lu, self.new_node_uuid, self.instance,
new_lv, True, GetInstanceInfoText(self.instance),
False, excl_stor)
except errors.DeviceCreationError, e:
raise errors.OpExecError("Can't create block device: %s" % e.message)
# Step 4: dbrd minors and drbd setups changes
# after this, we must manually remove the drbd minors on both the
......
......@@ -1400,7 +1400,7 @@ class KVMHypervisor(hv_base.BaseHypervisor):
kvm_cmd.extend(["-uuid", instance.uuid])
if hvp[constants.HV_KVM_EXTRA]:
kvm_cmd.extend([hvp[constants.HV_KVM_EXTRA]])
kvm_cmd.extend(hvp[constants.HV_KVM_EXTRA].split(" "))
# Save the current instance nics, but defer their expansion as parameters,
# as we'll need to generate executable temp files for them.
......
......@@ -170,7 +170,7 @@ def GetPaths():
getent.masterd_uid, getent.daemons_gid, False),
(pathutils.JOB_QUEUE_VERSION_FILE, FILE, constants.JOB_QUEUE_FILES_PERMS,
getent.masterd_uid, getent.daemons_gid, False),
(pathutils.JOB_QUEUE_ARCHIVE_DIR, DIR, 0740,
(pathutils.JOB_QUEUE_ARCHIVE_DIR, DIR, 0750,
getent.masterd_uid, getent.daemons_gid),
(rapi_dir, DIR, 0750, getent.rapi_uid, getent.masterd_gid),
(pathutils.RAPI_USERS_FILE, FILE, 0640,
......
......@@ -733,7 +733,9 @@ kvm\_extra
Valid for the KVM hypervisor.
Any other option to the KVM hypervisor, useful tweaking anything
that Ganeti doesn't support.
that Ganeti doesn't support. Note that values set with this
parameter are split on a space character and currently don't support
quoting.
machine\_version
Valid for the KVM hypervisor.
......
......@@ -131,6 +131,15 @@ Example::
# gnt-node evacuate -I hail node3.example.com
Note that, due to an issue with the iallocator interface, evacuation of
all instances at once is not yet implemented. Full evacuation can
currently be achieved by sequentially evacuating primaries and
secondaries.
::
# gnt-node evacuate -p node3.example.com
# gnt-node evacuate -s node3.example.com
FAILOVER
~~~~~~~~
......
......@@ -61,6 +61,18 @@ def _ShutdownInstance(name):
raise qa_error.Error("instance shutdown failed")
def _StartInstance(name):
"""Starts instance and waits for completion.
@param name: full name of the instance
"""
AssertCommand(["gnt-instance", "start", name])
if not bool(_InstanceRunning(name)):
raise qa_error.Error("instance start failed")
def _ResetWatcherDaemon():
"""Removes the watcher daemon's state file.
......@@ -130,6 +142,7 @@ def TestInstanceConsecutiveFailures(instance):
"""
inst_name = qa_utils.ResolveInstanceName(instance.name)
inst_was_running = bool(_InstanceRunning(inst_name))
_ResetWatcherDaemon()
......@@ -146,3 +159,6 @@ def TestInstanceConsecutiveFailures(instance):
raise qa_error.Error(msg)
AssertCommand(["gnt-instance", "info", inst_name])
if inst_was_running:
_StartInstance(inst_name)
......@@ -48,12 +48,15 @@ module Ganeti.Query.Filter
( compileFilter
, evaluateFilter
, requestedNames
, FilterConstructor
, makeSimpleFilter
, makeHostnameFilter
) where
import Control.Applicative
import Control.Monad (liftM)
import qualified Data.Map as Map
import Data.Maybe (fromJust)
import Data.Traversable (traverse)
import Text.JSON (JSValue(..), fromJSString)
import Text.JSON.Pretty (pp_value)
......@@ -239,8 +242,30 @@ requestedNames namefield (EQFilter fld val) =
else Nothing
requestedNames _ _ = Nothing
type FilterConstructor = String -> [Either String Integer] -> Filter FilterField
-- | Builds a simple filter from a list of names.
makeSimpleFilter :: String -> [Either String Integer] -> Filter FilterField
makeSimpleFilter _ [] = EmptyFilter
makeSimpleFilter namefield vals =
OrFilter $ map (EQFilter namefield . either QuotedString NumericValue) vals
-- | List of symbols with a special meaning for regular expressions.
reSpecialSymbols :: String
reSpecialSymbols = "\\.|()[]"
-- | Quote symbols that have special meaning in regular expressions.
quoteForRegex :: String -> String
quoteForRegex s = s >>= \x ->
if x `elem` reSpecialSymbols then ['\\', x] else [x]
-- | Builds a filter for hostnames from a list of names.
makeHostnameFilter :: String -> [Either String Integer] -> Filter FilterField
makeHostnameFilter _ [] = EmptyFilter
makeHostnameFilter namefield vals =
OrFilter . flip map vals
$ either (RegexpFilter namefield . fromJust . mkRegex
. (\ s -> "^(" ++ s ++ "|" ++ s ++ "\\..*)$")
. quoteForRegex)
(EQFilter namefield . NumericValue)
......@@ -54,7 +54,8 @@ import Ganeti.OpCodes (TagObject(..))
import qualified Ganeti.Query.Language as Qlang
import qualified Ganeti.Query.Cluster as QCluster
import Ganeti.Query.Query
import Ganeti.Query.Filter (makeSimpleFilter)
import Ganeti.Query.Filter (FilterConstructor, makeSimpleFilter
, makeHostnameFilter)
-- | A type for functions that can return the configuration when
-- executed.
......@@ -66,12 +67,16 @@ handleClassicQuery :: ConfigData -- ^ Cluster config
-> [Either String Integer] -- ^ Requested names
-- (empty means all)
-> [String] -- ^ Requested fields
-> Maybe FilterConstructor -- ^ the filter algorithm
-- to be used, defaults to
-- makeSimpleFilter
-> Bool -- ^ Whether to do sync queries or not
-> IO (GenericResult GanetiException JSValue)
handleClassicQuery _ _ _ _ True =
handleClassicQuery _ _ _ _ _ True =
return . Bad $ OpPrereqError "Sync queries are not allowed" ECodeInval
handleClassicQuery cfg qkind names fields _ = do
let flt = makeSimpleFilter (nameField qkind) names
handleClassicQuery cfg qkind names fields filterconstr _ = do
let fltcon = fromMaybe makeSimpleFilter filterconstr
flt = fltcon (nameField qkind) names
qr <- query cfg True (Qlang.Query qkind fields flt)
return $ showJSON <$> (qr >>= queryCompat)
......@@ -169,15 +174,15 @@ handleCall _ (QueryFields qkind qfields) = do
handleCall cfg (QueryNodes names fields lock) =
handleClassicQuery cfg (Qlang.ItemTypeOpCode Qlang.QRNode)
(map Left names) fields lock
(map Left names) fields (Just makeHostnameFilter) lock
handleCall cfg (QueryGroups names fields lock) =
handleClassicQuery cfg (Qlang.ItemTypeOpCode Qlang.QRGroup)
(map Left names) fields lock
(map Left names) fields Nothing lock
handleCall cfg (QueryJobs names fields) =
handleClassicQuery cfg (Qlang.ItemTypeLuxi Qlang.QRJob)
(map (Right . fromIntegral . fromJobId) names) fields False
(map (Right . fromIntegral . fromJobId) names) fields Nothing False
handleCall _ op =
return . Bad $
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment