Commit b142ef15 authored by Iustin Pop's avatar Iustin Pop
Browse files

Merge commit 'origin/next' into branch-2.1



* commit 'origin/next': (28 commits)
  Fix a typo in InitCluster
  Ignore results from drained nodes in iallocator
  Ship the ethers hook
  Ethers hook, compatibility with old lockfile
  Remove a few unused imports from noded/masterd
  Move HVM's device_model to a hypervisor parameter
  Implement the KERNEL_PATH parameter for xen-hvm
  Upgrade be/hv params with default values
  Add cluster-init --no-etc-hosts parameter
  objects: add configuration upgrade system
  Update NEWS and version for 2.0.3 release
  example ethers hook: use lockfile-progs
  ethers hook lock: use logger not echo
  ethers hook: reduce the probability of data loss
  devel/upload: revert rsync -p
  export: add meaningful exit code
  Fix detecting of errors in export
  Implement gnt-cluster check-disk-sizes
  rpc: add rpc call for getting disk size
  bdev: Add function for reading actual disk size
  ...

Conflicts:
	daemons/ganeti-masterd   - trivial, kept 2.1 version
	lib/bootstrap.py         - trivial, kept 2.1 version
	lib/cmdlib.py            - integrated the 2.0.3 changes
	lib/constants.py         - trivial
	lib/hypervisor/hv_xen.py - trivial, kept 2.1 version
	lib/objects.py           - trivial, kept 2.1 version
	lib/opcodes.py           - integrated the 2.0.3 changes
Signed-off-by: default avatarIustin Pop <iustin@google.com>
parents 05d47e33 bec0522b
...@@ -26,6 +26,7 @@ DIRS = \ ...@@ -26,6 +26,7 @@ DIRS = \
devel \ devel \
doc \ doc \
doc/examples \ doc/examples \
doc/examples/hooks \
lib \ lib \
lib/http \ lib/http \
lib/hypervisor \ lib/hypervisor \
...@@ -182,6 +183,7 @@ EXTRA_DIST = \ ...@@ -182,6 +183,7 @@ EXTRA_DIST = \
doc/examples/ganeti.initd.in \ doc/examples/ganeti.initd.in \
doc/examples/ganeti.cron.in \ doc/examples/ganeti.cron.in \
doc/examples/dumb-allocator \ doc/examples/dumb-allocator \
doc/examples/hooks/ethers \
doc/locking.txt \ doc/locking.txt \
test/testutils.py \ test/testutils.py \
test/mocks.py \ test/mocks.py \
...@@ -341,6 +343,7 @@ $(REPLACE_VARS_SED): Makefile stamp-directories ...@@ -341,6 +343,7 @@ $(REPLACE_VARS_SED): Makefile stamp-directories
echo 's#@CUSTOM_XEN_KERNEL@#$(XEN_KERNEL)#g'; \ echo 's#@CUSTOM_XEN_KERNEL@#$(XEN_KERNEL)#g'; \
echo 's#@CUSTOM_XEN_INITRD@#$(XEN_INITRD)#g'; \ echo 's#@CUSTOM_XEN_INITRD@#$(XEN_INITRD)#g'; \
echo 's#@RPL_FILE_STORAGE_DIR@#$(FILE_STORAGE_DIR)#g'; \ echo 's#@RPL_FILE_STORAGE_DIR@#$(FILE_STORAGE_DIR)#g'; \
echo 's#@PKGLIBDIR@#$(pkglibdir)#g'; \
} > $@ } > $@
# We need to create symlinks because "make distcheck" will not install Python # We need to create symlinks because "make distcheck" will not install Python
......
Version 2.0.3
- Added “--ignore-size” to the “gnt-instance activate-disks” command
to allow using the pre-2.0.2 behaviour in activation, if any
existing instances have mismatched disk sizes in the configuration
- Added “gnt-cluster repair-disk-sizes” command to check and update
any configuration mismatches for disk sizes
- Added “gnt-master cluste-failover --no-voting” to allow master
failover to work on two-node clusters
- Fixed the ‘--net’ option of “gnt-backup import”, which was unusable
- Fixed detection of OS script errors in “gnt-backup export”
- Fixed exit code of “gnt-backup export”
Version 2.0.2 Version 2.0.2
- Added experimental support for stripped logical volumes; this should - Added experimental support for stripped logical volumes; this should
enhance performance but comes with a higher complexity in the block enhance performance but comes with a higher complexity in the block
......
# Configure script for Ganeti # Configure script for Ganeti
m4_define([gnt_version_major], [2]) m4_define([gnt_version_major], [2])
m4_define([gnt_version_minor], [0]) m4_define([gnt_version_minor], [0])
m4_define([gnt_version_revision], [2]) m4_define([gnt_version_revision], [3])
m4_define([gnt_version_suffix], []) m4_define([gnt_version_suffix], [])
m4_define([gnt_version_full], m4_define([gnt_version_full],
m4_format([%d.%d.%d%s], m4_format([%d.%d.%d%s],
......
...@@ -28,13 +28,10 @@ inheritance from parent classes requires it. ...@@ -28,13 +28,10 @@ inheritance from parent classes requires it.
import os import os
import errno
import sys import sys
import SocketServer import SocketServer
import time import time
import collections import collections
import Queue
import random
import signal import signal
import logging import logging
......
...@@ -26,9 +26,7 @@ ...@@ -26,9 +26,7 @@
import os import os
import sys import sys
import traceback
import SocketServer import SocketServer
import errno
import logging import logging
import signal import signal
...@@ -250,6 +248,14 @@ class NodeHttpServer(http.server.HttpServer): ...@@ -250,6 +248,14 @@ class NodeHttpServer(http.server.HttpServer):
disks = [objects.Disk.FromDict(cf) for cf in params[1]] disks = [objects.Disk.FromDict(cf) for cf in params[1]]
return backend.BlockdevClose(params[0], disks) return backend.BlockdevClose(params[0], disks)
@staticmethod
def perspective_blockdev_getsize(params):
"""Compute the sizes of the given block devices.
"""
disks = [objects.Disk.FromDict(cf) for cf in params[0]]
return backend.BlockdevGetsize(disks)
# blockdev/drbd specific methods ---------- # blockdev/drbd specific methods ----------
@staticmethod @staticmethod
......
...@@ -27,14 +27,22 @@ ...@@ -27,14 +27,22 @@
set -e set -e
PREFIX='@PREFIX@'
SYSCONFDIR='@SYSCONFDIR@'
PKGLIBDIR='@PKGLIBDIR@'
NO_RESTART= NO_RESTART=
NO_CRON=
hosts= hosts=
while [ "$#" -gt 0 ]; do while [ "$#" -gt 0 ]; do
opt="$1" opt="$1"
case "$opt" in case "$opt" in
--no-restart) --no-restart)
NO_RESTART=1 NO_RESTART=1
;; ;;
--no-cron)
NO_CRON=1
;;
-h|--help) -h|--help)
echo "Usage: $0 [--no-restart] hosts..." echo "Usage: $0 [--no-restart] hosts..."
exit 0 exit 0
...@@ -42,10 +50,10 @@ while [ "$#" -gt 0 ]; do ...@@ -42,10 +50,10 @@ while [ "$#" -gt 0 ]; do
-*) -*)
echo "Unknown option: $opt" >&2 echo "Unknown option: $opt" >&2
exit 1 exit 1
;; ;;
*) *)
hosts="$hosts $opt" hosts="$hosts $opt"
;; ;;
esac esac
shift shift
done done
...@@ -58,39 +66,36 @@ trap 'rm -rf $TXD' EXIT ...@@ -58,39 +66,36 @@ trap 'rm -rf $TXD' EXIT
# install ganeti as a real tree # install ganeti as a real tree
make install DESTDIR="$TXD" make install DESTDIR="$TXD"
# copy additional needed files
install -D --mode=0755 doc/examples/ganeti.initd \
"$TXD/$SYSCONFDIR/init.d/ganeti"
install -D --mode=0644 doc/examples/bash_completion \
"$TXD/$SYSCONFDIR/bash_completion.d/ganeti"
if [ -z "$NO_CRON" ]; then
install -D --mode=0644 doc/examples/ganeti.cron \
"$TXD/$SYSCONFDIR/cron.d/ganeti"
fi
install -D --mode=0755 doc/examples/dumb-allocator \
"$TXD/$PKGLIBDIR/iallocators/dumb"
echo --- echo ---
( cd "$TXD" && find; ) ( cd "$TXD" && find; )
echo --- echo ---
PREFIX='@PREFIX@'
# and now put it under $prefix on the target node(s) # and now put it under $prefix on the target node(s)
for host; do for host; do
echo Uploading code to ${host}... echo Uploading code to ${host}...
rsync -v -rlDc --exclude="*.py[oc]" --exclude="*.pdf" --exclude="*.html" \ rsync -v -rlDc --exclude="*.py[oc]" --exclude="*.pdf" --exclude="*.html" \
"$TXD/$PREFIX/" \ "$TXD/" \
root@${host}:$PREFIX/ & root@${host}:/ &
done
wait
INIT_SCRIPT="$TXD/ganeti.initd"
install --mode=0755 doc/examples/ganeti.initd $INIT_SCRIPT
for host; do
echo Uploading init script to ${host}...
scp $INIT_SCRIPT root@${host}:/etc/init.d/ganeti &
done done
wait wait
if [ -f ganeti-master-cron ]; then
for host; do
echo Uploading cron files to ${host}...
scp ganeti-master-cron root@${host}:/etc/ganeti/master-cron &
done
fi
wait
if test -z "${NO_RESTART}"; then if test -z "${NO_RESTART}"; then
for host; do for host; do
echo Restarting ganeti-noded on ${host}... echo Restarting ganeti-noded on ${host}...
......
...@@ -90,7 +90,7 @@ _gnt_cluster() ...@@ -90,7 +90,7 @@ _gnt_cluster()
if [[ -e "@LOCALSTATEDIR@/lib/ganeti/ssconf_cluster_name" ]]; then if [[ -e "@LOCALSTATEDIR@/lib/ganeti/ssconf_cluster_name" ]]; then
cmds="add-tags command copyfile destroy getmaster info list-tags \ cmds="add-tags command copyfile destroy getmaster info list-tags \
masterfailover modify queue redist-conf remove-tags rename \ masterfailover modify queue redist-conf remove-tags rename \
search-tags verify verify-disks version" repair-disk-sizes search-tags verify verify-disks version"
else else
cmds="init" cmds="init"
fi fi
......
...@@ -35,7 +35,13 @@ ...@@ -35,7 +35,13 @@
# bridge to /etc/ethers. # bridge to /etc/ethers.
TARGET_BRIDGE="br0" TARGET_BRIDGE="br0"
DAEMON_PID_FILE="/var/run/dnsmasq.pid" DAEMON_PID_FILE="/var/run/dnsmasq.pid"
LOCKFILE="/var/lock/ganeti_ethers.lock"
# In order to handle concurrent execution of this lock, we use the $LOCKFILE.
# LOCKFILE_CREATE and LOCKFILE_REMOVE are the path names for the lockfile-progs
# programs which we use as helpers.
LOCKFILE="/var/lock/ganeti_ethers"
LOCKFILE_CREATE="/usr/bin/lockfile-create"
LOCKFILE_REMOVE="/usr/bin/lockfile-remove"
hooks_path=$GANETI_HOOKS_PATH hooks_path=$GANETI_HOOKS_PATH
[ -n "$hooks_path" ] || exit 1 [ -n "$hooks_path" ] || exit 1
...@@ -44,11 +50,8 @@ instance=$GANETI_INSTANCE_NAME ...@@ -44,11 +50,8 @@ instance=$GANETI_INSTANCE_NAME
nic_count=$GANETI_INSTANCE_NIC_COUNT nic_count=$GANETI_INSTANCE_NIC_COUNT
acquire_lockfile() { acquire_lockfile() {
if ! ( set -o noclobber; echo "$$" > $LOCKFILE) 2> /dev/null; then $LOCKFILE_CREATE $LOCKFILE || exit 1
logger -s "Cannot acquire lockfile for ethers update" trap "$LOCKFILE_REMOVE $LOCKFILE" EXIT
exit 1
fi
trap "rm -f $LOCKFILE" EXIT
} }
update_ethers_from_new() { update_ethers_from_new() {
......
...@@ -176,7 +176,7 @@ instances ...@@ -176,7 +176,7 @@ instances
nodes nodes
dictionary with the data for the nodes in the cluster, indexed by dictionary with the data for the nodes in the cluster, indexed by
the node name; the dict contains: the node name; the dict contains [*]_ :
total_disk total_disk
the total disk size of this node (mebibytes) the total disk size of this node (mebibytes)
...@@ -225,9 +225,13 @@ nodes ...@@ -225,9 +225,13 @@ nodes
or ``offline`` flags set. More details about these of node status or ``offline`` flags set. More details about these of node status
flags is available in the manpage :manpage:`ganeti(7)`. flags is available in the manpage :manpage:`ganeti(7)`.
.. [*] Note that no run-time data is present for offline or drained nodes;
this means the tags total_memory, reserved_memory, free_memory, total_disk,
free_disk, total_cpus, i_pri_memory and i_pri_up memory will be absent
Respone message
~~~~~~~~~~~~~~~ Response message
~~~~~~~~~~~~~~~~
The response message is much more simple than the input one. It is The response message is much more simple than the input one. It is
also a dict having three keys: also a dict having three keys:
......
...@@ -1433,6 +1433,32 @@ def BlockdevFind(disk): ...@@ -1433,6 +1433,32 @@ def BlockdevFind(disk):
return rbd.GetSyncStatus() return rbd.GetSyncStatus()
def BlockdevGetsize(disks):
"""Computes the size of the given disks.
If a disk is not found, returns None instead.
@type disks: list of L{objects.Disk}
@param disks: the list of disk to compute the size for
@rtype: list
@return: list with elements None if the disk cannot be found,
otherwise the size
"""
result = []
for cf in disks:
try:
rbd = _RecursiveFindBD(cf)
except errors.BlockDeviceError, err:
result.append(None)
continue
if rbd is None:
result.append(None)
else:
result.append(rbd.GetActualSize())
return result
def UploadFile(file_name, data, mode, uid, gid, atime, mtime): def UploadFile(file_name, data, mode, uid, gid, atime, mtime):
"""Write a file to the filesystem. """Write a file to the filesystem.
...@@ -1820,8 +1846,8 @@ def ExportSnapshot(disk, dest_node, instance, cluster_name, idx): ...@@ -1820,8 +1846,8 @@ def ExportSnapshot(disk, dest_node, instance, cluster_name, idx):
# the target command is built out of three individual commands, # the target command is built out of three individual commands,
# which are joined by pipes; we check each individual command for # which are joined by pipes; we check each individual command for
# valid parameters # valid parameters
expcmd = utils.BuildShellCmd("cd %s; %s 2>%s", inst_os.path, expcmd = utils.BuildShellCmd("set -e; set -o pipefail; cd %s; %s 2>%s",
export_script, logfile) inst_os.path, export_script, logfile)
comprcmd = "gzip" comprcmd = "gzip"
...@@ -1834,7 +1860,7 @@ def ExportSnapshot(disk, dest_node, instance, cluster_name, idx): ...@@ -1834,7 +1860,7 @@ def ExportSnapshot(disk, dest_node, instance, cluster_name, idx):
# all commands have been checked, so we're safe to combine them # all commands have been checked, so we're safe to combine them
command = '|'.join([expcmd, comprcmd, utils.ShellQuoteArgs(remotecmd)]) command = '|'.join([expcmd, comprcmd, utils.ShellQuoteArgs(remotecmd)])
result = utils.RunCmd(command, env=export_env) result = utils.RunCmd(["bash", "-c", command], env=export_env)
if result.failed: if result.failed:
_Fail("OS snapshot export command '%s' returned error: %s" _Fail("OS snapshot export command '%s' returned error: %s"
......
...@@ -304,6 +304,23 @@ class BlockDev(object): ...@@ -304,6 +304,23 @@ class BlockDev(object):
""" """
raise NotImplementedError raise NotImplementedError
def GetActualSize(self):
"""Return the actual disk size.
@note: the device needs to be active when this is called
"""
assert self.attached, "BlockDevice not attached in GetActualSize()"
result = utils.RunCmd(["blockdev", "--getsize64", self.dev_path])
if result.failed:
_ThrowError("blockdev failed (%s): %s",
result.fail_reason, result.output)
try:
sz = int(result.output.strip())
except (ValueError, TypeError), err:
_ThrowError("Failed to parse blockdev output: %s", str(err))
return sz
def __repr__(self): def __repr__(self):
return ("<%s: unique_id: %s, children: %s, %s:%s, %s>" % return ("<%s: unique_id: %s, children: %s, %s:%s, %s>" %
(self.__class__, self.unique_id, self._children, (self.__class__, self.unique_id, self._children,
...@@ -1166,9 +1183,10 @@ class DRBD8(BaseDRBD): ...@@ -1166,9 +1183,10 @@ class DRBD8(BaseDRBD):
""" """
args = ["drbdsetup", cls._DevPath(minor), "disk", args = ["drbdsetup", cls._DevPath(minor), "disk",
backend, meta, "0", backend, meta, "0",
"-d", "%sm" % size,
"-e", "detach", "-e", "detach",
"--create-device"] "--create-device"]
if size:
args.extend(["-d", "%sm" % size])
result = utils.RunCmd(args) result = utils.RunCmd(args)
if result.failed: if result.failed:
_ThrowError("drbd%d: can't attach local disk: %s", minor, result.output) _ThrowError("drbd%d: can't attach local disk: %s", minor, result.output)
...@@ -1777,6 +1795,19 @@ class FileStorage(BlockDev): ...@@ -1777,6 +1795,19 @@ class FileStorage(BlockDev):
self.attached = os.path.exists(self.dev_path) self.attached = os.path.exists(self.dev_path)
return self.attached return self.attached
def GetActualSize(self):
"""Return the actual disk size.
@note: the device needs to be active when this is called
"""
assert self.attached, "BlockDevice not attached in GetActualSize()"
try:
st = os.stat(self.dev_path)
return st.st_size
except OSError, err:
_ThrowError("Can't stat %s: %s", self.dev_path, err)
@classmethod @classmethod
def Create(cls, unique_id, children, size): def Create(cls, unique_id, children, size):
"""Create a new file. """Create a new file.
......
...@@ -1500,6 +1500,100 @@ class LUVerifyDisks(NoHooksLU): ...@@ -1500,6 +1500,100 @@ class LUVerifyDisks(NoHooksLU):
return result return result
class LURepairDiskSizes(NoHooksLU):
"""Verifies the cluster disks sizes.
"""
_OP_REQP = ["instances"]
REQ_BGL = False
def ExpandNames(self):
if not isinstance(self.op.instances, list):
raise errors.OpPrereqError("Invalid argument type 'instances'")
if self.op.instances:
self.wanted_names = []
for name in self.op.instances:
full_name = self.cfg.ExpandInstanceName(name)
if full_name is None:
raise errors.OpPrereqError("Instance '%s' not known" % name)
self.wanted_names.append(full_name)
self.needed_locks[locking.LEVEL_INSTANCE] = self.wanted_names
self.needed_locks = {
locking.LEVEL_NODE: [],
locking.LEVEL_INSTANCE: self.wanted_names,
}
self.recalculate_locks[locking.LEVEL_NODE] = constants.LOCKS_REPLACE
else:
self.wanted_names = None
self.needed_locks = {
locking.LEVEL_NODE: locking.ALL_SET,
locking.LEVEL_INSTANCE: locking.ALL_SET,
}
self.share_locks = dict(((i, 1) for i in locking.LEVELS))
def DeclareLocks(self, level):
if level == locking.LEVEL_NODE and self.wanted_names is not None:
self._LockInstancesNodes(primary_only=True)
def CheckPrereq(self):
"""Check prerequisites.
This only checks the optional instance list against the existing names.
"""
if self.wanted_names is None:
self.wanted_names = self.acquired_locks[locking.LEVEL_INSTANCE]
self.wanted_instances = [self.cfg.GetInstanceInfo(name) for name
in self.wanted_names]
def Exec(self, feedback_fn):
"""Verify the size of cluster disks.
"""
# TODO: check child disks too
# TODO: check differences in size between primary/secondary nodes
per_node_disks = {}
for instance in self.wanted_instances:
pnode = instance.primary_node
if pnode not in per_node_disks:
per_node_disks[pnode] = []
for idx, disk in enumerate(instance.disks):
per_node_disks[pnode].append((instance, idx, disk))
changed = []
for node, dskl in per_node_disks.items():
result = self.rpc.call_blockdev_getsizes(node, [v[2] for v in dskl])
if result.failed:
self.LogWarning("Failure in blockdev_getsizes call to node"
" %s, ignoring", node)
continue
if len(result.data) != len(dskl):
self.LogWarning("Invalid result from node %s, ignoring node results",
node)
continue
for ((instance, idx, disk), size) in zip(dskl, result.data):
if size is None:
self.LogWarning("Disk %d of instance %s did not return size"
" information, ignoring", idx, instance.name)
continue
if not isinstance(size, (int, long)):
self.LogWarning("Disk %d of instance %s did not return valid"
" size information, ignoring", idx, instance.name)
continue
size = size >> 20
if size != disk.size:
self.LogInfo("Disk %d of instance %s has mismatched size,"
" correcting: recorded %d, actual %d", idx,
instance.name, disk.size, size)
disk.size = size
self.cfg.Update(instance)
changed.append((instance.name, idx, size))
return changed
class LURenameCluster(LogicalUnit): class LURenameCluster(LogicalUnit):
"""Rename the cluster. """Rename the cluster.
...@@ -3000,19 +3094,24 @@ class LUActivateInstanceDisks(NoHooksLU): ...@@ -3000,19 +3094,24 @@ class LUActivateInstanceDisks(NoHooksLU):
assert self.instance is not None, \ assert self.instance is not None, \
"Cannot retrieve locked instance %s" % self.op.instance_name "Cannot retrieve locked instance %s" % self.op.instance_name
_CheckNodeOnline(self, self.instance.primary_node) _CheckNodeOnline(self, self.instance.primary_node)
if not hasattr(self.op, "ignore_size"):
self.op.ignore_size = False
def Exec(self, feedback_fn): def Exec(self, feedback_fn):
"""Activate the disks. """Activate the disks.
""" """
disks_ok, disks_info = _AssembleInstanceDisks(self, self.instance) disks_ok, disks_info = \
_AssembleInstanceDisks(self, self.instance,
ignore_size=self.op.ignore_size)
if not disks_ok: if not disks_ok:
raise errors.OpExecError("Cannot activate block devices") raise errors.OpExecError("Cannot activate block devices")
return disks_info return disks_info
def _AssembleInstanceDisks(lu, instance, ignore_secondaries=False): def _AssembleInstanceDisks(lu, instance, ignore_secondaries=False,
ignore_size=False):
"""Prepare the block devices for an instance. """Prepare the block devices for an instance.
This sets up the block devices on all nodes. This sets up the block devices on all nodes.
...@@ -3024,6 +3123,10 @@ def _AssembleInstanceDisks(lu, instance, ignore_secondaries=False): ...@@ -3024,6 +3123,10 @@ def _AssembleInstanceDisks(lu, instance, ignore_secondaries=False):
@type ignore_secondaries: boolean