Commit 7dcf333d authored by Michael Hanselmann's avatar Michael Hanselmann
Browse files

Merge branch 'devel-2.5'

* devel-2.5:
  Fix wrong headers and licences
  Update NEWS and increase to 2.4.5
  Fix parameters of RpcResult in hooks unit tests
  Fix a too long line.
  Move RenameFile to the new functions
  ensure_dirs: Move some useful functions into utils.
  Use JoinDisjointDicts in mcpu
  Add the JoinDisjointDicts function to utils.algo
  Fix queue archive creation with wrong permissions
  Ensure permission on the job queue version file
  OpGroupVerifyDisks: Fix wrong result type declaration
  RAPI: Make node evacuation actually work
  Bump version to 2.5.0~rc2
  Update NEWS for unreleased 2.4.5
  RAPI: Fix resource for replacing disks

Conflicts:
	lib/rapi/rlib2.py: Merged bugfix from commit 539d65ba

Signed-off-by: default avatarMichael Hanselmann <hansmi@google.com>
Reviewed-by: default avatarRené Nussbaumer <rn@google.com>
parents e3ac8406 fd7b69c0
News
====
Version 2.5.0 rc1
Version 2.5.0 rc2
-----------------
*(Released Tue, 4 Oct 2011)*
*(Released Tue, 18 Oct 2011)*
Incompatible/important changes and bugfixes
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
......@@ -129,6 +129,14 @@ Misc
- DRBD metadata volumes are overwritten with zeros during disk creation.
Version 2.5.0 rc1
-----------------
*(Released Tue, 4 Oct 2011)*
This was the first release candidate of the 2.5 series.
Version 2.5.0 beta3
-------------------
......@@ -153,6 +161,19 @@ Version 2.5.0 beta1
This was the first beta release of the 2.5 series.
Version 2.4.5
-------------
*(Released Thu, 27 Oct 2011)*
- Fixed bug when parsing command line parameter values ending in
backslash
- Fixed assertion error after unclean master shutdown
- Disable HTTP client pool for RPC, significantly reducing memory usage
of master daemon
- Fixed queue archive creation with wrong permissions
Version 2.4.4
-------------
......
......@@ -2,7 +2,7 @@
m4_define([gnt_version_major], [2])
m4_define([gnt_version_minor], [5])
m4_define([gnt_version_revision], [0])
m4_define([gnt_version_suffix], [~rc1])
m4_define([gnt_version_suffix], [~rc2])
m4_define([gnt_version_full],
m4_format([%d.%d.%d%s],
gnt_version_major, gnt_version_minor,
......
......@@ -772,16 +772,15 @@ It supports the following commands: ``POST``.
``POST``
~~~~~~~~
Takes the parameters ``mode`` (one of ``replace_on_primary``,
``replace_on_secondary``, ``replace_new_secondary`` or
``replace_auto``), ``disks`` (comma separated list of disk indexes),
``remote_node`` and ``iallocator``.
Returns a job ID.
Body parameters:
Either ``remote_node`` or ``iallocator`` needs to be defined when using
``mode=replace_new_secondary``.
.. opcode_params:: OP_INSTANCE_REPLACE_DISKS
:exclude: instance_name
``mode`` is a mandatory parameter. ``replace_auto`` tries to determine
the broken disk(s) on its own and replacing it.
Ganeti 2.4 and below used query parameters. Those are deprecated and
should no longer be used.
``/2/instances/[instance_name]/activate-disks``
......
......@@ -2722,7 +2722,10 @@ def JobQueueRename(old, new):
_EnsureJobQueueFile(old)
_EnsureJobQueueFile(new)
utils.RenameFile(old, new, mkdir=True)
getents = runtime.GetEnts()
utils.RenameFile(old, new, mkdir=True, mkdir_mode=0700,
dir_uid=getents.masterd_uid, dir_gid=getents.masterd_gid)
def BlockdevClose(instance_name, disks):
......
......@@ -3154,7 +3154,7 @@ class LUGroupVerifyDisks(NoHooksLU):
# any leftover items in nv_dict are missing LVs, let's arrange the data
# better
for key, inst in nv_dict.iteritems():
res_missing.setdefault(inst, []).append(key)
res_missing.setdefault(inst, []).append(list(key))
return (res_nodes, list(res_instances), res_missing)
......
......@@ -534,8 +534,7 @@ class HooksMaster(object):
env["GANETI_MASTER"] = cfg.GetMasterNode()
if phase_env:
assert not (set(env) & set(phase_env)), "Environment variables conflict"
env.update(phase_env)
env = utils.algo.JoinDisjointDicts(env, phase_env)
# Convert everything to strings
env = dict([(str(key), str(val)) for key, val in env.iteritems()])
......
......@@ -691,7 +691,8 @@ class OpGroupVerifyDisks(OpCode):
ht.TAnd(ht.TIsLength(3),
ht.TItems([ht.TDictOf(ht.TString, ht.TString),
ht.TListOf(ht.TString),
ht.TDictOf(ht.TString, ht.TListOf(ht.TString))]))
ht.TDictOf(ht.TString,
ht.TListOf(ht.TListOf(ht.TString)))]))
class OpClusterRepairDiskSizes(OpCode):
......
......@@ -63,6 +63,10 @@ REPLACE_DISK_SECONDARY = "replace_on_secondary"
REPLACE_DISK_CHG = "replace_new_secondary"
REPLACE_DISK_AUTO = "replace_auto"
NODE_EVAC_PRI = "primary-only"
NODE_EVAC_SEC = "secondary-only"
NODE_EVAC_ALL = "all"
NODE_ROLE_DRAINED = "drained"
NODE_ROLE_MASTER_CANDIATE = "master-candidate"
NODE_ROLE_MASTER = "master"
......@@ -981,7 +985,7 @@ class GanetiRapiClient(object): # pylint: disable=R0904
(GANETI_RAPI_VERSION, instance)), query, None)
def ReplaceInstanceDisks(self, instance, disks=None, mode=REPLACE_DISK_AUTO,
remote_node=None, iallocator=None, dry_run=False):
remote_node=None, iallocator=None):
"""Replaces disks on an instance.
@type instance: str
......@@ -996,8 +1000,6 @@ class GanetiRapiClient(object): # pylint: disable=R0904
@type iallocator: str or None
@param iallocator: instance allocator plugin to use (for use with
replace_auto mode)
@type dry_run: bool
@param dry_run: whether to perform a dry run
@rtype: string
@return: job id
......@@ -1007,18 +1009,17 @@ class GanetiRapiClient(object): # pylint: disable=R0904
("mode", mode),
]
if disks:
# TODO: Convert to body parameters
if disks is not None:
query.append(("disks", ",".join(str(idx) for idx in disks)))
if remote_node:
if remote_node is not None:
query.append(("remote_node", remote_node))
if iallocator:
if iallocator is not None:
query.append(("iallocator", iallocator))
if dry_run:
query.append(("dry-run", 1))
return self._SendRequest(HTTP_POST,
("/%s/instances/%s/replace-disks" %
(GANETI_RAPI_VERSION, instance)), query, None)
......@@ -1312,7 +1313,7 @@ class GanetiRapiClient(object): # pylint: disable=R0904
def EvacuateNode(self, node, iallocator=None, remote_node=None,
dry_run=False, early_release=None,
primary=None, secondary=None, accept_old=False):
mode=None, accept_old=False):
"""Evacuates instances from a Ganeti node.
@type node: str
......@@ -1325,10 +1326,8 @@ class GanetiRapiClient(object): # pylint: disable=R0904
@param dry_run: whether to perform a dry run
@type early_release: bool
@param early_release: whether to enable parallelization
@type primary: bool
@param primary: Whether to evacuate primary instances
@type secondary: bool
@param secondary: Whether to evacuate secondary instances
@type mode: string
@param mode: Node evacuation mode
@type accept_old: bool
@param accept_old: Whether caller is ready to accept old-style (pre-2.5)
results
......@@ -1351,6 +1350,7 @@ class GanetiRapiClient(object): # pylint: disable=R0904
query.append(("dry-run", 1))
if _NODE_EVAC_RES1 in self.GetFeatures():
# Server supports body parameters
body = {}
if iallocator is not None:
......@@ -1359,10 +1359,8 @@ class GanetiRapiClient(object): # pylint: disable=R0904
body["remote_node"] = remote_node
if early_release is not None:
body["early_release"] = early_release
if primary is not None:
body["primary"] = primary
if secondary is not None:
body["secondary"] = secondary
if mode is not None:
body["mode"] = mode
else:
# Pre-2.5 request format
body = None
......@@ -1372,7 +1370,8 @@ class GanetiRapiClient(object): # pylint: disable=R0904
" not accept old-style results (parameter"
" accept_old)")
if primary or primary is None or not (secondary is None or secondary):
# Pre-2.5 servers can only evacuate secondaries
if mode is not None and mode != NODE_EVAC_SEC:
raise GanetiApiError("Server can only evacuate secondary instances")
if iallocator:
......
......@@ -980,18 +980,34 @@ class R_2_instances_name_replace_disks(baserlib.OpcodeResource):
"instance_name": self.items[0],
}
if self.request_body:
data = self.request_body
elif self.queryargs:
# Legacy interface, do not modify/extend
data = {
"remote_node": self._checkStringVariable("remote_node", default=None),
"mode": self._checkStringVariable("mode", default=None),
"disks": self._checkStringVariable("disks", default=None),
"iallocator": self._checkStringVariable("iallocator", default=None),
}
else:
data = {}
# Parse disks
try:
raw_disks = data["disks"]
raw_disks = data.pop("disks")
except KeyError:
pass
else:
if not ht.TListOf(ht.TInt)(raw_disks): # pylint: disable=E1102
# Backwards compatibility for strings of the format "1, 2, 3"
try:
data["disks"] = [int(part) for part in raw_disks.split(",")]
except (TypeError, ValueError), err:
raise http.HttpBadRequest("Invalid disk index passed: %s" % err)
if raw_disks:
if ht.TListOf(ht.TInt)(raw_disks): # pylint: disable=E1102
data["disks"] = raw_disks
else:
# Backwards compatibility for strings of the format "1, 2, 3"
try:
data["disks"] = [int(part) for part in raw_disks.split(",")]
except (TypeError, ValueError), err:
raise http.HttpBadRequest("Invalid disk index passed: %s" % err)
return (data, static)
......
......@@ -22,12 +22,10 @@
"""
import errno
import os
import os.path
import optparse
import sys
import stat
import logging
from ganeti import constants
......@@ -49,79 +47,6 @@ ALL_TYPES = frozenset([
])
class EnsureError(errors.GenericError):
"""Top-level error class related to this script.
"""
def EnsurePermission(path, mode, uid=-1, gid=-1, must_exist=True,
_chmod_fn=os.chmod, _chown_fn=os.chown, _stat_fn=os.stat):
"""Ensures that given path has given mode.
@param path: The path to the file
@param mode: The mode of the file
@param uid: The uid of the owner of this file
@param gid: The gid of the owner of this file
@param must_exist: Specifies if non-existance of path will be an error
@param _chmod_fn: chmod function to use (unittest only)
@param _chown_fn: chown function to use (unittest only)
"""
logging.debug("Checking %s", path)
try:
st = _stat_fn(path)
fmode = stat.S_IMODE(st[stat.ST_MODE])
if fmode != mode:
logging.debug("Changing mode of %s from %#o to %#o", path, fmode, mode)
_chmod_fn(path, mode)
if max(uid, gid) > -1:
fuid = st[stat.ST_UID]
fgid = st[stat.ST_GID]
if fuid != uid or fgid != gid:
logging.debug("Changing owner of %s from UID %s/GID %s to"
" UID %s/GID %s", path, fuid, fgid, uid, gid)
_chown_fn(path, uid, gid)
except EnvironmentError, err:
if err.errno == errno.ENOENT:
if must_exist:
raise EnsureError("Path %s should exist, but does not" % path)
else:
raise EnsureError("Error while changing permissions on %s: %s" %
(path, err))
def EnsureDir(path, mode, uid, gid, _lstat_fn=os.lstat, _mkdir_fn=os.mkdir,
_ensure_fn=EnsurePermission):
"""Ensures that given path is a dir and has given mode, uid and gid set.
@param path: The path to the file
@param mode: The mode of the file
@param uid: The uid of the owner of this file
@param gid: The gid of the owner of this file
@param _lstat_fn: Stat function to use (unittest only)
@param _mkdir_fn: mkdir function to use (unittest only)
@param _ensure_fn: ensure function to use (unittest only)
"""
logging.debug("Checking directory %s", path)
try:
# We don't want to follow symlinks
st = _lstat_fn(path)
except EnvironmentError, err:
if err.errno != errno.ENOENT:
raise EnsureError("stat(2) on %s failed: %s" % (path, err))
_mkdir_fn(path)
else:
if not stat.S_ISDIR(st[stat.ST_MODE]):
raise EnsureError("Path %s is expected to be a directory, but isn't" %
path)
_ensure_fn(path, mode, uid=uid, gid=gid)
def RecursiveEnsure(path, uid, gid, dir_perm, file_perm):
"""Ensures permissions recursively down a directory.
......@@ -141,11 +66,12 @@ def RecursiveEnsure(path, uid, gid, dir_perm, file_perm):
for root, dirs, files in os.walk(path):
for subdir in dirs:
EnsurePermission(os.path.join(root, subdir), dir_perm, uid=uid, gid=gid)
utils.EnforcePermission(os.path.join(root, subdir), dir_perm, uid=uid,
gid=gid)
for filename in files:
EnsurePermission(os.path.join(root, filename), file_perm, uid=uid,
gid=gid)
utils.EnforcePermission(os.path.join(root, filename), file_perm, uid=uid,
gid=gid)
def EnsureQueueDir(path, mode, uid, gid):
......@@ -159,7 +85,8 @@ def EnsureQueueDir(path, mode, uid, gid):
"""
for filename in utils.ListVisibleFiles(path):
if constants.JOB_FILE_RE.match(filename):
EnsurePermission(utils.PathJoin(path, filename), mode, uid=uid, gid=gid)
utils.EnforcePermission(utils.PathJoin(path, filename), mode, uid=uid,
gid=gid)
def ProcessPath(path):
......@@ -176,12 +103,13 @@ def ProcessPath(path):
# No additional parameters
assert len(path[5:]) == 0
if pathtype == DIR:
EnsureDir(pathname, mode, uid, gid)
utils.MakeDirWithPerm(pathname, mode, uid, gid)
elif pathtype == QUEUE_DIR:
EnsureQueueDir(pathname, mode, uid, gid)
elif pathtype == FILE:
(must_exist, ) = path[5:]
EnsurePermission(pathname, mode, uid=uid, gid=gid, must_exist=must_exist)
utils.EnforcePermission(pathname, mode, uid=uid, gid=gid,
must_exist=must_exist)
def GetPaths():
......@@ -231,6 +159,8 @@ def GetPaths():
getent.masterd_uid, getent.masterd_gid, False),
(constants.JOB_QUEUE_SERIAL_FILE, FILE, 0600,
getent.masterd_uid, getent.masterd_gid, False),
(constants.JOB_QUEUE_VERSION_FILE, FILE, 0600,
getent.masterd_uid, getent.masterd_gid, False),
(constants.JOB_QUEUE_ARCHIVE_DIR, DIR, 0700,
getent.masterd_uid, getent.masterd_gid),
(rapi_dir, DIR, 0750, getent.rapi_uid, getent.masterd_gid),
......@@ -325,7 +255,7 @@ def Main():
if opts.full_run:
RecursiveEnsure(constants.JOB_QUEUE_ARCHIVE_DIR, getent.masterd_uid,
getent.masterd_gid, 0700, 0600)
except EnsureError, err:
except errors.GenericError, err:
logging.error("An error occurred while setting permissions: %s", err)
return constants.EXIT_FAILURE
......
......@@ -46,6 +46,29 @@ def UniqueSequence(seq):
return [i for i in seq if i not in seen and not seen.add(i)]
def JoinDisjointDicts(dict_a, dict_b):
"""Joins dictionaries with no conflicting keys.
Enforces the constraint that the two key sets must be disjoint, and then
merges the two dictionaries in a new dictionary that is returned to the
caller.
@type dict_a: dict
@param dict_a: the first dictionary
@type dict_b: dict
@param dict_b: the second dictionary
@rtype: dict
@return: a new dictionary containing all the key/value pairs contained in the
two dictionaries.
"""
assert not (set(dict_a) & set(dict_b)), ("Duplicate keys found while joining"
" %s and %s" % (dict_a, dict_b))
result = dict_a.copy()
result.update(dict_b)
return result
def FindDuplicates(seq):
"""Identifies duplicates in a list.
......
......@@ -28,6 +28,7 @@ import shutil
import tempfile
import errno
import time
import stat
from ganeti import errors
from ganeti import constants
......@@ -295,9 +296,13 @@ def RemoveDir(dirname):
raise
def RenameFile(old, new, mkdir=False, mkdir_mode=0750):
def RenameFile(old, new, mkdir=False, mkdir_mode=0750, dir_uid=None,
dir_gid=None):
"""Renames a file.
This just creates the very least directory if it does not exist and C{mkdir}
is set to true.
@type old: string
@param old: Original path
@type new: string
......@@ -306,6 +311,10 @@ def RenameFile(old, new, mkdir=False, mkdir_mode=0750):
@param mkdir: Whether to create target directory if it doesn't exist
@type mkdir_mode: int
@param mkdir_mode: Mode for newly created directories
@type dir_uid: int
@param dir_uid: The uid for the (if fresh created) dir
@type dir_gid: int
@param dir_gid: The gid for the (if fresh created) dir
"""
try:
......@@ -316,13 +325,89 @@ def RenameFile(old, new, mkdir=False, mkdir_mode=0750):
# as efficient.
if mkdir and err.errno == errno.ENOENT:
# Create directory and try again
Makedirs(os.path.dirname(new), mode=mkdir_mode)
dir_path = os.path.dirname(new)
MakeDirWithPerm(dir_path, mkdir_mode, dir_uid, dir_gid)
return os.rename(old, new)
raise
def EnforcePermission(path, mode, uid=None, gid=None, must_exist=True,
_chmod_fn=os.chmod, _chown_fn=os.chown, _stat_fn=os.stat):
"""Enforces that given path has given permissions.
@param path: The path to the file
@param mode: The mode of the file
@param uid: The uid of the owner of this file
@param gid: The gid of the owner of this file
@param must_exist: Specifies if non-existance of path will be an error
@param _chmod_fn: chmod function to use (unittest only)
@param _chown_fn: chown function to use (unittest only)
"""
logging.debug("Checking %s", path)
# chown takes -1 if you want to keep one part of the ownership, however
# None is Python standard for that. So we remap them here.
if uid is None:
uid = -1
if gid is None:
gid = -1
try:
st = _stat_fn(path)
fmode = stat.S_IMODE(st[stat.ST_MODE])
if fmode != mode:
logging.debug("Changing mode of %s from %#o to %#o", path, fmode, mode)
_chmod_fn(path, mode)
if max(uid, gid) > -1:
fuid = st[stat.ST_UID]
fgid = st[stat.ST_GID]
if fuid != uid or fgid != gid:
logging.debug("Changing owner of %s from UID %s/GID %s to"
" UID %s/GID %s", path, fuid, fgid, uid, gid)
_chown_fn(path, uid, gid)
except EnvironmentError, err:
if err.errno == errno.ENOENT:
if must_exist:
raise errors.GenericError("Path %s should exist, but does not" % path)
else:
raise errors.GenericError("Error while changing permissions on %s: %s" %
(path, err))
def MakeDirWithPerm(path, mode, uid, gid, _lstat_fn=os.lstat,
_mkdir_fn=os.mkdir, _perm_fn=EnforcePermission):
"""Enforces that given path is a dir and has given mode, uid and gid set.
@param path: The path to the file
@param mode: The mode of the file
@param uid: The uid of the owner of this file
@param gid: The gid of the owner of this file
@param _lstat_fn: Stat function to use (unittest only)
@param _mkdir_fn: mkdir function to use (unittest only)
@param _perm_fn: permission setter function to use (unittest only)
"""
logging.debug("Checking directory %s", path)
try:
# We don't want to follow symlinks
st = _lstat_fn(path)
except EnvironmentError, err:
if err.errno != errno.ENOENT:
raise errors.GenericError("stat(2) on %s failed: %s" % (path, err))
_mkdir_fn(path)
else:
if not stat.S_ISDIR(st[stat.ST_MODE]):
raise errors.GenericError(("Path %s is expected to be a directory, but "
"isn't") % path)
_perm_fn(path, mode, uid=uid, gid=gid)
def Makedirs(path, mode=0750):
"""Super-mkdir; create a leaf directory and all intermediate ones.
......
......@@ -382,6 +382,7 @@ def RunHardwareFailureTests(instance, pnode, snode):
if qa_config.TestEnabled("instance-replace-disks"):
othernode = qa_config.AcquireNode(exclude=[pnode, snode])
try:
RunTestIf("rapi", qa_rapi.TestRapiInstanceReplaceDisks, instance)
RunTest(qa_instance.TestReplaceDisks,
instance, pnode, snode, othernode)
finally:
......
#
#
# Copyright (C) 2007, 2008, 2009, 2010, 2011 Google Inc.
#
......@@ -618,6 +619,14 @@ def TestRapiInstanceReinstall(instance):
_WaitForRapiJob(_rapi_client.ReinstallInstance(instance["name"]))
def TestRapiInstanceReplaceDisks(instance):
"""Test replacing instance disks via RAPI"""
_WaitForRapiJob(_rapi_client.ReplaceInstanceDisks(instance["name"],
mode=constants.REPLACE_DISK_AUTO, disks=[]))
_WaitForRapiJob(_rapi_client.ReplaceInstanceDisks(instance["name"],
mode=constants.REPLACE_DISK_SEC, disks="0"))
def TestRapiInstanceModify(instance):
"""Test modifying instance via RAPI"""
def _ModifyInstance(**kwargs):
......
#
#
# Copyright (C) 2007 Google Inc.
#
# This program is free software; you can redistribute it and/or modify
......
......@@ -16,7 +16,7 @@
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
# 0.0510-1301, USA.
# 02110-1301, USA.
"""Script for unittesting the cmdlib module"""
......