Commit e7307f08 authored by Michael Hanselmann's avatar Michael Hanselmann
Browse files

Merge branch 'devel-2.1'

* devel-2.1:
  Bump version to 2.1.0~rc5
  Makefile.am: Targets used directly should depend on BUILT_SOURCES
  Make the snapshot decision based on disk type
  Three small typos in man pages
  Fix missing bridge for xen instances
  Fix flipping MC flag bug
  ganeti-watcher: ensure confd is running as well
  Add capability to use syslog for logging
  node daemon: allow working with broken queue dir
  utils.FileLock: handle init errors properly
  daemon-utils: remove usage of here-docs
  Fix typo in ganeti-os-interface(7)
  locking: add/fix @type information
  Fix __slots__ definitions
  Fix the mocks.py for 2.0 unittests
  Add a crude disable for DRBD barriers
  LURemoveNode safety in face of wrong node list
  Fix an unsafe formatting bug
  Ensure all int/float conversions are handled right

Conflicts:
	lib/daemon.py: Trivial
parents 8388e9ff 9f53d9ce
......@@ -447,6 +447,8 @@ lib/_autoconf.py: Makefile stamp-directories
echo "TOOLSDIR = '$(toolsdir)'"; \
echo "GNT_SCRIPTS = [$(foreach i,$(notdir $(gnt_scripts)),'$(i)',)]"; \
echo "PKGLIBDIR = '$(pkglibdir)'"; \
echo "DRBD_BARRIERS = $(DRBD_BARRIERS)"; \
echo "SYSLOG_USAGE = '$(SYSLOG_USAGE)'"; \
} > $@
$(REPLACE_VARS_SED): Makefile
......@@ -484,7 +486,7 @@ check-local:
$(CHECK_PYTHON_CODE) $(check_python_code)
.PHONY: lint
lint: ganeti
lint: ganeti $(BUILT_SOURCES)
@test -n "$(PYLINT)" || { echo 'pylint' not found during configure; exit 1; }
$(PYLINT) $(LINT_OPTS) $(lint_python_code)
......@@ -513,14 +515,14 @@ stamp-directories: Makefile
touch $@
.PHONY: apidoc
apidoc: epydoc.conf $(RUN_IN_TEMPDIR)
apidoc: epydoc.conf $(RUN_IN_TEMPDIR) $(BUILT_SOURCES)
test -e doc/api || mkdir doc/api
$(RUN_IN_TEMPDIR) epydoc -v \
--conf $(CURDIR)/epydoc.conf \
--output $(CURDIR)/doc/api
.PHONY: TAGS
TAGS:
TAGS: $(BUILT_SOURCES)
rm -f TAGS
find . -path './lib/*.py' -o -path 'scripts/gnt-*' -o \
-path 'daemons/ganeti-*' -o -path 'tools/*' | \
......
......@@ -2,7 +2,7 @@
m4_define([gnt_version_major], [2])
m4_define([gnt_version_minor], [1])
m4_define([gnt_version_revision], [0])
m4_define([gnt_version_suffix], [~rc4])
m4_define([gnt_version_suffix], [~rc5])
m4_define([gnt_version_full],
m4_format([%d.%d.%d%s],
gnt_version_major, gnt_version_minor,
......@@ -108,7 +108,7 @@ AC_ARG_WITH([kvm-path],
[kvm_path="/usr/bin/kvm"])
AC_SUBST(KVM_PATH, $kvm_path)
# ---with-lvm-stripecount=...
# --with-lvm-stripecount=...
AC_ARG_WITH([lvm-stripecount],
[AS_HELP_STRING([--with-lvm-stripecount=NUM],
[the number of stripes to use for LVM volumes]
......@@ -118,6 +118,46 @@ AC_ARG_WITH([lvm-stripecount],
[lvm_stripecount="1"])
AC_SUBST(LVM_STRIPECOUNT, $lvm_stripecount)
# --enable-drbd-barriers
AC_ARG_ENABLE([drbd-barriers],
[AS_HELP_STRING([--enable-drbd-barriers],
[enable the DRBD barrier functionality (>= 8.0.12) (default: enabled)])],
[[if test "$enableval" != no; then
DRBD_BARRIERS=True
else
DRBD_BARRIERS=False
fi
]],
[DRBD_BARRIERS=True])
AC_SUBST(DRBD_BARRIERS, $DRBD_BARRIERS)
# --enable-syslog[=no/yes/only]
AC_ARG_ENABLE([syslog],
[AS_HELP_STRING([--enable-syslog],
[enable use of syslog (default: disabled), one of no/yes/only])],
[[case "$enableval" in
no)
SYSLOG=no
;;
yes)
SYSLOG=yes
;;
only)
SYSLOG=only
;;
*)
SYSLOG=
;;
esac
]],
[SYSLOG=no])
if test -z "$SYSLOG"
then
AC_MSG_ERROR([invalid value for syslog, choose one of no/yes/only])
fi
AC_SUBST(SYSLOG_USAGE, $SYSLOG)
# Check common programs
AC_PROG_INSTALL
AC_PROG_LN_S
......
......@@ -59,7 +59,7 @@ start() {
local name="$1"; shift
# Convert daemon name to uppercase after removing "ganeti-" prefix
local ucname=$(tr a-z A-Z <<< ${name#ganeti-})
local ucname=$(echo ${name#ganeti-} | tr a-z A-Z)
# Read $<daemon>_ARGS and $EXTRA_<daemon>_ARGS
eval local args="\"\$${ucname}_ARGS \$EXTRA_${ucname}_ARGS\""
......
......@@ -53,6 +53,25 @@ import ganeti.http.server # pylint: disable-msg=W0611
queue_lock = None
def _PrepareQueueLock():
"""Try to prepare the queue lock.
@return: None for success, otherwise an exception object
"""
global queue_lock # pylint: disable-msg=W0603
if queue_lock is not None:
return None
# Prepare job queue
try:
queue_lock = jstore.InitAndVerifyQueue(must_lock=False)
return None
except EnvironmentError, err:
return err
def _RequireJobQueueLock(fn):
"""Decorator for job queue manipulating functions.
......@@ -62,6 +81,9 @@ def _RequireJobQueueLock(fn):
def wrapper(*args, **kwargs):
# Locking in exclusive, blocking mode because there could be several
# children running at the same time. Waiting up to 10 seconds.
if _PrepareQueueLock() is not None:
raise errors.JobQueueError("Job queue failed initialization,"
" cannot update jobs")
queue_lock.Exclusive(blocking=True, timeout=QUEUE_LOCK_TIMEOUT)
try:
return fn(*args, **kwargs)
......@@ -805,8 +827,6 @@ def ExecNoded(options, _):
"""Main node daemon function, executed with the PID file held.
"""
global queue_lock # pylint: disable-msg=W0603
# Read SSL certificate
if options.ssl:
ssl_params = http.HttpSslParams(ssl_key_path=options.ssl_key,
......@@ -814,8 +834,12 @@ def ExecNoded(options, _):
else:
ssl_params = None
# Prepare job queue
queue_lock = jstore.InitAndVerifyQueue(must_lock=False)
err = _PrepareQueueLock()
if err is not None:
# this might be some kind of file-system/permission error; while
# this breaks the job queue functionality, we shouldn't prevent
# startup of the whole node daemon because of this
logging.critical("Can't init/verify the queue, proceeding anyway: %s", err)
mainloop = daemon.Mainloop()
server = NodeHttpServer(mainloop, options.bind_address, options.port,
......
......@@ -486,6 +486,8 @@ def main():
try:
# on master or not, try to start the node dameon
EnsureDaemon(constants.NODED)
# start confd as well. On non candidates it will be in disabled mode.
EnsureDaemon(constants.CONFD)
notepad = WatcherState()
try:
......
......@@ -884,7 +884,7 @@ def _GetVGInfo(vg_name):
"vg_free": int(round(float(valarr[1]), 0)),
"pv_count": int(valarr[2]),
}
except ValueError, err:
except (TypeError, ValueError), err:
logging.exception("Fail to parse vgs output: %s", err)
else:
logging.error("vgs output has the wrong number of fields (expected"
......@@ -1926,19 +1926,15 @@ def BlockdevSnapshot(disk):
@return: snapshot disk path
"""
if disk.children:
if len(disk.children) == 1:
# only one child, let's recurse on it
return BlockdevSnapshot(disk.children[0])
else:
# more than one child, choose one that matches
for child in disk.children:
if child.size == disk.size:
# return implies breaking the loop
return BlockdevSnapshot(child)
if disk.dev_type == constants.LD_DRBD8:
if not disk.children:
_Fail("DRBD device '%s' without backing storage cannot be snapshotted",
disk.unique_id)
return BlockdevSnapshot(disk.children[0])
elif disk.dev_type == constants.LD_LV:
r_dev = _RecursiveFindBD(disk)
if r_dev is not None:
# FIXME: choose a saner value for the snapshot size
# let's stay on the safe side and ask for the full size, for now
return r_dev.Snapshot(disk.size)
else:
......
......@@ -512,7 +512,7 @@ class LogicalVolume(BlockDev):
try:
major = int(major)
minor = int(minor)
except ValueError, err:
except (TypeError, ValueError), err:
logging.error("lvs major/minor cannot be parsed: %s", str(err))
try:
......@@ -933,7 +933,7 @@ class BaseDRBD(BlockDev): # pylint: disable-msg=W0223
result.fail_reason, result.output)
try:
sectors = int(result.stdout)
except ValueError:
except (TypeError, ValueError):
_ThrowError("Invalid output from blockdev: '%s'", result.stdout)
bytes = sectors * 512
if bytes < 128 * 1024 * 1024: # less than 128MiB
......@@ -1215,6 +1215,24 @@ class DRBD8(BaseDRBD):
"--create-device"]
if size:
args.extend(["-d", "%sm" % size])
if not constants.DRBD_BARRIERS: # disable barriers, if configured so
version = cls._GetVersion()
# various DRBD versions support different disk barrier options;
# what we aim here is to revert back to the 'drain' method of
# disk flushes and to disable metadata barriers, in effect going
# back to pre-8.0.7 behaviour
vmaj = version['k_major']
vmin = version['k_minor']
vrel = version['k_point']
assert vmaj == 8
if vmin == 0: # 8.0.x
if vrel >= 12:
args.extend(['-i', '-m'])
elif vmin == 2: # 8.2.x
if vrel >= 7:
args.extend(['-i', '-m'])
elif vmaj >= 3: # 8.3.x or newer
args.extend(['-i', '-a', 'm'])
result = utils.RunCmd(args)
if result.failed:
_ThrowError("drbd%d: can't attach local disk: %s", minor, result.output)
......
......@@ -1561,7 +1561,7 @@ def GenerateTable(headers, fields, separator, data,
if unitfields.Matches(fields[idx]):
try:
val = int(val)
except ValueError:
except (TypeError, ValueError):
pass
else:
val = row[idx] = utils.FormatUnit(val, units)
......@@ -1642,7 +1642,7 @@ def ParseTimespec(value):
if value[-1] not in suffix_map:
try:
value = int(value)
except ValueError:
except (TypeError, ValueError):
raise errors.OpPrereqError("Invalid time specification '%s'" % value)
else:
multiplier = suffix_map[value[-1]]
......@@ -1652,7 +1652,7 @@ def ParseTimespec(value):
" suffix passed)")
try:
value = int(value) * multiplier
except ValueError:
except (TypeError, ValueError):
raise errors.OpPrereqError("Invalid time specification '%s'" % value)
return value
......
......@@ -2424,8 +2424,11 @@ class LURemoveNode(LogicalUnit):
"NODE_NAME": self.op.node_name,
}
all_nodes = self.cfg.GetNodeList()
if self.op.node_name in all_nodes:
try:
all_nodes.remove(self.op.node_name)
except ValueError:
logging.warning("Node %s which is about to be removed not found"
" in the all nodes list", self.op.node_name)
return env, all_nodes, all_nodes
def CheckPrereq(self):
......@@ -3203,7 +3206,7 @@ class LUSetNodeParams(LogicalUnit):
# If we're being deofflined/drained, we'll MC ourself if needed
if (deoffline_or_drain and not offline_or_drain and not
self.op.master_candidate == True):
self.op.master_candidate == True and not node.master_candidate):
self.op.master_candidate = _DecideSelfPromotion(self)
if self.op.master_candidate:
self.LogInfo("Autopromoting node to master candidate")
......@@ -5814,7 +5817,7 @@ class LUCreateInstance(LogicalUnit):
raise errors.OpPrereqError("Missing disk size", errors.ECODE_INVAL)
try:
size = int(size)
except ValueError:
except (TypeError, ValueError):
raise errors.OpPrereqError("Invalid disk size '%s'" % size,
errors.ECODE_INVAL)
self.disks.append({"size": size, "mode": mode})
......@@ -7474,7 +7477,7 @@ class LUSetInstanceParams(LogicalUnit):
errors.ECODE_INVAL)
try:
size = int(size)
except ValueError, err:
except (TypeError, ValueError), err:
raise errors.OpPrereqError("Invalid disk size parameter: %s" %
str(err), errors.ECODE_INVAL)
disk_dict['size'] = size
......
......@@ -149,6 +149,13 @@ LOG_WATCHER = LOG_DIR + "watcher.log"
LOG_COMMANDS = LOG_DIR + "commands.log"
LOG_BURNIN = LOG_DIR + "burnin.log"
# one of 'no', 'yes', 'only'
SYSLOG_USAGE = _autoconf.SYSLOG_USAGE
SYSLOG_NO = "no"
SYSLOG_YES = "yes"
SYSLOG_ONLY = "only"
SYSLOG_SOCKET = "/dev/log"
OS_SEARCH_PATH = _autoconf.OS_SEARCH_PATH
EXPORT_DIR = _autoconf.EXPORT_DIR
......@@ -249,6 +256,7 @@ LDS_BLOCK = frozenset([LD_LV, LD_DRBD8])
# drbd constants
DRBD_HMAC_ALG = "md5"
DRBD_NET_PROTOCOL = "C"
DRBD_BARRIERS = _autoconf.DRBD_BARRIERS
# file backend driver
FD_LOOP = "loop"
......
......@@ -253,6 +253,12 @@ def GenericMain(daemon_name, optionparser, dirs, check_fn, exec_fn,
optionparser.add_option("-d", "--debug", dest="debug",
help="Enable some debug messages",
default=False, action="store_true")
optionparser.add_option("--syslog", dest="syslog",
help="Enable logging to syslog (except debug"
" messages); one of 'no', 'yes' or 'only' [%s]" %
constants.SYSLOG_USAGE,
default=constants.SYSLOG_USAGE,
choices=["no", "yes", "only"])
if daemon_name in constants.DAEMONS_PORTS:
default_bind_address = "0.0.0.0"
......@@ -316,7 +322,9 @@ def GenericMain(daemon_name, optionparser, dirs, check_fn, exec_fn,
utils.SetupLogging(logfile=constants.DAEMONS_LOGFILES[daemon_name],
debug=options.debug,
stderr_logging=not options.fork,
multithreaded=multithreaded)
multithreaded=multithreaded,
program=daemon_name,
syslog=options.syslog)
logging.info("%s daemon startup", daemon_name)
exec_fn(options, args)
finally:
......
......@@ -1022,7 +1022,7 @@ class HttpMessageReader(object):
if hdr_content_length:
try:
self.content_length = int(hdr_content_length)
except ValueError:
except (TypeError, ValueError):
self.content_length = None
if self.content_length is not None and self.content_length < 0:
self.content_length = None
......
......@@ -137,7 +137,7 @@ class _HttpServerToClientMessageReader(http.HttpMessageReader):
status = int(status)
if status < 100 or status > 999:
status = -1
except ValueError:
except (TypeError, ValueError):
status = -1
if status == -1:
......
......@@ -134,7 +134,7 @@ class XenHypervisor(hv_base.BaseHypervisor):
data[2] = int(data[2])
data[3] = int(data[3])
data[5] = float(data[5])
except ValueError, err:
except (TypeError, ValueError), err:
raise errors.HypervisorError("Can't parse output of xm list,"
" line: %s, error: %s" % (line, err))
......@@ -496,9 +496,9 @@ class XenPvmHypervisor(XenHypervisor):
ip = getattr(nic, "ip", None)
if ip is not None:
nic_str += ", ip=%s" % ip
vif_data.append("'%s'" % nic_str)
if nic.nicparams[constants.NIC_MODE] == constants.NIC_MODE_BRIDGED:
nic_str += ", bridge=%s" % nic.nicparams[constants.NIC_LINK]
vif_data.append("'%s'" % nic_str)
disk_data = cls._GetConfigFileDiskData(block_devices)
......@@ -624,9 +624,9 @@ class XenHvmHypervisor(XenHypervisor):
ip = getattr(nic, "ip", None)
if ip is not None:
nic_str += ", ip=%s" % ip
vif_data.append("'%s'" % nic_str)
if nic.nicparams[constants.NIC_MODE] == constants.NIC_MODE_BRIDGED:
nic_str += ", bridge=%s" % nic.nicparams[constants.NIC_LINK]
vif_data.append("'%s'" % nic_str)
config.write("vif = [%s]\n" % ",".join(vif_data))
disk_data = cls._GetConfigFileDiskData(block_devices)
......
......@@ -216,7 +216,7 @@ class SingleNotifyPipeCondition(_BaseCondition):
"""
__slots__ = _BaseCondition.__slots__ + [
__slots__ = [
"_poller",
"_read_fd",
"_write_fd",
......@@ -309,7 +309,7 @@ class PipeCondition(_BaseCondition):
there are any waiting threads.
"""
__slots__ = _BaseCondition.__slots__ + [
__slots__ = [
"_nwaiters",
"_single_condition",
]
......@@ -600,7 +600,7 @@ class SharedLock(object):
def acquire(self, shared=0, timeout=None, test_notify=None):
"""Acquire a shared lock.
@type shared: int
@type shared: integer (0/1) used as a boolean
@param shared: whether to acquire in shared mode; by default an
exclusive lock will be acquired
@type timeout: float
......@@ -713,6 +713,7 @@ class LockSet:
def __init__(self, members=None):
"""Constructs a new LockSet.
@type members: list of strings
@param members: initial members of the set
"""
......@@ -811,8 +812,10 @@ class LockSet:
def acquire(self, names, timeout=None, shared=0, test_notify=None):
"""Acquire a set of resource locks.
@type names: list of strings (or string)
@param names: the names of the locks which shall be acquired
(special lock names, or instance/node names)
@type shared: integer (0/1) used as a boolean
@param shared: whether to acquire in shared mode; by default an
exclusive lock will be acquired
@type timeout: float or None
......@@ -968,6 +971,7 @@ class LockSet:
You must have acquired the locks, either in shared or in exclusive mode,
before releasing them.
@type names: list of strings, or None
@param names: the names of the locks which shall be released
(defaults to all the locks acquired at that level).
......@@ -1001,8 +1005,11 @@ class LockSet:
def add(self, names, acquired=0, shared=0):
"""Add a new set of elements to the set
@type names: list of strings
@param names: names of the new elements to add
@type acquired: integer (0/1) used as a boolean
@param acquired: pre-acquire the new resource?
@type shared: integer (0/1) used as a boolean
@param shared: is the pre-acquisition shared?
"""
......@@ -1062,6 +1069,7 @@ class LockSet:
You can either not hold anything in the lockset or already hold a superset
of the elements you want to delete, exclusively.
@type names: list of strings
@param names: names of the resource to remove.
@return: a list of locks which we removed; the list is always
......@@ -1227,10 +1235,12 @@ class GanetiLockManager:
def acquire(self, level, names, timeout=None, shared=0):
"""Acquire a set of resource locks, at the same level.
@param level: the level at which the locks shall be acquired;
it must be a member of LEVELS.
@type level: member of locking.LEVELS
@param level: the level at which the locks shall be acquired
@type names: list of strings (or string)
@param names: the names of the locks which shall be acquired
(special lock names, or instance/node names)
@type shared: integer (0/1) used as a boolean
@param shared: whether to acquire in shared mode; by default
an exclusive lock will be acquired
@type timeout: float
......@@ -1261,8 +1271,9 @@ class GanetiLockManager:
You must have acquired the locks, either in shared or in exclusive
mode, before releasing them.
@param level: the level at which the locks shall be released;
it must be a member of LEVELS
@type level: member of locking.LEVELS
@param level: the level at which the locks shall be released
@type names: list of strings, or None
@param names: the names of the locks which shall be released
(defaults to all the locks acquired at that level)
......@@ -1281,10 +1292,13 @@ class GanetiLockManager:
def add(self, level, names, acquired=0, shared=0):
"""Add locks at the specified level.
@param level: the level at which the locks shall be added;
it must be a member of LEVELS_MOD.
@type level: member of locking.LEVELS_MOD
@param level: the level at which the locks shall be added
@type names: list of strings
@param names: names of the locks to acquire
@type acquired: integer (0/1) used as a boolean
@param acquired: whether to acquire the newly added locks
@type shared: integer (0/1) used as a boolean
@param shared: whether the acquisition will be shared
"""
......@@ -1301,8 +1315,9 @@ class GanetiLockManager:
You must either already own the locks you are trying to remove
exclusively or not own any lock at an upper level.
@param level: the level at which the locks shall be removed;
it must be a member of LEVELS_MOD
@type level: member of locking.LEVELS_MOD
@param level: the level at which the locks shall be removed
@type names: list of strings
@param names: the names of the locks which shall be removed
(special lock names, or instance/node names)
......
......@@ -223,7 +223,7 @@ class TaggableObject(ConfigObject):
"""An generic class supporting tags.
"""
__slots__ = ConfigObject.__slots__ + ["tags"]
__slots__ = ["tags"]
VALID_TAG_RE = re.compile("^[\w.+*/:@-]+$")
@classmethod
......@@ -635,7 +635,7 @@ class Disk(ConfigObject):
class Instance(TaggableObject):
"""Config object representing an instance."""
__slots__ = TaggableObject.__slots__ + [
__slots__ = [
"name",
"primary_node",
"os",
......@@ -747,7 +747,7 @@ class Instance(TaggableObject):
try:
idx = int(idx)
return self.disks[idx]
except ValueError, err:
except (TypeError, ValueError), err:
raise errors.OpPrereqError("Invalid disk index: '%s'" % str(err),
errors.ECODE_INVAL)
except IndexError:
......@@ -815,7 +815,7 @@ class OS(ConfigObject):
class Node(TaggableObject):
"""Config object representing a node."""
__slots__ = TaggableObject.__slots__ + [
__slots__ = [
"name",
"primary_ip",
"secondary_ip",
......@@ -828,7 +828,7 @@ class Node(TaggableObject):
class Cluster(TaggableObject):
"""Config object representing the cluster."""
__slots__ = TaggableObject.__slots__ + [
__slots__ = [
"serial_no",
"rsahostkeypub",
"highest_used_port",
......
......@@ -109,7 +109,7 @@ class OpCode(BaseOpCode):
"""
OP_ID = "OP_ABSTRACT"
__slots__ = BaseOpCode.__slots__ + ["dry_run"]
__slots__ = ["dry_run"]
def __getstate__(self):
"""Specialized getstate for opcodes.
......@@ -178,7 +178,7 @@ class OpPostInitCluster(OpCode):
"""
OP_ID = "OP_CLUSTER_POST_INIT"
__slots__ = OpCode.__slots__ + []
__slots__ = []
class OpDestroyCluster(OpCode):
......@@ -189,13 +189,13 @@ class OpDestroyCluster(OpCode):
"""
OP_ID = "OP_CLUSTER_DESTROY"
__slots__ = OpCode.__slots__ + []
__slots__ = []
class OpQueryClusterInfo(OpCode):
"""Query cluster information."""
OP_ID = "OP_CLUSTER_QUERY"
__slots__ = OpCode.__slots__ + []
__slots__ = []
class OpVerifyCluster(OpCode):
......@@ -209,8 +209,8 @@ class OpVerifyCluster(OpCode):
"""
OP_ID = "OP_CLUSTER_VERIFY"
__slots__ = OpCode.__slots__ + ["skip_checks", "verbose", "error_codes",
"debug_simulate_errors"]
__slots__ = ["skip_checks", "verbose", "error_codes",
"debug_simulate_errors"]
class OpVerifyDisks(OpCode):
......@@ -235,7 +235,7 @@ class OpVerifyDisks(OpCode):
"""