Commit b9f8d18f authored by Michael Hanselmann's avatar Michael Hanselmann
Browse files

Merge branch 'devel-2.1'

* devel-2.1: (38 commits)
  Bump version to 2.1.0~rc4
  Update man page of gnt-instance
  KVM: fix pylint warning
  KVM: be more resilient on broken migration answers
  Allow filtering for (node-)tags
  Add unittests for cli.GenerateTable
  cli: Fix bug when not using headers
  daemon-util: Fix quoting issue
  Bump version to 2.1.0~rc3
  Switch the SplitKeyVal function to accept escapes
  Fix long-standing race condition bug in locking unittest
  confd client: copy the peers in UpdatePeerList
  testutils: Print name of test program before running it
  Don't use hardcoded name for pylint
  Partially revert "Makefile.am: Run pylint on all Python code"
  build-bash-completion: Take care of pylint warnings
  Add an UnescapeAndSplit function
  Makefile.am: Run pylint on all Python code
  Small improvements for release script
  check-python-code: Use “set -e” to abort on errors
  ...
parents 2feecf12 d80cb8c4
......@@ -254,6 +254,8 @@ EXTRA_DIST = \
doc/examples/ganeti.cron.in \
doc/examples/gnt-config-backup.in \
doc/examples/dumb-allocator \
doc/examples/ganeti.default \
doc/examples/ganeti.default-debug \
doc/examples/hooks/ethers \
doc/examples/hooks/ipsec.in \
test/testutils.py \
......@@ -353,9 +355,15 @@ srclink_files = \
$(all_python_code)
check_python_code = \
autotools/build-bash-completion \
$(BUILD_BASH_COMPLETION) \
$(all_python_code)
lint_python_code = \
ganeti \
$(dist_sbin_SCRIPTS) \
$(dist_tools_SCRIPTS) \
$(BUILD_BASH_COMPLETION)
devel/upload: devel/upload.in $(REPLACE_VARS_SED)
sed -f $(REPLACE_VARS_SED) < $< > $@
chmod u+x $@
......@@ -477,7 +485,8 @@ check-local:
.PHONY: lint
lint: ganeti
pylint ganeti $(dist_sbin_SCRIPTS) $(dist_tools_SCRIPTS)
@test -n "$(PYLINT)" || { echo 'pylint' not found during configure; exit 1; }
$(PYLINT) $(LINT_OPTS) $(lint_python_code)
# a dist hook rule for catching revision control directories
distcheck-hook:
......
......@@ -19,9 +19,14 @@
# 02110-1301, USA.
import optparse
"""Script to generate bash_completion script for Ganeti.
"""
# pylint: disable-msg=C0103
# [C0103] Invalid name build-bash-completion
import os
import sys
import re
from cStringIO import StringIO
......@@ -263,6 +268,8 @@ class CompletionWriter:
self.args = args
for opt in opts:
# While documented, these variables aren't seen as public attributes by
# pylint. pylint: disable-msg=W0212
opt.all_names = sorted(opt._short_opts + opt._long_opts)
def _FindFirstArgument(self, sw):
......@@ -591,7 +598,7 @@ def GetCommands(filename, module):
"""
try:
commands = getattr(module, "commands")
except AttributeError, err:
except AttributeError:
raise Exception("Script %s doesn't have 'commands' attribute" %
filename)
......
......@@ -18,7 +18,11 @@
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
# 02110-1301, USA.
let problems=0
set -e
# "[...] If the last ARG evaluates to 0, let returns 1; 0 is returned
# otherwise.", hence ignoring the return value.
let problems=0 || :
for script; do
if grep -n -H -F $'\t' "$script"; then
......
......@@ -2,7 +2,7 @@
m4_define([gnt_version_major], [2])
m4_define([gnt_version_minor], [1])
m4_define([gnt_version_revision], [0])
m4_define([gnt_version_suffix], [~rc2])
m4_define([gnt_version_suffix], [~rc4])
m4_define([gnt_version_full],
m4_format([%d.%d.%d%s],
gnt_version_major, gnt_version_minor,
......@@ -153,6 +153,14 @@ then
AC_MSG_WARN([dot (from the graphviz suite) not found, documentation rebuild not possible])
fi
# Check for pylint
AC_ARG_VAR(PYLINT, [pylint path])
AC_PATH_PROG(PYLINT, [pylint], [])
if test -z "$PYLINT"
then
AC_MSG_WARN([pylint not found, checking code will not be possible])
fi
# Check for socat
AC_ARG_VAR(SOCAT, [socat path])
AC_PATH_PROG(SOCAT, [socat], [])
......
......@@ -62,7 +62,7 @@ start() {
local ucname=$(tr a-z A-Z <<< ${name#ganeti-})
# Read $<daemon>_ARGS and $EXTRA_<daemon>_ARGS
eval local args="\$${ucname}_ARGS \$EXTRA_${ucname}_ARGS"
eval local args="\"\$${ucname}_ARGS \$EXTRA_${ucname}_ARGS\""
start-stop-daemon --start --quiet --oknodo \
--pidfile $(_daemon_pidfile $name) \
......
......@@ -100,7 +100,8 @@ class IOServer(SocketServer.UnixStreamServer):
def setup_queue(self):
self.context = GanetiContext()
self.request_workers = workerpool.WorkerPool(CLIENT_REQUEST_WORKERS,
self.request_workers = workerpool.WorkerPool("ClientReq",
CLIENT_REQUEST_WORKERS,
ClientRequestWorker)
def process_request(self, request, client_address):
......@@ -280,6 +281,12 @@ class ClientOps:
op = opcodes.OpQueryClusterInfo()
return self._Query(op)
elif method == luxi.REQ_QUERY_TAGS:
kind, name = args
logging.info("Received tags query request")
op = opcodes.OpGetTags(kind=kind, name=name)
return self._Query(op)
elif method == luxi.REQ_QUEUE_SET_DRAIN_FLAG:
drain_flag = args
logging.info("Received queue drain flag change request to %s",
......
......@@ -32,18 +32,35 @@ set -e
: ${URL:=git://git.ganeti.org/ganeti.git}
TAG="$1"
TMPDIR=`mktemp -d`
if [[ -z "$TAG" ]]; then
echo "Usage: $0 <tree-ish>" >&2
exit 1
fi
echo "Using Git repository $URL"
TMPDIR=$(mktemp -d -t gntrelease.XXXXXXXXXX)
cd $TMPDIR
echo "Cloning the repository under $TMPDIR ..."
git clone -q "$URL" dist
cd dist
git checkout $TAG
./autogen.sh
./configure
VERSION=$(sed -n -e '/^PACKAGE_VERSION =/ s/^PACKAGE_VERSION = // p' Makefile)
make distcheck
fakeroot make dist
tar tzvf ganeti-$VERSION.tar.gz
echo
echo 'MD5:'
md5sum ganeti-$VERSION.tar.gz
echo
echo 'SHA1:'
sha1sum ganeti-$VERSION.tar.gz
echo
echo "The archive is at $PWD/ganeti-$VERSION.tar.gz"
echo "Please copy it and remove the temporary directory when done."
......@@ -78,6 +78,10 @@ make $make_args install DESTDIR="$TXD"
install -D --mode=0755 doc/examples/ganeti.initd \
"$TXD/$SYSCONFDIR/init.d/ganeti"
[ -f doc/examples/ganeti.default-debug ] && \
install -D --mode=0644 doc/examples/ganeti.default-debug \
"$TXD/$SYSCONFDIR/default/ganeti"
[ -f doc/examples/bash_completion ] && \
install -D --mode=0644 doc/examples/bash_completion \
"$TXD/$SYSCONFDIR/bash_completion.d/ganeti"
......
# Default arguments for Ganeti daemons
NODED_ARGS=""
MASTERD_ARGS=""
RAPI_ARGS=""
CONFD_ARGS=""
# Default arguments for Ganeti daemons (debug mode)
NODED_ARGS="-d"
MASTERD_ARGS="-d"
RAPI_ARGS="-d"
CONFD_ARGS="-d"
......@@ -549,6 +549,21 @@ pass ``--enabled-hypervisors=kvm`` to the init command.
You can also invoke the command with the ``--help`` option in order to
see all the possibilities.
Hypervisor/Network/Cluster parameters
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Please note that the default hypervisor/network/cluster parameters may
not be the correct one for your environment. Carefully check them, and
change them at cluster init time, or later with ``gnt-cluster modify``.
Your instance types, networking environment, hypervisor type and version
may all affect what kind of parameters should be used on your cluster.
For example kvm instances are by default configured to use a host
kernel, and to be reached via serial console, which works nice for linux
paravirtualized instances. If you want fully virtualized instances you
may want to handle their kernel inside the instance, and to use VNC.
Joining the nodes to the cluster
++++++++++++++++++++++++++++++++
......
......@@ -107,7 +107,7 @@ def GenerateHmacKey(file_name):
@param file_name: Path to output file
"""
utils.WriteFile(file_name, data=utils.GenerateSecret(), mode=0400)
utils.WriteFile(file_name, data="%s\n" % utils.GenerateSecret(), mode=0400)
def _InitGanetiServerSetup(master_name):
......
......@@ -252,7 +252,6 @@ ARGS_ONE_INSTANCE = [ArgInstance(min=1, max=1)]
ARGS_ONE_NODE = [ArgNode(min=1, max=1)]
def _ExtractTagsObject(opts, args):
"""Extract the tag type object.
......@@ -313,8 +312,8 @@ def ListTags(opts, args):
"""
kind, name = _ExtractTagsObject(opts, args)
op = opcodes.OpGetTags(kind=kind, name=name)
result = SubmitOpCode(op)
cl = GetClient()
result = cl.QueryTags(kind, name)
result = list(result)
result.sort()
for tag in result:
......@@ -385,7 +384,7 @@ def _SplitKeyVal(opt, data):
"""
kv_dict = {}
if data:
for elem in data.split(","):
for elem in utils.UnescapeAndSplit(data, sep=","):
if "=" in elem:
key, val = elem.split("=", 1)
else:
......@@ -1581,6 +1580,12 @@ def GenerateTable(headers, fields, separator, data,
args.append(hdr)
result.append(format % tuple(args))
if separator is None:
assert len(mlens) == len(fields)
if fields and not numfields.Matches(fields[-1]):
mlens[-1] = 0
for line in data:
args = []
if line is None:
......
......@@ -1843,7 +1843,8 @@ class LURenameCluster(LogicalUnit):
"NEW_NAME": self.op.name,
}
mn = self.cfg.GetMasterNode()
return env, [mn], [mn]
all_nodes = self.cfg.GetNodeList()
return env, [mn], all_nodes
def CheckPrereq(self):
"""Verify that the passed name is a valid one.
......@@ -5769,16 +5770,14 @@ class LUCreateInstance(LogicalUnit):
# MAC address verification
mac = nic.get("mac", constants.VALUE_AUTO)
if mac not in (constants.VALUE_AUTO, constants.VALUE_GENERATE):
if not utils.IsValidMac(mac.lower()):
raise errors.OpPrereqError("Invalid MAC address specified: %s" %
mac, errors.ECODE_INVAL)
else:
try:
self.cfg.ReserveMAC(mac, self.proc.GetECId())
except errors.ReservationError:
raise errors.OpPrereqError("MAC address %s already in use"
" in cluster" % mac,
errors.ECODE_NOTUNIQUE)
mac = utils.NormalizeAndValidateMac(mac)
try:
self.cfg.ReserveMAC(mac, self.proc.GetECId())
except errors.ReservationError:
raise errors.OpPrereqError("MAC address %s already in use"
" in cluster" % mac,
errors.ECODE_NOTUNIQUE)
# bridge verification
bridge = nic.get("bridge", None)
......@@ -7532,9 +7531,8 @@ class LUSetInstanceParams(LogicalUnit):
if 'mac' in nic_dict:
nic_mac = nic_dict['mac']
if nic_mac not in (constants.VALUE_AUTO, constants.VALUE_GENERATE):
if not utils.IsValidMac(nic_mac):
raise errors.OpPrereqError("Invalid MAC address %s" % nic_mac,
errors.ECODE_INVAL)
nic_mac = utils.NormalizeAndValidateMac(nic_mac)
if nic_op != constants.DDM_ADD and nic_mac == constants.VALUE_AUTO:
raise errors.OpPrereqError("'auto' is not a valid MAC address when"
" modifying an existing nic",
......
......@@ -133,7 +133,8 @@ class ConfdClient:
# pylint: disable-msg=W0201
if not isinstance(peers, list):
raise errors.ProgrammerError("peers must be a list")
self._peers = peers
# make a copy of peers, since we're going to shuffle the list, later
self._peers = list(peers)
def _PackRequest(self, request, now=None):
"""Prepare a request to be sent on the wire.
......
......@@ -95,6 +95,14 @@ class HttpClientRequest(object):
self.resp_headers = None
self.resp_body = None
def __repr__(self):
status = ["%s.%s" % (self.__class__.__module__, self.__class__.__name__),
"%s:%s" % (self.host, self.port),
self.method,
self.path]
return "<%s at %#x>" % (" ".join(status), id(self))
class _HttpClientToServerMessageWriter(http.HttpMessageWriter):
pass
......@@ -328,6 +336,12 @@ class _HttpClientPendingRequest(object):
# Thread synchronization
self.done = threading.Event()
def __repr__(self):
status = ["%s.%s" % (self.__class__.__module__, self.__class__.__name__),
"req=%r" % self.request]
return "<%s at %#x>" % (" ".join(status), id(self))
class HttpClientWorker(workerpool.BaseWorker):
"""HTTP client worker class.
......@@ -342,7 +356,8 @@ class HttpClientWorker(workerpool.BaseWorker):
class HttpClientWorkerPool(workerpool.WorkerPool):
def __init__(self, manager):
workerpool.WorkerPool.__init__(self, HTTP_CLIENT_THREADS,
workerpool.WorkerPool.__init__(self, "HttpClient",
HTTP_CLIENT_THREADS,
HttpClientWorker)
self.manager = manager
......
......@@ -80,6 +80,8 @@ class KVMHypervisor(hv_base.BaseHypervisor):
_MIGRATION_STATUS_RE = re.compile('Migration\s+status:\s+(\w+)',
re.M | re.I)
_MIGRATION_INFO_MAX_BAD_ANSWERS = 5
_MIGRATION_INFO_RETRY_DELAY = 2
_KVM_NETWORK_SCRIPT = constants.SYSCONFDIR + "/ganeti/kvm-vif-bridge"
......@@ -675,26 +677,37 @@ class KVMHypervisor(hv_base.BaseHypervisor):
info_command = 'info migrate'
done = False
broken_answers = 0
while not done:
result = self._CallMonitorCommand(instance_name, info_command)
match = self._MIGRATION_STATUS_RE.search(result.stdout)
if not match:
raise errors.HypervisorError("Unknown 'info migrate' result: %s" %
result.stdout)
broken_answers += 1
if not result.stdout:
logging.info("KVM: empty 'info migrate' result")
else:
logging.warning("KVM: unknown 'info migrate' result: %s",
result.stdout)
time.sleep(self._MIGRATION_INFO_RETRY_DELAY)
else:
status = match.group(1)
if status == 'completed':
done = True
elif status == 'active':
time.sleep(2)
# reset the broken answers count
broken_answers = 0
time.sleep(self._MIGRATION_INFO_RETRY_DELAY)
elif status == 'failed' or status == 'cancelled':
if not live:
self._CallMonitorCommand(instance_name, 'cont')
raise errors.HypervisorError("Migration %s at the kvm level" %
status)
else:
logging.info("KVM: unknown migration status '%s'", status)
time.sleep(2)
logging.warning("KVM: unknown migration status '%s'", status)
broken_answers += 1
time.sleep(self._MIGRATION_INFO_RETRY_DELAY)
if broken_answers >= self._MIGRATION_INFO_MAX_BAD_ANSWERS:
raise errors.HypervisorError("Too many 'info migrate' broken answers")
utils.KillProcess(pid)
self._RemoveInstanceRuntimeFiles(pidfile, instance_name)
......
......@@ -190,6 +190,13 @@ class _QueuedJob(object):
# Condition to wait for changes
self.change = threading.Condition(self.queue._lock)
def __repr__(self):
status = ["%s.%s" % (self.__class__.__module__, self.__class__.__name__),
"id=%s" % self.id,
"ops=%s" % ",".join([op.input.Summary() for op in self.ops])]
return "<%s at %#x>" % (" ".join(status), id(self))
@classmethod
def Restore(cls, queue, state):
"""Restore a _QueuedJob from serialized state:
......@@ -430,8 +437,7 @@ class _JobQueueWorker(workerpool.BaseWorker):
@param job: the job to be processed
"""
logging.info("Worker %s processing job %s",
self.worker_id, job.id)
logging.info("Processing job %s", job.id)
proc = mcpu.Processor(self.pool.queue.context, job.id)
queue = job.queue
try:
......@@ -527,8 +533,7 @@ class _JobQueueWorker(workerpool.BaseWorker):
finally:
queue.release()
logging.info("Worker %s finished job %s, status = %s",
self.worker_id, job_id, status)
logging.info("Finished job %s, status = %s", job_id, status)
class _JobQueueWorkerPool(workerpool.WorkerPool):
......@@ -536,7 +541,8 @@ class _JobQueueWorkerPool(workerpool.WorkerPool):
"""
def __init__(self, queue):
super(_JobQueueWorkerPool, self).__init__(JOBQUEUE_THREADS,
super(_JobQueueWorkerPool, self).__init__("JobQueue",
JOBQUEUE_THREADS,
_JobQueueWorker)
self.queue = queue
......@@ -1315,7 +1321,7 @@ class JobQueue(object):
all_job_ids = self._GetJobIDsUnlocked(archived=False)
pending = []
for idx, job_id in enumerate(all_job_ids):
last_touched = idx
last_touched = idx + 1
# Not optimal because jobs could be pending
# TODO: Measure average duration for job archival and take number of
......@@ -1345,7 +1351,7 @@ class JobQueue(object):
if pending:
archived_count += self._ArchiveJobsUnlocked(pending)
return (archived_count, len(all_job_ids) - last_touched - 1)
return (archived_count, len(all_job_ids) - last_touched)
@staticmethod
def _GetJobInfoUnlocked(job, fields):
......
......@@ -775,7 +775,9 @@ class LockSet:
def _release_and_delete_owned(self):
"""Release and delete all resources owned by the current thread"""
for lname in self._list_owned():
self.__lockdict[lname].release()
lock = self.__lockdict[lname]
if lock._is_owned():
lock.release()
self._del_owned(name=lname)
def __names(self):
......@@ -839,8 +841,6 @@ class LockSet:
# Support passing in a single resource to acquire rather than many
if isinstance(names, basestring):
names = [names]
else:
names = sorted(names)
return self.__acquire_inner(names, False, shared,
running_timeout.Remaining, test_notify)
......@@ -889,11 +889,11 @@ class LockSet:
# First we look the locks up on __lockdict. We have no way of being sure
# they will still be there after, but this makes it a lot faster should
# just one of them be the already wrong
for lname in utils.UniqueSequence(names):
# just one of them be the already wrong. Using a sorted sequence to prevent
# deadlocks.
for lname in sorted(utils.UniqueSequence(names)):
try:
lock = self.__lockdict[lname] # raises KeyError if lock is not there
acquire_list.append((lname, lock))
except KeyError:
if want_all:
# We are acquiring all the set, it doesn't matter if this particular
......@@ -902,6 +902,8 @@ class LockSet:
raise errors.LockError("Non-existing lock in set (%s)" % lname)
acquire_list.append((lname, lock))
# This will hold the locknames we effectively acquired.
acquired = set()
......
......@@ -57,6 +57,7 @@ REQ_QUERY_NODES = "QueryNodes"
REQ_QUERY_EXPORTS = "QueryExports"
REQ_QUERY_CONFIG_VALUES = "QueryConfigValues"
REQ_QUERY_CLUSTER_INFO = "QueryClusterInfo"
REQ_QUERY_TAGS = "QueryTags"
REQ_QUEUE_SET_DRAIN_FLAG = "SetDrainFlag"
REQ_SET_WATCHER_PAUSE = "SetWatcherPause"
......@@ -444,5 +445,8 @@ class Client(object):
def QueryConfigValues(self, fields):
return self.CallMethod(REQ_QUERY_CONFIG_VALUES, fields)
def QueryTags(self, kind, name):
return self.CallMethod(REQ_QUERY_TAGS, (kind, name))
# TODO: class Server(object)
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment