Commit d91427ea authored by Iustin Pop's avatar Iustin Pop
Browse files

Merge commit 'origin/branch-2.1' into feature/containers

* commit 'origin/branch-2.1': (66 commits)
  Add automated disk repair changes to design doc
  Add review script
  Implement “gnt-node physical-volumes” command
  Add new opcode to list physical volumes
  storage: Use constants.py instead of local constants
  storage: Fix semantics for directory size
  Add “gnt-job watch” command
  jqueue: Fix error when WaitForJobChange gets invalid ID
  jqueue: Update message for cancelling running job
  cmdlib: Change tasklet logging to debug level
  rapi: Add /2/nodes/[node_name]/migrate resource
  gnt-node: Use new opcode to migrate node
  cmdlib: Add new opcode to migrate node
  rapi: Add default parameter to _checkIntVariable
  cmdlib: Add logging for tasklets
  cmdlib: Fix tasklets handling if no tasklets are added
  rapi: Add /2/[node_name]/evacuate resource
  Add information about storage units framework
  Add RPC calls for storage unit list
  Add first implementation of generic storage unit framework
  ...
parents 961166d9 395aa879
......@@ -6,6 +6,7 @@
# global ignores
*.py[co]
*.swp
# /
/Makefile
......
......@@ -85,6 +85,7 @@ pkgpython_PYTHON = \
lib/serializer.py \
lib/ssconf.py \
lib/ssh.py \
lib/storage.py \
lib/utils.py \
lib/workerpool.py
......@@ -112,6 +113,7 @@ http_PYTHON = \
docrst = \
doc/admin.rst \
doc/design-2.0.rst \
doc/design-2.1.rst \
doc/glossary.rst \
doc/hooks.rst \
doc/iallocator.rst \
......
......@@ -43,6 +43,7 @@ from optparse import OptionParser
from ganeti import config
from ganeti import constants
from ganeti import daemon
from ganeti import mcpu
from ganeti import opcodes
from ganeti import jqueue
......@@ -383,35 +384,6 @@ class GanetiContext(object):
self.glm.remove(locking.LEVEL_NODE, name)
def ParseOptions():
"""Parse the command line options.
@return: (options, args) as from OptionParser.parse_args()
"""
parser = OptionParser(description="Ganeti master daemon",
usage="%prog [-f] [-d]",
version="%%prog (ganeti) %s" %
constants.RELEASE_VERSION)
parser.add_option("-f", "--foreground", dest="fork",
help="Don't detach from the current terminal",
default=True, action="store_false")
parser.add_option("-d", "--debug", dest="debug",
help="Enable some debug messages",
default=False, action="store_true")
parser.add_option("--no-voting", dest="no_voting",
help="Do not check that the nodes agree on this node"
" being the master and start the daemon unconditionally",
default=False, action="store_true")
parser.add_option("--yes-do-it", dest="yes_do_it",
help="Override interactive check for --no-voting",
default=False, action="store_true")
options, args = parser.parse_args()
return options, args
def CheckAgreement():
"""Check the agreement on who is the master.
......@@ -468,17 +440,10 @@ def CheckAgreement():
return result
def CheckMASTERD(options, args):
"""Initial checks whether to run or exit with a failure
def main():
"""Main function"""
options, args = ParseOptions()
utils.debug = options.debug
utils.no_fork = True
if options.fork:
utils.CloseFDs()
"""
rpc.Init()
try:
ssconf.CheckMaster(options.debug)
......@@ -496,31 +461,20 @@ def main():
elif not options.no_voting:
if not CheckAgreement():
return
dirs = [(constants.RUN_GANETI_DIR, constants.RUN_DIRS_MODE),
(constants.SOCKET_DIR, constants.SOCKET_DIR_MODE),
]
utils.EnsureDirs(dirs)
# This is safe to do as the pid file guarantees against
# concurrent execution.
utils.RemoveFile(constants.MASTER_SOCKET)
master = IOServer(constants.MASTER_SOCKET, ClientRqHandler)
finally:
rpc.Shutdown()
# become a daemon
if options.fork:
utils.Daemonize(logfile=constants.LOG_MASTERDAEMON)
utils.WritePidFile(constants.MASTERD_PID)
try:
utils.SetupLogging(constants.LOG_MASTERDAEMON, debug=options.debug,
stderr_logging=not options.fork, multithreaded=True)
def ExecMASTERD(options, args):
"""Main MASTERD function, executed with the pidfile held.
logging.info("Ganeti master daemon startup")
"""
# This is safe to do as the pid file guarantees against
# concurrent execution.
utils.RemoveFile(constants.MASTER_SOCKET)
master = IOServer(constants.MASTER_SOCKET, ClientRqHandler)
try:
rpc.Init()
try:
# activate ip
......@@ -538,9 +492,27 @@ def main():
finally:
rpc.Shutdown()
finally:
utils.RemovePidFile(constants.MASTERD_PID)
utils.RemoveFile(constants.MASTER_SOCKET)
def main():
"""Main function"""
parser = OptionParser(description="Ganeti master daemon",
usage="%prog [-f] [-d]",
version="%%prog (ganeti) %s" %
constants.RELEASE_VERSION)
parser.add_option("--no-voting", dest="no_voting",
help="Do not check that the nodes agree on this node"
" being the master and start the daemon unconditionally",
default=False, action="store_true")
parser.add_option("--yes-do-it", dest="yes_do_it",
help="Override interactive check for --no-voting",
default=False, action="store_true")
dirs = [(constants.RUN_GANETI_DIR, constants.RUN_DIRS_MODE),
(constants.SOCKET_DIR, constants.SOCKET_DIR_MODE),
]
daemon.GenericMain(constants.MASTERD, parser, dirs,
CheckMASTERD, ExecMASTERD)
if __name__ == "__main__":
main()
......@@ -42,6 +42,7 @@ from ganeti import jstore
from ganeti import daemon
from ganeti import http
from ganeti import utils
from ganeti import storage
import ganeti.http.server
......@@ -354,6 +355,16 @@ class NodeHttpServer(http.server.HttpServer):
"""
return backend.ListVolumeGroups()
# Storage --------------------------
@staticmethod
def perspective_storage_list(params):
"""Get list of storage units.
"""
(su_name, su_args, name, fields) = params
return storage.GetStorage(su_name, *su_args).List(name, fields)
# bridge --------------------------
@staticmethod
......@@ -732,86 +743,44 @@ class NodeHttpServer(http.server.HttpServer):
return backend.ValidateHVParams(hvname, hvparams)
def ParseOptions():
"""Parse the command line options.
@return: (options, args) as from OptionParser.parse_args()
def ExecNODED(options, args):
"""Main NODED function, executed with the pidfile held.
"""
parser = OptionParser(description="Ganeti node daemon",
usage="%prog [-f] [-d] [-b ADDRESS]",
version="%%prog (ganeti) %s" %
constants.RELEASE_VERSION)
global queue_lock
parser.add_option("-f", "--foreground", dest="fork",
help="Don't detach from the current terminal",
default=True, action="store_false")
parser.add_option("-d", "--debug", dest="debug",
help="Enable some debug messages",
default=False, action="store_true")
parser.add_option("-b", "--bind", dest="bind_address",
help="Bind address",
default="", metavar="ADDRESS")
# Read SSL certificate
if options.ssl:
ssl_params = http.HttpSslParams(ssl_key_path=options.ssl_key,
ssl_cert_path=options.ssl_cert)
else:
ssl_params = None
options, args = parser.parse_args()
return options, args
# Prepare job queue
queue_lock = jstore.InitAndVerifyQueue(must_lock=False)
mainloop = daemon.Mainloop()
server = NodeHttpServer(mainloop, options.bind_address, options.port,
ssl_params=ssl_params, ssl_verify_peer=True)
server.Start()
try:
mainloop.Run()
finally:
server.Stop()
def main():
"""Main function for the node daemon.
"""
global queue_lock
options, args = ParseOptions()
utils.debug = options.debug
if options.fork:
utils.CloseFDs()
for fname in (constants.SSL_CERT_FILE,):
if not os.path.isfile(fname):
print "config %s not there, will not run." % fname
sys.exit(5)
try:
port = utils.GetNodeDaemonPort()
except errors.ConfigurationError, err:
print "Cluster configuration incomplete: '%s'" % str(err)
sys.exit(5)
parser = OptionParser(description="Ganeti node daemon",
usage="%prog [-f] [-d] [-p port] [-b ADDRESS]",
version="%%prog (ganeti) %s" %
constants.RELEASE_VERSION)
dirs = [(val, constants.RUN_DIRS_MODE) for val in constants.SUB_RUN_DIRS]
dirs.append((constants.LOG_OS_DIR, 0750))
dirs.append((constants.LOCK_DIR, 1777))
utils.EnsureDirs(dirs)
# become a daemon
if options.fork:
utils.Daemonize(logfile=constants.LOG_NODESERVER)
utils.WritePidFile(constants.NODED_PID)
try:
utils.SetupLogging(logfile=constants.LOG_NODESERVER, debug=options.debug,
stderr_logging=not options.fork)
logging.info("ganeti node daemon startup")
# Read SSL certificate
ssl_params = http.HttpSslParams(ssl_key_path=constants.SSL_CERT_FILE,
ssl_cert_path=constants.SSL_CERT_FILE)
# Prepare job queue
queue_lock = jstore.InitAndVerifyQueue(must_lock=False)
mainloop = daemon.Mainloop()
server = NodeHttpServer(mainloop, options.bind_address, port,
ssl_params=ssl_params, ssl_verify_peer=True)
server.Start()
try:
mainloop.Run()
finally:
server.Stop()
finally:
utils.RemovePidFile(constants.NODED_PID)
daemon.GenericMain(constants.NODED, parser, dirs, None, ExecNODED)
if __name__ == '__main__':
......
......@@ -177,92 +177,51 @@ class RemoteApiHttpServer(http.auth.HttpServerRequestAuthentication,
return result
def ParseOptions():
"""Parse the command line options.
@return: (options, args) as from OptionParser.parse_args()
def CheckRAPI(options, args):
"""Initial checks whether to run or exit with a failure
"""
parser = optparse.OptionParser(description="Ganeti Remote API",
usage="%prog [-d] [-p port]",
version="%%prog (ganeti) %s" %
constants.RAPI_VERSION)
parser.add_option("-d", "--debug", dest="debug",
help="Enable some debug messages",
default=False, action="store_true")
parser.add_option("-p", "--port", dest="port",
help="Port to run API (%s default)." %
constants.RAPI_PORT,
default=constants.RAPI_PORT, type="int")
parser.add_option("--no-ssl", dest="ssl",
help="Do not secure HTTP protocol with SSL",
default=True, action="store_false")
parser.add_option("-K", "--ssl-key", dest="ssl_key",
help="SSL key",
default=constants.RAPI_CERT_FILE, type="string")
parser.add_option("-C", "--ssl-cert", dest="ssl_cert",
help="SSL certificate",
default=constants.RAPI_CERT_FILE, type="string")
parser.add_option("-f", "--foreground", dest="fork",
help="Don't detach from the current terminal",
default=True, action="store_false")
options, args = parser.parse_args()
if len(args) != 0:
print >> sys.stderr, "Usage: %s [-d] [-p port]" % sys.argv[0]
sys.exit(1)
print >> sys.stderr, "Usage: %s [-f] [-d] [-p port] [-b ADDRESS]" % \
sys.argv[0]
sys.exit(constants.EXIT_FAILURE)
if options.ssl and not (options.ssl_cert and options.ssl_key):
print >> sys.stderr, ("For secure mode please provide "
"--ssl-key and --ssl-cert arguments")
sys.exit(1)
ssconf.CheckMaster(options.debug)
return options, args
def main():
"""Main function.
def ExecRAPI(options, args):
"""Main RAPI function, executed with the pidfile held.
"""
options, args = ParseOptions()
if options.fork:
utils.CloseFDs()
# Read SSL certificate
if options.ssl:
# Read SSL certificate
try:
ssl_params = http.HttpSslParams(ssl_key_path=options.ssl_key,
ssl_cert_path=options.ssl_cert)
except Exception, err:
sys.stderr.write("Can't load the SSL certificate/key: %s\n" % (err,))
sys.exit(1)
ssl_params = http.HttpSslParams(ssl_key_path=options.ssl_key,
ssl_cert_path=options.ssl_cert)
else:
ssl_params = None
ssconf.CheckMaster(options.debug)
mainloop = daemon.Mainloop()
server = RemoteApiHttpServer(mainloop, options.bind_address, options.port,
ssl_params=ssl_params, ssl_verify_peer=False,
request_executor_class=JsonErrorRequestExecutor)
server.Start()
try:
mainloop.Run()
finally:
server.Stop()
if options.fork:
utils.Daemonize(logfile=constants.LOG_RAPISERVER)
utils.SetupLogging(constants.LOG_RAPISERVER, debug=options.debug,
stderr_logging=not options.fork)
def main():
"""Main function.
utils.WritePidFile(constants.RAPI_PID)
try:
mainloop = daemon.Mainloop()
server = RemoteApiHttpServer(mainloop, "", options.port,
ssl_params=ssl_params, ssl_verify_peer=False,
request_executor_class=
JsonErrorRequestExecutor)
server.Start()
try:
mainloop.Run()
finally:
server.Stop()
finally:
utils.RemovePidFile(constants.RAPI_PID)
"""
parser = optparse.OptionParser(description="Ganeti Remote API",
usage="%prog [-f] [-d] [-p port] [-b ADDRESS]",
version="%%prog (ganeti) %s" % constants.RAPI_VERSION)
dirs = [(val, constants.RUN_DIRS_MODE) for val in constants.SUB_RUN_DIRS]
dirs.append((constants.LOG_OS_DIR, 0750))
daemon.GenericMain(constants.RAPI, parser, dirs, CheckRAPI, ExecRAPI)
if __name__ == '__main__':
......
......@@ -478,9 +478,8 @@ def main():
update_file = False
try:
# on master or not, try to start the node dameon (use _PID but is
# the same as daemon name)
EnsureDaemon(constants.NODED_PID)
# on master or not, try to start the node dameon
EnsureDaemon(constants.NODED)
notepad = WatcherState()
try:
......@@ -500,8 +499,8 @@ def main():
# else retry the connection
client = cli.GetClient()
# we are on master now (use _PID but is the same as daemon name)
EnsureDaemon(constants.RAPI_PID)
# we are on master now
EnsureDaemon(constants.RAPI)
try:
watcher = Watcher(options, notepad)
......
#!/bin/bash
# Copyright (C) 2009 Google Inc.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
# 02110-1301, USA.
# To set user mappings, use this command:
# git config gnt-review.johndoe 'John Doe <johndoe@domain.tld>'
set -e
# Get absolute path to myself
me_plain="$0"
me=$(readlink -f "$me_plain")
add_reviewed_by() {
local msgfile="$1"
grep -q '^Reviewed-by: ' "$msgfile" && return
perl -i -e '
my $sob = 0;
while (<>) {
if ($sob == 0 and m/^Signed-off-by:/) {
$sob = 1;
} elsif ($sob == 1 and not m/^Signed-off-by:/) {
print "Reviewed-by: \n";
$sob = -1;
}
print;
}
if ($sob == 1) {
print "Reviewed-by: \n";
}
' "$msgfile"
}
replace_users() {
local msgfile="$1"
perl -i -e '
sub map_username {
my ($name) = @_;
return $name unless $name;
my @cmd = ("git", "config", "--get", "gnt-review.$name");
open(my $fh, "-|", @cmd) or die "Command \"@cmd\" failed: $!";
my $output = do { local $/ = undef; <$fh> };
close($fh);
if ($? == 0) {
chomp $output;
$output =~ s/\s+/ /;
return $output;
}
return $name;
}
while (<>) {
if (m/^Reviewed-by:(.*)$/) {
my @names = grep {
# Ignore empty entries
!/^$/
} map {
# Normalize whitespace
$_ =~ s/(^\s+|\s+$)//g;
$_ =~ s/\s+/ /g;
# Map names
$_ = map_username($_);
$_;
} split(m/,/, $1);
foreach (sort @names) {
print "Reviewed-by: $_\n";
}
} else {
print;
}
}
' "$msgfile"
if ! grep -q '^Reviewed-by: ' "$msgfile"
then
echo 'Missing Reviewed-by: line' >&2
sleep 1
return 1
fi
return 0
}
run_editor() {
local filename="$1"
local editor=${EDITOR:-vi}
local args
case "$(basename "$editor")" in
vi* | *vim)
# Start edit mode at Reviewed-by: line
args='+/^Reviewed-by: +nohlsearch +startinsert!'
;;
*)
args=
;;
esac
$editor $args "$filename"
}
commit_editor() {
local msgfile="$1"
local tmpf=$(mktemp)
trap "rm -f $tmpf" EXIT
cp "$msgfile" "$tmpf"
while :
do
add_reviewed_by "$tmpf"
run_editor "$tmpf"
replace_users "$tmpf" && break
done
cp "$tmpf" "$msgfile"
}
copy_commit() {
local rev="$1" target_branch="$2"
echo "Copying commit $rev ..."
git cherry-pick -n "$rev"
GIT_EDITOR="$me --commit-editor \"\$@\"" git commit -c "$rev" -s
}
main() {
local range="$1" target_branch="$2"
if [[ -z "$target_branch" || "$range" != *..* ]]
then
echo "Usage: $me_plain <from..to> <target-branch>" >&2
exit 1
fi
git checkout "$target_branch"
local old_head=$(git rev-parse HEAD)
for rev in $(git rev-list --reverse "$range")
do
copy_commit "$rev"
done
git log "$old_head..$target_branch"
}
if [[ "$1" == --commit-editor ]]
then
shift
commit_editor "$@"