Commit 1f7d3f7d authored by René Nussbaumer's avatar René Nussbaumer Committed by Michael Hanselmann

Adding tool for automated cluster-merger

This is the implementation of docs/design-cluster-merger.rst. It allows
the automatic merging of one or more clusters into the invoking cluster.

While this version is tested and working it still needs some tweaking
here and there for error handling and user experience.
Signed-off-by: default avatarRené Nussbaumer <rn@google.com>
Reviewed-by: default avatarMichael Hanselmann <hansmi@google.com>
parent 72ca1dcb
...@@ -233,6 +233,7 @@ dist_tools_SCRIPTS = \ ...@@ -233,6 +233,7 @@ dist_tools_SCRIPTS = \
tools/burnin \ tools/burnin \
tools/cfgshell \ tools/cfgshell \
tools/cfgupgrade \ tools/cfgupgrade \
tools/cluster-merge \
tools/lvmstrap tools/lvmstrap
pkglib_SCRIPTS = \ pkglib_SCRIPTS = \
......
#!/usr/bin/python
#
# Copyright (C) 2010 Google Inc.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
# 02110-1301, USA.
"""Tool to merge two or more clusters together.
The clusters have to run the same version of Ganeti!
"""
# pylint: disable-msg=C0103
# C0103: Invalid name cluster-merge
import logging
import os
import optparse
import shutil
import sys
import tempfile
from ganeti import cli
from ganeti import config
from ganeti import constants
from ganeti import errors
from ganeti import ssh
from ganeti import utils
PAUSE_PERIOD_OPT = cli.cli_option("-p", "--watcher-pause-period", default=1800,
action="store", type="int",
dest="pause_period",
help=("Amount of time in seconds watcher"
" should be suspended from running"))
def Flatten(unflatten_list):
"""Flattens a list.
@param unflatten_list: A list of unflatten list objects.
@return: A flatten list
"""
flatten_list = []
for item in unflatten_list:
if isinstance(item, list):
flatten_list.extend(Flatten(item))
else:
flatten_list.append(item)
return flatten_list
class MergerData(object):
"""Container class to hold data used for merger.
"""
def __init__(self, cluster, key_path, nodes, instances, config_path=None):
"""Initialize the container.
@param cluster: The name of the cluster
@param key_path: Path to the ssh private key used for authentication
@param config_path: Path to the merging cluster config
@param nodes: List of nodes in the merging cluster
@param instances: List of instances running on merging cluster
"""
self.cluster = cluster
self.key_path = key_path
self.config_path = config_path
self.instances = instances
self.nodes = nodes
class Merger(object):
"""Handling the merge.
"""
def __init__(self, clusters, pause_period):
"""Initialize object with sane defaults and infos required.
@param clusters: The list of clusters to merge in
@param pause_period: The time watcher shall be disabled for
"""
self.merger_data = []
self.clusters = clusters
self.pause_period = pause_period
self.work_dir = tempfile.mkdtemp(suffix="cluster-merger")
self.cluster_name = cli.GetClient().QueryConfigValues(["cluster_name"])
self.ssh_runner = ssh.SshRunner(self.cluster_name)
def Setup(self):
"""Sets up our end so we can do the merger.
This method is setting us up as a preparation for the merger.
It makes the initial contact and gathers information needed.
@raise errors.RemoteError: for errors in communication/grabbing
"""
(remote_path, _, _) = ssh.GetUserFiles("root")
# Fetch remotes private key
for cluster in self.clusters:
result = self._RunCmd(cluster, "cat %s" % remote_path, batch=False,
ask_key=False)
if result.failed:
raise errors.RemoteError("There was an error while grabbing ssh private"
" key from %s. Fail reason: %s; output: %s" %
(cluster, result.fail_reason, result.output))
key_path = os.path.join(self.work_dir, cluster)
utils.WriteFile(key_path, mode=0600, data=result.stdout)
result = self._RunCmd(cluster, "gnt-node list -o name --no-header",
private_key=key_path)
if result.failed:
raise errors.RemoteError("Unable to retrieve list of nodes from %s."
" Fail reason: %s; output: %s" %
(cluster, result.fail_reason, result.output))
nodes = result.stdout.splitlines()
result = self._RunCmd(cluster, "gnt-instance list -o name --no-header",
private_key=key_path)
if result.failed:
raise errors.RemoteError("Unable to retrieve list of instances from"
" %s. Fail reason: %s; output: %s" %
(cluster, result.fail_reason, result.output))
instances = result.stdout.splitlines()
self.merger_data.append(MergerData(cluster, key_path, nodes, instances))
def _PrepareAuthorizedKeys(self):
"""Prepare the authorized_keys on every merging node.
This method add our public key to remotes authorized_key for further
communication.
"""
(_, pub_key_file, auth_keys) = ssh.GetUserFiles("root")
pub_key = utils.ReadFile(pub_key_file)
for data in self.merger_data:
for node in data.nodes:
result = self._RunCmd(node, ("cat >> %s << '!EOF.'\n%s!EOF.\n" %
(auth_keys, pub_key)),
private_key=data.key_path)
if result.failed:
raise errors.RemoteError("Unable to add our public key to %s in %s."
" Fail reason: %s; output: %s" %
(node, data.cluster, result.fail_reason,
result.output))
def _RunCmd(self, hostname, command, user="root", use_cluster_key=False,
strict_host_check=False, private_key=None, batch=True,
ask_key=False):
"""Wrapping SshRunner.Run with default parameters.
For explanation of parameters see L{ssh.SshRunner.Run}.
"""
return self.ssh_runner.Run(hostname=hostname, command=command, user=user,
use_cluster_key=use_cluster_key,
strict_host_check=strict_host_check,
private_key=private_key, batch=batch,
ask_key=ask_key)
def _StopMergingInstances(self):
"""Stop instances on merging clusters.
"""
for cluster in self.clusters:
result = self._RunCmd(cluster, "gnt-instance shutdown --all"
" --force-multiple")
if result.failed:
raise errors.RemoteError("Unable to stop instances on %s."
" Fail reason: %s; output: %s" %
(cluster, result.fail_reason, result.output))
def _DisableWatcher(self):
"""Disable watch on all merging clusters, including ourself.
"""
for cluster in ["localhost"] + self.clusters:
result = self._RunCmd(cluster, "gnt-cluster watcher pause %d" %
self.pause_period)
if result.failed:
raise errors.RemoteError("Unable to pause watcher on %s."
" Fail reason: %s; output: %s" %
(cluster, result.fail_reason, result.output))
# R0201: Method could be a function
def _EnableWatcher(self): # pylint: disable-msg=R0201
"""Reenable watcher (locally).
"""
result = utils.RunCmd(["gnt-cluster", "watcher", "continue"])
if result.failed:
logging.warning("Unable to continue watcher. Fail reason: %s;"
" output: %s" % (result.fail_reason,
result.output))
def _StopDaemons(self):
"""Stop all daemons on merging nodes.
"""
# FIXME: Worth to put this into constants?
cmds = []
for daemon in (constants.RAPI, constants.MASTERD,
constants.NODED, constants.CONFD):
cmds.append("%s stop %s" % (constants.DAEMON_UTIL, daemon))
for data in self.merger_data:
for node in data.nodes:
result = self._RunCmd(node, " && ".join(cmds))
if result.failed:
raise errors.RemoteError("Unable to stop daemons on %s."
" Fail reason: %s; output: %s." %
(node, result.fail_reason, result.output))
def _FetchRemoteConfig(self):
"""Fetches and stores remote cluster config from the master.
This step is needed before we can merge the config.
"""
for data in self.merger_data:
result = self._RunCmd(data.cluster, "cat %s" %
constants.CLUSTER_CONF_FILE)
if result.failed:
raise errors.RemoteError("Unable to retrieve remote config on %s."
" Fail reason: %s; output %s" %
(data.cluster, result.fail_reason,
result.output))
data.config_path = os.path.join(self.work_dir, "%s_config.data" %
data.cluster)
utils.WriteFile(data.config_path, data=result.stdout)
# R0201: Method could be a function
def _KillMasterDaemon(self): # pylint: disable-msg=R0201
"""Kills the local master daemon.
@raise errors.CommandError: If unable to kill
"""
result = utils.RunCmd([constants.DAEMON_UTIL, "stop-master"])
if result.failed:
raise errors.CommandError("Unable to stop master daemons."
" Fail reason: %s; output: %s" %
(result.fail_reason, result.output))
def _MergeConfig(self):
"""Merges all foreign config into our own config.
"""
my_config = config.ConfigWriter(offline=True)
fake_ec_id = 0 # Needs to be uniq over the whole config merge
for data in self.merger_data:
other_config = config.ConfigWriter(data.config_path)
for node in other_config.GetNodeList():
node_info = other_config.GetNodeInfo(node)
node_info.master_candidate = False
my_config.AddNode(node_info, str(fake_ec_id))
fake_ec_id += 1
for instance in other_config.GetInstanceList():
instance_info = other_config.GetInstanceInfo(instance)
# Update the DRBD port assignments
# This is a little bit hackish
for dsk in instance_info.disks:
if dsk.dev_type in constants.LDS_DRBD:
port = my_config.AllocatePort()
logical_id = list(dsk.logical_id)
logical_id[2] = port
dsk.logical_id = tuple(logical_id)
physical_id = list(dsk.physical_id)
physical_id[1] = physical_id[3] = port
dsk.physical_id = tuple(physical_id)
my_config.AddInstance(instance_info, str(fake_ec_id))
fake_ec_id += 1
# R0201: Method could be a function
def _StartMasterDaemon(self, no_vote=False): # pylint: disable-msg=R0201
"""Starts the local master daemon.
@param no_vote: Should the masterd started without voting? default: False
@raise errors.CommandError: If unable to start daemon.
"""
env = {}
if no_vote:
env["EXTRA_MASTERD_ARGS"] = "--no-voting --yes-do-it"
result = utils.RunCmd([constants.DAEMON_UTIL, "start-master"], env=env)
if result.failed:
raise errors.CommandError("Couldn't start ganeti master."
" Fail reason: %s; output: %s" %
(result.fail_reason, result.output))
def _ReaddMergedNodesAndRedist(self):
"""Readds all merging nodes and make sure their config is up-to-date.
@raise errors.CommandError: If anything fails.
"""
for data in self.merger_data:
for node in data.nodes:
result = utils.RunCmd(["gnt-node", "add", "--readd",
"--no-ssh-key-check", node])
if result.failed:
raise errors.CommandError("Couldn't readd node %s. Fail reason: %s;"
" output: %s" % (node, result.fail_reason,
result.output))
result = utils.RunCmd(["gnt-cluster", "redist-conf"])
if result.failed:
raise errors.CommandError("Redistribution failed. Fail reason: %s;"
" output: %s" % (result.fail_reason,
result.output))
# R0201: Method could be a function
def _StartupAllInstances(self): # pylint: disable-msg=R0201
"""Starts up all instances (locally).
@raise errors.CommandError: If unable to start clusters
"""
result = utils.RunCmd(["gnt-instance", "startup", "--all",
"--force-multiple"])
if result.failed:
raise errors.CommandError("Unable to start all instances."
" Fail reason: %s; output: %s" %
(result.fail_reason, result.output))
# R0201: Method could be a function
def _VerifyCluster(self): # pylint: disable-msg=R0201
"""Runs gnt-cluster verify to verify the health.
@raise errors.ProgrammError: If cluster fails on verification
"""
result = utils.RunCmd(["gnt-cluster", "verify"])
if result.failed:
raise errors.CommandError("Verification of cluster failed."
" Fail reason: %s; output: %s" %
(result.fail_reason, result.output))
def Merge(self):
"""Does the actual merge.
It runs all the steps in the right order and updates the user about steps
taken. Also it keeps track of rollback_steps to undo everything.
"""
rbsteps = []
try:
logging.info("Pre cluster verification")
self._VerifyCluster()
logging.info("Prepare authorized_keys")
rbsteps.append("Remove our key from authorized_keys on nodes:"
" %(nodes)s")
self._PrepareAuthorizedKeys()
rbsteps.append("Start all instances again on the merging"
" clusters: %(clusters)s")
logging.info("Stopping merging instances (takes a while)")
self._StopMergingInstances()
logging.info("Disable watcher")
self._DisableWatcher()
logging.info("Stop daemons on merging nodes")
self._StopDaemons()
logging.info("Merging config")
self._FetchRemoteConfig()
self._KillMasterDaemon()
rbsteps.append("Restore %s from another master candidate" %
constants.CLUSTER_CONF_FILE)
self._MergeConfig()
self._StartMasterDaemon(no_vote=True)
# Point of no return, delete rbsteps
del rbsteps[:]
logging.warning("We are at the point of no return. Merge can not easily"
" be undone after this point.")
logging.info("Readd nodes and redistribute config")
self._ReaddMergedNodesAndRedist()
self._KillMasterDaemon()
self._StartMasterDaemon()
logging.info("Starting instances again")
self._StartupAllInstances()
logging.info("Post cluster verification")
self._VerifyCluster()
except errors.GenericError, e:
logging.exception(e)
if rbsteps:
nodes = Flatten([data.nodes for data in self.merger_data])
info = {
"clusters": self.clusters,
"nodes": nodes,
}
logging.critical("In order to rollback do the following:")
for step in rbsteps:
logging.critical(" * %s" % (step % info))
else:
logging.critical("Nothing to rollback.")
# TODO: Keep track of steps done for a flawless resume?
def Cleanup(self):
"""Clean up our environment.
This cleans up remote private keys and configs and after that
deletes the temporary directory.
"""
shutil.rmtree(self.work_dir)
def SetupLogging(options):
"""Setting up logging infrastructure.
@param options: Parsed command line options
"""
formatter = logging.Formatter("%(asctime)s: %(levelname)s %(message)s")
stderr_handler = logging.StreamHandler()
stderr_handler.setFormatter(formatter)
if options.debug:
stderr_handler.setLevel(logging.NOTSET)
elif options.verbose:
stderr_handler.setLevel(logging.INFO)
else:
stderr_handler.setLevel(logging.ERROR)
root_logger = logging.getLogger("")
root_logger.setLevel(logging.NOTSET)
root_logger.addHandler(stderr_handler)
def main():
"""Main routine.
"""
program = os.path.basename(sys.argv[0])
parser = optparse.OptionParser(usage=("%prog [--debug|--verbose]"
" [--watcher-pause-period SECONDS]"
" <cluster> <cluster...>"),
prog=program)
parser.add_option(cli.DEBUG_OPT)
parser.add_option(cli.VERBOSE_OPT)
parser.add_option(PAUSE_PERIOD_OPT)
(options, args) = parser.parse_args()
SetupLogging(options)
if not args:
parser.error("No clusters specified")
cluster_merger = Merger(utils.UniqueSequence(args), options.pause_period)
try:
try:
cluster_merger.Setup()
cluster_merger.Merge()
except errors.GenericError, e:
logging.exception(e)
return constants.EXIT_FAILURE
finally:
cluster_merger.Cleanup()
return constants.EXIT_SUCCESS
if __name__ == "__main__":
sys.exit(main())
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment