Commit b954f097 authored by Constantinos Venetsanopoulos's avatar Constantinos Venetsanopoulos Committed by Iustin Pop

Add the gnt-storage client

Add a new client called 'gnt-storage'.
The client interacts with the ExtStorage interface, similarly to
the way gnt-os interacts with the OS interface.

For now, only two commands are supported: 'info' and 'diagnose'.

'diagnose' calculates the node status of each provider on each node,
similarly to gnt-os diagnose. Furthermore, for every provider, it
calculates it's nodegroup validity for each nodegroup. This is done
inside the LU and not the client (marked as 'TODO' for the  global
validity of gnt-os diagnose).

In the future, gnt-storage can be used to manage storage pools,
or even be extended to diagnose other storage types supported by
Ganeti, such as lvm, drbd (INT_MIRROR) or rbd (EXT_MIRROR).
Signed-off-by: default avatarConstantinos Venetsanopoulos <cven@grnet.gr>
Signed-off-by: default avatarIustin Pop <iustin@google.com>
[iustin@google.com: fixed Haskell compatibility and style fixes]
Reviewed-by: default avatarIustin Pop <iustin@google.com>
parent c64c3854
......@@ -282,7 +282,8 @@ client_PYTHON = \
lib/client/gnt_job.py \
lib/client/gnt_node.py \
lib/client/gnt_network.py \
lib/client/gnt_os.py
lib/client/gnt_os.py \
lib/client/gnt_storage.py
hypervisor_PYTHON = \
lib/hypervisor/__init__.py \
......@@ -621,7 +622,8 @@ gnt_scripts = \
scripts/gnt-job \
scripts/gnt-network \
scripts/gnt-node \
scripts/gnt-os
scripts/gnt-os \
scripts/gnt-storage
PYTHON_BOOTSTRAP_SBIN = \
daemons/ganeti-masterd \
......
......@@ -357,6 +357,8 @@ class CompletionWriter:
WriteCompReply(sw, "-W \"$(_ganeti_instances)\"", cur=cur)
elif suggest == cli.OPT_COMPL_ONE_OS:
WriteCompReply(sw, "-W \"$(_ganeti_os)\"", cur=cur)
elif suggest == cli.OPT_COMPL_ONE_EXTSTORAGE:
WriteCompReply(sw, "-W \"$(_ganeti_extstorage)\"", cur=cur)
elif suggest == cli.OPT_COMPL_ONE_IALLOCATOR:
WriteCompReply(sw, "-W \"$(_ganeti_iallocator)\"", cur=cur)
elif suggest == cli.OPT_COMPL_ONE_NODEGROUP:
......@@ -467,6 +469,8 @@ class CompletionWriter:
choices = "$(_ganeti_jobs)"
elif isinstance(arg, cli.ArgOs):
choices = "$(_ganeti_os)"
elif isinstance(arg, cli.ArgExtStorage):
choices = "$(_ganeti_extstorage)"
elif isinstance(arg, cli.ArgFile):
choices = ""
compgenargs.append("-f")
......
......@@ -291,6 +291,8 @@ instance Arbitrary OpCodes.OpCode where
genMaybe genNameNE <*> genMaybe genNamesNE
"OP_OS_DIAGNOSE" ->
OpCodes.OpOsDiagnose <$> genFieldsNE <*> genNamesNE
"OP_EXT_STORAGE_DIAGNOSE" ->
OpCodes.OpOsDiagnose <$> genFieldsNE <*> genNamesNE
"OP_BACKUP_QUERY" ->
OpCodes.OpBackupQuery <$> arbitrary <*> genNodeNamesNE
"OP_BACKUP_PREPARE" ->
......
......@@ -444,6 +444,9 @@ $(genOpCode "OpCode"
, ("OpOsDiagnose",
[ pOutputFields
, pNames ])
, ("OpExtStorageDiagnose",
[ pOutputFields
, pNames ])
, ("OpBackupQuery",
[ pUseLocking
, pNodes
......
......@@ -2481,6 +2481,51 @@ def OSEnvironment(instance, inst_os, debug=0):
return result
def DiagnoseExtStorage(top_dirs=None):
"""Compute the validity for all ExtStorage Providers.
@type top_dirs: list
@param top_dirs: the list of directories in which to
search (if not given defaults to
L{pathutils.ES_SEARCH_PATH})
@rtype: list of L{objects.ExtStorage}
@return: a list of tuples (name, path, status, diagnose, parameters)
for all (potential) ExtStorage Providers under all
search paths, where:
- name is the (potential) ExtStorage Provider
- path is the full path to the ExtStorage Provider
- status True/False is the validity of the ExtStorage Provider
- diagnose is the error message for an invalid ExtStorage Provider,
otherwise empty
- parameters is a list of (name, help) parameters, if any
"""
if top_dirs is None:
top_dirs = pathutils.ES_SEARCH_PATH
result = []
for dir_name in top_dirs:
if os.path.isdir(dir_name):
try:
f_names = utils.ListVisibleFiles(dir_name)
except EnvironmentError, err:
logging.exception("Can't list the ExtStorage directory %s: %s",
dir_name, err)
break
for name in f_names:
es_path = utils.PathJoin(dir_name, name)
status, es_inst = bdev.ExtStorageFromDisk(name, base_dir=dir_name)
if status:
diagnose = ""
parameters = es_inst.supported_parameters
else:
diagnose = es_inst
parameters = []
result.append((name, es_path, status, diagnose, parameters))
return result
def BlockdevGrow(disk, amount, dryrun, backingstore):
"""Grow a stack of block devices.
......
......@@ -262,6 +262,7 @@ __all__ = [
"ArgNetwork",
"ArgNode",
"ArgOs",
"ArgExtStorage",
"ArgSuggest",
"ArgUnknown",
"OPT_COMPL_INST_ADD_NODES",
......@@ -272,6 +273,7 @@ __all__ = [
"OPT_COMPL_ONE_NODEGROUP",
"OPT_COMPL_ONE_NETWORK",
"OPT_COMPL_ONE_OS",
"OPT_COMPL_ONE_EXTSTORAGE",
"cli_option",
"SplitNodeOption",
"CalculateOSNames",
......@@ -422,6 +424,12 @@ class ArgOs(_Argument):
"""
class ArgExtStorage(_Argument):
"""ExtStorage argument.
"""
ARGS_NONE = []
ARGS_MANY_INSTANCES = [ArgInstance()]
ARGS_MANY_NETWORKS = [ArgNetwork()]
......@@ -672,16 +680,18 @@ def check_maybefloat(option, opt, value): # pylint: disable=W0613
OPT_COMPL_ONE_NODE,
OPT_COMPL_ONE_INSTANCE,
OPT_COMPL_ONE_OS,
OPT_COMPL_ONE_EXTSTORAGE,
OPT_COMPL_ONE_IALLOCATOR,
OPT_COMPL_ONE_NETWORK,
OPT_COMPL_INST_ADD_NODES,
OPT_COMPL_ONE_NODEGROUP) = range(100, 108)
OPT_COMPL_ONE_NODEGROUP) = range(100, 109)
OPT_COMPL_ALL = compat.UniqueFrozenset([
OPT_COMPL_MANY_NODES,
OPT_COMPL_ONE_NODE,
OPT_COMPL_ONE_INSTANCE,
OPT_COMPL_ONE_OS,
OPT_COMPL_ONE_EXTSTORAGE,
OPT_COMPL_ONE_IALLOCATOR,
OPT_COMPL_ONE_NETWORK,
OPT_COMPL_INST_ADD_NODES,
......
#
#
# Copyright (C) 2012 Google Inc.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
# 02110-1301, USA.
"""External Storage related commands"""
# pylint: disable=W0401,W0613,W0614,C0103
# W0401: Wildcard import ganeti.cli
# W0613: Unused argument, since all functions follow the same API
# W0614: Unused import %s from wildcard import (since we need cli)
# C0103: Invalid name gnt-storage
from ganeti.cli import *
from ganeti import opcodes
from ganeti import utils
def ShowExtStorageInfo(opts, args):
"""List detailed information about ExtStorage providers.
@param opts: the command line options selected by the user
@type args: list
@param args: empty list or list of ExtStorage providers' names
@rtype: int
@return: the desired exit code
"""
op = opcodes.OpExtStorageDiagnose(output_fields=["name", "nodegroup_status",
"parameters"],
names=[])
result = SubmitOpCode(op, opts=opts)
if not result:
ToStderr("Can't get the ExtStorage providers list")
return 1
do_filter = bool(args)
for (name, nodegroup_data, parameters) in result:
if do_filter:
if name not in args:
continue
else:
args.remove(name)
nodegroups_valid = []
for nodegroup_name, nodegroup_status in nodegroup_data.iteritems():
if nodegroup_status:
nodegroups_valid.append(nodegroup_name)
ToStdout("%s:", name)
if nodegroups_valid != []:
ToStdout(" - Valid for nodegroups:")
for ndgrp in utils.NiceSort(nodegroups_valid):
ToStdout(" %s", ndgrp)
ToStdout(" - Supported parameters:")
for pname, pdesc in parameters:
ToStdout(" %s: %s", pname, pdesc)
else:
ToStdout(" - Invalid for all nodegroups")
ToStdout("")
if args:
for name in args:
ToStdout("%s: Not Found", name)
ToStdout("")
return 0
def _ExtStorageStatus(status, diagnose):
"""Beautifier function for ExtStorage status.
@type status: boolean
@param status: is the ExtStorage provider valid
@type diagnose: string
@param diagnose: the error message for invalid ExtStorages
@rtype: string
@return: a formatted status
"""
if status:
return "valid"
else:
return "invalid - %s" % diagnose
def DiagnoseExtStorage(opts, args):
"""Analyse all ExtStorage providers.
@param opts: the command line options selected by the user
@type args: list
@param args: should be an empty list
@rtype: int
@return: the desired exit code
"""
op = opcodes.OpExtStorageDiagnose(output_fields=["name", "node_status",
"nodegroup_status"],
names=[])
result = SubmitOpCode(op, opts=opts)
if not result:
ToStderr("Can't get the list of ExtStorage providers")
return 1
for provider_name, node_data, nodegroup_data in result:
nodes_valid = {}
nodes_bad = {}
nodegroups_valid = {}
nodegroups_bad = {}
# Per node diagnose
for node_name, node_info in node_data.iteritems():
if node_info: # at least one entry in the per-node list
(fo_path, fo_status, fo_msg, fo_params) = node_info.pop(0)
fo_msg = "%s (path: %s)" % (_ExtStorageStatus(fo_status, fo_msg),
fo_path)
if fo_params:
fo_msg += (" [parameters: %s]" %
utils.CommaJoin([v[0] for v in fo_params]))
else:
fo_msg += " [no parameters]"
if fo_status:
nodes_valid[node_name] = fo_msg
else:
nodes_bad[node_name] = fo_msg
else:
nodes_bad[node_name] = "ExtStorage provider not found"
# Per nodegroup diagnose
for nodegroup_name, nodegroup_status in nodegroup_data.iteritems():
status = nodegroup_status
if status:
nodegroups_valid[nodegroup_name] = "valid"
else:
nodegroups_bad[nodegroup_name] = "invalid"
def _OutputPerNodegroupStatus(msg_map):
map_k = utils.NiceSort(msg_map.keys())
for nodegroup in map_k:
ToStdout(" For nodegroup: %s --> %s", nodegroup,
msg_map[nodegroup])
def _OutputPerNodeStatus(msg_map):
map_k = utils.NiceSort(msg_map.keys())
for node_name in map_k:
ToStdout(" Node: %s, status: %s", node_name, msg_map[node_name])
# Print the output
st_msg = "Provider: %s" % provider_name
ToStdout(st_msg)
ToStdout("---")
_OutputPerNodeStatus(nodes_valid)
_OutputPerNodeStatus(nodes_bad)
ToStdout(" --")
_OutputPerNodegroupStatus(nodegroups_valid)
_OutputPerNodegroupStatus(nodegroups_bad)
ToStdout("")
return 0
commands = {
"diagnose": (
DiagnoseExtStorage, ARGS_NONE, [PRIORITY_OPT],
"", "Diagnose all ExtStorage providers"),
"info": (
ShowExtStorageInfo, [ArgOs()], [PRIORITY_OPT],
"", "Show info about ExtStorage providers"),
}
def Main():
return GenericMain(commands)
......@@ -5163,6 +5163,159 @@ class LUOsDiagnose(NoHooksLU):
return self.oq.OldStyleQuery(self)
class _ExtStorageQuery(_QueryBase):
FIELDS = query.EXTSTORAGE_FIELDS
def ExpandNames(self, lu):
# Lock all nodes in shared mode
# Temporary removal of locks, should be reverted later
# TODO: reintroduce locks when they are lighter-weight
lu.needed_locks = {}
#self.share_locks[locking.LEVEL_NODE] = 1
#self.needed_locks[locking.LEVEL_NODE] = locking.ALL_SET
# The following variables interact with _QueryBase._GetNames
if self.names:
self.wanted = self.names
else:
self.wanted = locking.ALL_SET
self.do_locking = self.use_locking
def DeclareLocks(self, lu, level):
pass
@staticmethod
def _DiagnoseByProvider(rlist):
"""Remaps a per-node return list into an a per-provider per-node dictionary
@param rlist: a map with node names as keys and ExtStorage objects as values
@rtype: dict
@return: a dictionary with extstorage providers as keys and as
value another map, with nodes as keys and tuples of
(path, status, diagnose, parameters) as values, eg::
{"provider1": {"node1": [(/usr/lib/..., True, "", [])]
"node2": [(/srv/..., False, "missing file")]
"node3": [(/srv/..., True, "", [])]
}
"""
all_es = {}
# we build here the list of nodes that didn't fail the RPC (at RPC
# level), so that nodes with a non-responding node daemon don't
# make all OSes invalid
good_nodes = [node_name for node_name in rlist
if not rlist[node_name].fail_msg]
for node_name, nr in rlist.items():
if nr.fail_msg or not nr.payload:
continue
for (name, path, status, diagnose, params) in nr.payload:
if name not in all_es:
# build a list of nodes for this os containing empty lists
# for each node in node_list
all_es[name] = {}
for nname in good_nodes:
all_es[name][nname] = []
# convert params from [name, help] to (name, help)
params = [tuple(v) for v in params]
all_es[name][node_name].append((path, status, diagnose, params))
return all_es
def _GetQueryData(self, lu):
"""Computes the list of nodes and their attributes.
"""
# Locking is not used
assert not (compat.any(lu.glm.is_owned(level)
for level in locking.LEVELS
if level != locking.LEVEL_CLUSTER) or
self.do_locking or self.use_locking)
valid_nodes = [node.name
for node in lu.cfg.GetAllNodesInfo().values()
if not node.offline and node.vm_capable]
pol = self._DiagnoseByProvider(lu.rpc.call_extstorage_diagnose(valid_nodes))
data = {}
nodegroup_list = lu.cfg.GetNodeGroupList()
for (es_name, es_data) in pol.items():
# For every provider compute the nodegroup validity.
# To do this we need to check the validity of each node in es_data
# and then construct the corresponding nodegroup dict:
# { nodegroup1: status
# nodegroup2: status
# }
ndgrp_data = {}
for nodegroup in nodegroup_list:
ndgrp = lu.cfg.GetNodeGroup(nodegroup)
nodegroup_nodes = ndgrp.members
nodegroup_name = ndgrp.name
node_statuses = []
for node in nodegroup_nodes:
if node in valid_nodes:
if es_data[node] != []:
node_status = es_data[node][0][1]
node_statuses.append(node_status)
else:
node_statuses.append(False)
if False in node_statuses:
ndgrp_data[nodegroup_name] = False
else:
ndgrp_data[nodegroup_name] = True
# Compute the provider's parameters
parameters = set()
for idx, esl in enumerate(es_data.values()):
valid = bool(esl and esl[0][1])
if not valid:
break
node_params = esl[0][3]
if idx == 0:
# First entry
parameters.update(node_params)
else:
# Filter out inconsistent values
parameters.intersection_update(node_params)
params = list(parameters)
# Now fill all the info for this provider
info = query.ExtStorageInfo(name=es_name, node_status=es_data,
nodegroup_status=ndgrp_data,
parameters=params)
data[es_name] = info
# Prepare data in requested order
return [data[name] for name in self._GetNames(lu, pol.keys(), None)
if name in data]
class LUExtStorageDiagnose(NoHooksLU):
"""Logical unit for ExtStorage diagnose/query.
"""
REQ_BGL = False
def CheckArguments(self):
self.eq = _ExtStorageQuery(qlang.MakeSimpleFilter("name", self.op.names),
self.op.output_fields, False)
def ExpandNames(self):
self.eq.ExpandNames(self)
def Exec(self, feedback_fn):
return self.eq.OldStyleQuery(self)
class LUNodeRemove(LogicalUnit):
"""Logical unit for removing a node.
......@@ -16604,6 +16757,7 @@ _QUERY_IMPL = {
constants.QR_GROUP: _GroupQuery,
constants.QR_NETWORK: _NetworkQuery,
constants.QR_OS: _OsQuery,
constants.QR_EXTSTORAGE: _ExtStorageQuery,
constants.QR_EXPORT: _ExportQuery,
}
......
......@@ -1732,6 +1732,7 @@ QR_OS = "os"
QR_JOB = "job"
QR_EXPORT = "export"
QR_NETWORK = "network"
QR_EXTSTORAGE = "extstorage"
#: List of resources which can be queried using L{opcodes.OpQuery}
QR_VIA_OP = compat.UniqueFrozenset([
......@@ -1742,6 +1743,7 @@ QR_VIA_OP = compat.UniqueFrozenset([
QR_OS,
QR_EXPORT,
QR_NETWORK,
QR_EXTSTORAGE,
])
#: List of resources which can be queried using Local UniX Interface
......
......@@ -1807,6 +1807,17 @@ class OpOsDiagnose(OpCode):
OP_RESULT = _TOldQueryResult
# ExtStorage opcodes
class OpExtStorageDiagnose(OpCode):
"""Compute the list of external storage providers."""
OP_PARAMS = [
_POutputFields,
("names", ht.EmptyList, ht.TListOf(ht.TNonEmptyString),
"Which ExtStorage Provider to diagnose"),
]
OP_RESULT = _TOldQueryResult
# Exports opcodes
class OpBackupQuery(OpCode):
"""Compute the list of exported images."""
......
......@@ -2234,6 +2234,36 @@ def _BuildOsFields():
return _PrepareFieldList(fields, [])
class ExtStorageInfo(objects.ConfigObject):
__slots__ = [
"name",
"node_status",
"nodegroup_status",
"parameters",
]
def _BuildExtStorageFields():
"""Builds list of fields for extstorage provider queries.
"""
fields = [
(_MakeField("name", "Name", QFT_TEXT, "ExtStorage provider name"),
None, 0, _GetItemAttr("name")),
(_MakeField("node_status", "NodeStatus", QFT_OTHER,
"Status from node"),
None, 0, _GetItemAttr("node_status")),
(_MakeField("nodegroup_status", "NodegroupStatus", QFT_OTHER,
"Overall Nodegroup status"),
None, 0, _GetItemAttr("nodegroup_status")),
(_MakeField("parameters", "Parameters", QFT_OTHER,
"ExtStorage provider parameters"),
None, 0, _GetItemAttr("parameters")),
]
return _PrepareFieldList(fields, [])
def _JobUnavailInner(fn, ctx, (job_id, job)): # pylint: disable=W0613
"""Return L{_FS_UNAVAIL} if job is None.
......@@ -2595,6 +2625,9 @@ GROUP_FIELDS = _BuildGroupFields()
#: Fields available for operating system queries
OS_FIELDS = _BuildOsFields()
#: Fields available for extstorage provider queries
EXTSTORAGE_FIELDS = _BuildExtStorageFields()
#: Fields available for job queries
JOB_FIELDS = _BuildJobFields()
......@@ -2612,6 +2645,7 @@ ALL_FIELDS = {
constants.QR_LOCK: LOCK_FIELDS,
constants.QR_GROUP: GROUP_FIELDS,
constants.QR_OS: OS_FIELDS,
constants.QR_EXTSTORAGE: EXTSTORAGE_FIELDS,
constants.QR_JOB: JOB_FIELDS,
constants.QR_EXPORT: EXPORT_FIELDS,
constants.QR_NETWORK: NETWORK_FIELDS,
......
......@@ -451,6 +451,11 @@ _OS_CALLS = [
], None, _OsGetPostProc, "Returns an OS definition"),
]
_EXTSTORAGE_CALLS = [
("extstorage_diagnose", MULTI, None, constants.RPC_TMO_FAST, [], None, None,
"Request a diagnose of ExtStorage Providers"),
]
_NODE_CALLS = [
("node_has_ip_address", SINGLE, None, constants.RPC_TMO_FAST, [
("address", None, "IP address"),
......@@ -530,7 +535,7 @@ CALLS = {
"RpcClientDefault":
_Prepare(_IMPEXP_CALLS + _X509_CALLS + _OS_CALLS + _NODE_CALLS +
_FILE_STORAGE_CALLS + _MISC_CALLS + _INSTANCE_CALLS +
_BLOCKDEV_CALLS + _STORAGE_CALLS),
_BLOCKDEV_CALLS + _STORAGE_CALLS + _EXTSTORAGE_CALLS),
"RpcClientJobQueue": _Prepare([
("jobqueue_update", MULTI, None, constants.RPC_TMO_URGENT, [
("file_name", None, None),
......
......@@ -869,6 +869,15 @@ class NodeRequestHandler(http.server.HttpServerHandler):
required, name, checks, params = params
return backend.ValidateOS(required, name, checks, params)
# extstorage -----------------------
@staticmethod
def perspective_extstorage_diagnose(params):
"""Query detailed information about existing extstorage providers.
"""
return backend.DiagnoseExtStorage()
# hooks -----------------------
@staticmethod
......
......@@ -58,6 +58,7 @@ RAPI_OPCODE_EXCLUDE = compat.UniqueFrozenset([
opcodes.OpTagsSearch,
opcodes.OpClusterActivateMasterIp,
opcodes.OpClusterDeactivateMasterIp,
opcodes.OpExtStorageDiagnose,
# Difficult if not impossible
opcodes.OpClusterDestroy,
......
Markdown is supported
0%