Commit c832f7e9 authored by Klaus Aehlig's avatar Klaus Aehlig

Merge branch 'stable-2.8' into master

* stable-2.8
  Change method dispatch in ClientOps to enforce luxi.REQ_ALL
  Allow modify_etc_hosts to be changed
  Add --modify-etc-hosts option for CLI tools
  Add luxiReqQueryNetworks to LuxiOp
  Log received message at debug level
  Set the correct group for confd
  Log RPC errors from inside executeRpcCall
  Factor out the logRpcErrors function
  Expose bulk parameter for GetJobs in RAPI client
  Add git send-email to the chroot
  Add tests for CanRead
  Only generate node lists with nodes having different names
  Cluster verify checks server.pem permissions
  Add function for checking file access permissions
  Prevent silent failure in case of connection problems
  Fix apt-get invocation in chroot_builder

* merged from stable-2.7
  Fix RAPI to include missing network fields
  Add support for querying network timestamps
  In the crontab example, look for the correct binary
  Fix wrong numbering in UPGRADE documentation

Conflicts:
	devel/build_chroot
	lib/client/gnt_cluster.py
Resolved by taking both additions.
Signed-off-by: default avatarKlaus Aehlig <aehlig@google.com>
Reviewed-by: default avatarGuido Trotter <ultrotter@google.com>
parents f3bde990 4eb06290
......@@ -69,8 +69,8 @@ groups on all nodes::
$ gnt-cluster redist-conf
#. If you use file storage, check that the ``/etc/ganeti/file-storage-paths``
#. is correct on all nodes. For security reasons it's not copied
#. automatically, but it can be copied manually via::
is correct on all nodes. For security reasons it's not copied
automatically, but it can be copied manually via::
$ gnt-cluster copyfile /etc/ganeti/file-storage-paths
......
......@@ -80,7 +80,7 @@ _daemon_usergroup() {
echo "@GNTMASTERUSER@:@GNTMASTERDGROUP@"
;;
confd)
echo "@GNTCONFDUSER@:@GNTCONFDGROUP@"
echo "@GNTCONFDUSER@:@GNTDAEMONSGROUP@"
;;
rapi)
echo "@GNTRAPIUSER@:@GNTRAPIGROUP@"
......@@ -253,11 +253,6 @@ start() {
$daemonexec $args "$@"
fi
# FIXME: This is a workaround for issue 477. Remove this once confd does not
# mess up the permissions anymore.
if [[ "$name" == ganeti-confd ]]; then
@PKGLIBDIR@/ensure-dirs;
fi
}
# Stops a daemon
......
......@@ -121,6 +121,7 @@ in_chroot -- \
in_chroot -- \
$APT_INSTALL -t squeeze-backports \
git \
git-email \
vim
in_chroot -- \
......
......@@ -4,7 +4,7 @@ PATH=/sbin:/bin:/usr/sbin:/usr/bin:/usr/local/sbin:/usr/local/bin
*/5 * * * * root [ -x @SBINDIR@/ganeti-watcher ] && @SBINDIR@/ganeti-watcher
# Clean job archive (at 01:45 AM)
45 1 * * * @GNTMASTERUSER@ [ -x @SBINDIR@/ganeti-master-cleaner ] && @SBINDIR@/ganeti-cleaner master
45 1 * * * @GNTMASTERUSER@ [ -x @SBINDIR@/ganeti-cleaner ] && @SBINDIR@/ganeti-cleaner master
# Clean job archive (at 02:45 AM)
45 2 * * * @GNTNODEDUSER@ [ -x @SBINDIR@/ganeti-cleaner ] && @SBINDIR@/ganeti-cleaner node
......@@ -116,6 +116,7 @@ __all__ = [
"MASTER_NETMASK_OPT",
"MC_OPT",
"MIGRATION_MODE_OPT",
"MODIFY_ETCHOSTS_OPT",
"NET_OPT",
"NETWORK_OPT",
"NETWORK6_OPT",
......@@ -1313,6 +1314,12 @@ NOMODIFY_ETCHOSTS_OPT = cli_option("--no-etc-hosts", dest="modify_etc_hosts",
help="Don't modify %s" % pathutils.ETC_HOSTS,
action="store_false", default=True)
MODIFY_ETCHOSTS_OPT = \
cli_option("--modify-etc-hosts", dest="modify_etc_hosts", metavar=_YORNO,
default=None, type="bool",
help="Defines whether the cluster should autonomously modify"
" and keep in sync the /etc/hosts file of the nodes")
NOMODIFY_SSH_SETUP_OPT = cli_option("--no-ssh-init", dest="modify_ssh_setup",
help="Don't initialize SSH keys",
action="store_false", default=True)
......
......@@ -995,7 +995,8 @@ def SetClusterParams(opts, args):
opts.ipolicy_std_specs is not None or
opts.ipolicy_disk_templates is not None or
opts.ipolicy_vcpu_ratio is not None or
opts.ipolicy_spindle_ratio is not None):
opts.ipolicy_spindle_ratio is not None or
opts.modify_etc_hosts is not None):
ToStderr("Please give at least one of the parameters.")
return 1
......@@ -1105,6 +1106,7 @@ def SetClusterParams(opts, args):
ipolicy=ipolicy,
candidate_pool_size=opts.candidate_pool_size,
maintain_node_health=mnh,
modify_etc_hosts=opts.modify_etc_hosts,
uid_pool=uid_pool,
add_uids=add_uids,
remove_uids=remove_uids,
......@@ -1623,7 +1625,8 @@ commands = {
RESERVED_LVS_OPT, DRY_RUN_OPT, PRIORITY_OPT, PREALLOC_WIPE_DISKS_OPT,
NODE_PARAMS_OPT, USE_EXTERNAL_MIP_SCRIPT, DISK_PARAMS_OPT, HV_STATE_OPT,
DISK_STATE_OPT] + SUBMIT_OPTS +
[ENABLED_DISK_TEMPLATES_OPT, IPOLICY_STD_SPECS_OPT] + INSTANCE_POLICY_OPTS,
[ENABLED_DISK_TEMPLATES_OPT, IPOLICY_STD_SPECS_OPT, MODIFY_ETCHOSTS_OPT] +
INSTANCE_POLICY_OPTS,
"[opts...]",
"Alters the parameters of the cluster"),
"renew-crypto": (
......
......@@ -1055,6 +1055,9 @@ class LUClusterSetParams(LogicalUnit):
" maintenance is not useful (still enabling it)")
self.cluster.maintain_node_health = self.op.maintain_node_health
if self.op.modify_etc_hosts is not None:
self.cluster.modify_etc_hosts = self.op.modify_etc_hosts
if self.op.prealloc_wipe_disks is not None:
self.cluster.prealloc_wipe_disks = self.op.prealloc_wipe_disks
......@@ -1355,6 +1358,13 @@ class LUClusterVerifyConfig(NoHooksLU, _VerifyErrors):
(errcode, msg) = _VerifyCertificate(cert_filename)
self._ErrorIf(errcode, constants.CV_ECLUSTERCERT, None, msg, code=errcode)
self._ErrorIf(not utils.CanRead(constants.CONFD_USER,
pathutils.NODED_CERT_FILE),
constants.CV_ECLUSTERCERT,
None,
pathutils.NODED_CERT_FILE + " must be accessible by the " +
constants.CONFD_USER + " user")
feedback_fn("* Verifying hypervisor parameters")
self._VerifyHVP(_GetAllHypervisorParameters(self.cfg.GetClusterInfo(),
......
......@@ -2800,6 +2800,7 @@ class ConfigWriter:
self._EnsureUUID(net, ec_id)
net.serial_no = 1
net.ctime = net.mtime = time.time()
self._config_data.networks[net.uuid] = net
self._config_data.cluster.serial_no += 1
......
......@@ -86,6 +86,7 @@ REQ_ALL = compat.UniqueFrozenset([
REQ_QUERY_INSTANCES,
REQ_QUERY_JOBS,
REQ_QUERY_NODES,
REQ_QUERY_NETWORKS,
REQ_QUERY_TAGS,
REQ_SET_DRAIN_FLAG,
REQ_SET_WATCHER_PAUSE,
......
......@@ -961,6 +961,8 @@ class OpClusterSetParams(OpCode):
ht.TMaybe(ht.TAnd(ht.TListOf(ht.TElemOf(constants.DISK_TEMPLATES)),
ht.TTrue)),
"List of enabled disk templates"),
("modify_etc_hosts", None, ht.TMaybeBool,
"Whether the cluster can modify and keep in sync the /etc/hosts files"),
]
OP_RESULT = ht.TNone
......
......@@ -2795,6 +2795,9 @@ def _BuildNetworkFields():
compat.partial(_GetNetworkStatsField, name, kind))
for (name, (title, kind, _, doc)) in _NETWORK_STATS_FIELDS.items()])
# Add timestamps
fields.extend(_GetItemTimestampFields(IQ_NETWORKS))
return _PrepareFieldList(fields, [])
#: Fields for cluster information
......
......@@ -1294,17 +1294,28 @@ class GanetiRapiClient(object): # pylint: disable=R0904
("/%s/instances/%s/console" %
(GANETI_RAPI_VERSION, instance)), None, None)
def GetJobs(self):
def GetJobs(self, bulk=False):
"""Gets all jobs for the cluster.
@type bulk: bool
@param bulk: Whether to return detailed information about jobs.
@rtype: list of int
@return: job ids for the cluster
@return: List of job ids for the cluster or list of dicts with detailed
information about the jobs if bulk parameter was true.
"""
return [int(j["id"])
for j in self._SendRequest(HTTP_GET,
"/%s/jobs" % GANETI_RAPI_VERSION,
None, None)]
query = []
_AppendIf(query, bulk, ("bulk", 1))
if bulk:
return self._SendRequest(HTTP_GET,
"/%s/jobs" % GANETI_RAPI_VERSION,
query, None)
else:
return [int(j["id"])
for j in self._SendRequest(HTTP_GET,
"/%s/jobs" % GANETI_RAPI_VERSION,
None, None)]
def GetJobStatus(self, job_id):
"""Gets the status of a job.
......
......@@ -72,7 +72,7 @@ I_FIELDS = ["name", "admin_state", "os",
"pnode", "snodes",
"disk_template",
"nic.ips", "nic.macs", "nic.modes",
"nic.links", "nic.networks", "nic.bridges",
"nic.links", "nic.networks", "nic.networks.names", "nic.bridges",
"network_port",
"disk.sizes", "disk.spindles", "disk_usage",
"beparams", "hvparams",
......@@ -97,8 +97,8 @@ NET_FIELDS = ["name", "network", "gateway",
"mac_prefix",
"free_count", "reserved_count",
"map", "group_list", "inst_list",
"external_reservations", "tags",
]
"external_reservations",
] + _COMMON_FIELDS
G_FIELDS = [
"alloc_policy",
......
......@@ -279,6 +279,10 @@ class ClientOps:
logging.info("Received invalid arguments of type '%s'", type(args))
raise ValueError("Invalid arguments type '%s'" % type(args))
if method not in luxi.REQ_ALL:
logging.info("Received invalid request '%s'", method)
raise ValueError("Invalid operation '%s'" % method)
# TODO: Rewrite to not exit in each 'if/elif' branch
if method == luxi.REQ_SUBMIT_JOB:
......@@ -446,8 +450,9 @@ class ClientOps:
return _SetWatcherPause(context, until)
else:
logging.info("Received invalid request '%s'", method)
raise ValueError("Invalid operation '%s'" % method)
logging.critical("Request '%s' in luxi.REQ_ALL, but not known", method)
raise errors.ProgrammerError("Operation '%s' in luxi.REQ_ALL,"
" but not implemented" % method)
def _Query(self, op):
"""Runs the specified opcode and returns the result.
......
......@@ -29,6 +29,8 @@ import tempfile
import errno
import time
import stat
import grp
import pwd
from ganeti import errors
from ganeti import constants
......@@ -1063,3 +1065,38 @@ class TemporaryFileManager(object):
"""
while self._files:
RemoveFile(self._files.pop())
def IsUserInGroup(uid, gid):
"""Returns True if the user belongs to the group.
@type uid: int
@param uid: the user id
@type gid: int
@param gid: the group id
@rtype: bool
"""
user = pwd.getpwuid(uid)
group = grp.getgrgid(gid)
return user.pw_gid == gid or user.pw_name in group.gr_mem
def CanRead(username, filename):
"""Returns True if the user can access (read) the file.
@type username: string
@param username: the name of the user
@type filename: string
@param filename: the name of the file
@rtype: bool
"""
filestats = os.stat(filename)
user = pwd.getpwnam(username)
uid = user.pw_uid
user_readable = filestats.st_mode & stat.S_IRUSR != 0
group_readable = filestats.st_mode & stat.S_IRGRP != 0
return ((filestats.st_uid == uid and user_readable)
or (filestats.st_uid != uid and
IsUserInGroup(uid, filestats.st_gid) and group_readable))
......@@ -119,6 +119,11 @@ $(genLuxiOp "LuxiOp"
, simpleField "fields" [t| [String] |]
, simpleField "lock" [t| Bool |]
])
, (luxiReqQueryNetworks,
[ simpleField "names" [t| [String] |]
, simpleField "fields" [t| [String] |]
, simpleField "lock" [t| Bool |]
])
, (luxiReqQueryInstances,
[ simpleField "names" [t| [String] |]
, simpleField "fields" [t| [String] |]
......@@ -341,6 +346,9 @@ decodeCall (LuxiCall call args) =
return $ QueryGroups names fields locking
ReqQueryClusterInfo ->
return QueryClusterInfo
ReqQueryNetworks -> do
(names, fields, locking) <- fromJVal args
return $ QueryNetworks names fields locking
ReqQuery -> do
(what, fields, qfilter) <- fromJVal args
return $ Query what fields qfilter
......
......@@ -260,6 +260,7 @@ $(buildObject "Network" "network" $
simpleField "ext_reservations" [t| String |]
]
++ uuidFields
++ timeStampFields
++ serialFields
++ tagsFields)
......@@ -272,6 +273,10 @@ instance TagsObject Network where
instance UuidObject Network where
uuidOf = networkUuid
instance TimeStampObject Network where
cTimeOf = networkCtime
mTimeOf = networkMtime
-- * NIC definitions
$(buildParam "Nic" "nicp"
......
......@@ -184,6 +184,7 @@ $(genOpCode "OpCode"
, pBlacklistedOs
, pUseExternalMipScript
, pEnabledDiskTemplates
, pModifyEtcHosts
])
, ("OpClusterRedistConf", [])
, ("OpClusterActivateMasterIp", [])
......
......@@ -121,6 +121,7 @@ module Ganeti.OpParams
, pAddUids
, pRemoveUids
, pMaintainNodeHealth
, pModifyEtcHosts
, pPreallocWipeDisks
, pNicParams
, pInstNics
......@@ -900,6 +901,10 @@ pRemoveUids = optionalField $ simpleField "remove_uids" [t| [[(Int, Int)]] |]
pMaintainNodeHealth :: Field
pMaintainNodeHealth = optionalField $ booleanField "maintain_node_health"
-- | Whether to modify and keep in sync the @/etc/hosts@ files of nodes.
pModifyEtcHosts :: Field
pModifyEtcHosts = optionalField $ booleanField "modify_etc_hosts"
-- | Whether to wipe disks before allocating them to instances.
pPreallocWipeDisks :: Field
pPreallocWipeDisks = optionalField $ booleanField "prealloc_wipe_disks"
......
......@@ -179,6 +179,10 @@ handleCall cfg (QueryJobs names fields) =
handleClassicQuery cfg (Qlang.ItemTypeLuxi Qlang.QRJob)
(map (Right . fromIntegral . fromJobId) names) fields False
handleCall cfg (QueryNetworks names fields lock) =
handleClassicQuery cfg (Qlang.ItemTypeOpCode Qlang.QRNetwork)
(map Left names) fields lock
handleCall _ op =
return . Bad $
GenericError ("Luxi call '" ++ strOfOp op ++ "' not implemented")
......@@ -209,6 +213,7 @@ handleClientMsg client creader args = do
handleClient :: Client -> ConfigReader -> IO Bool
handleClient client creader = do
!msg <- recvMsgExt client
logDebug $ "Received message: " ++ show msg
case msg of
RecvConnClosed -> logDebug "Connection closed" >> return False
RecvError err -> logWarning ("Error during message receiving: " ++ err) >>
......
......@@ -33,6 +33,7 @@ module Ganeti.Rpc
, ERpcError
, explainRpcError
, executeRpcCall
, logRpcErrors
, rpcCallName
, rpcCallTimeout
......@@ -83,6 +84,7 @@ import qualified Ganeti.Path as P
import Ganeti.BasicTypes
import qualified Ganeti.Constants as C
import Ganeti.Logging
import Ganeti.Objects
import Ganeti.THH
import Ganeti.Types
......@@ -194,6 +196,15 @@ parseHttpResponse call res =
J.JSString msg -> Left $ RpcResultError (J.fromJSString msg)
_ -> Left . JsonDecodeError $ show (pp_value jerr)
-- | Scan the list of results produced by executeRpcCall and log all the RPC
-- errors.
logRpcErrors :: [(a, ERpcError b)] -> IO ()
logRpcErrors allElems =
let logOneRpcErr (_, Right _) = return ()
logOneRpcErr (_, Left err) =
logError $ "Error in the RPC HTTP reply: " ++ show err
in mapM_ logOneRpcErr allElems
-- | Execute RPC call for many nodes in parallel.
executeRpcCall :: (Rpc a b) => [Node] -> a -> IO [(Node, ERpcError b)]
executeRpcCall nodes call = do
......@@ -220,7 +231,9 @@ executeRpcCall nodes call = do
Ok r -> return r
-- now parse the replies
let results'' = map (parseHttpReply call) results'
return $ zip nodes results''
pairedList = zip nodes results''
logRpcErrors pairedList
return pairedList
-- | Helper function that is used to read dictionaries of values.
sanitizeDictResults :: [(String, J.Result a)] -> ERpcError [(String, a)]
......
......@@ -134,7 +134,9 @@ instance Arbitrary Node.Node where
-- lists here.
genNodeList :: Gen Node.Node -> Gen Node.List
genNodeList ngen = fmap (snd . Loader.assignIndices) names_nodes
where names_nodes = (fmap . map) (\n -> (Node.name n, n)) $ listOf1 ngen
where names_nodes = (fmap . map) (\n -> (Node.name n, n)) nodes
nodes = listOf1 ngen `suchThat`
((\ns -> ns == nub ns) . map Node.name)
-- | Node list generator where node names are unique
genUniqueNodeList :: Gen Node.Node -> Gen (Node.List, Types.NameAssoc)
......
......@@ -64,6 +64,8 @@ instance Arbitrary Luxi.LuxiOp where
genFields <*> arbitrary
Luxi.ReqQueryGroups -> Luxi.QueryGroups <$> arbitrary <*>
arbitrary <*> arbitrary
Luxi.ReqQueryNetworks -> Luxi.QueryNetworks <$> arbitrary <*>
arbitrary <*> arbitrary
Luxi.ReqQueryInstances -> Luxi.QueryInstances <$> listOf genFQDN <*>
genFields <*> arbitrary
Luxi.ReqQueryJobs -> Luxi.QueryJobs <$> arbitrary <*> genFields
......
......@@ -249,8 +249,10 @@ genValidNetwork = do
res <- liftM Just (genBitString $ netmask2NumHosts netmask)
ext_res <- liftM Just (genBitString $ netmask2NumHosts netmask)
uuid <- arbitrary
ctime <- arbitrary
mtime <- arbitrary
let n = Network name mac_prefix (Ip4Network net netmask) net6 gateway
gateway6 res ext_res uuid 0 Set.empty
gateway6 res ext_res uuid ctime mtime 0 Set.empty
return n
-- | Generate an arbitrary string consisting of '0' and '1' of the given length.
......
......@@ -163,7 +163,7 @@ instance Arbitrary OpCodes.OpCode where
arbitrary <*> arbitrary <*> arbitrary <*>
emptyMUD <*> emptyMUD <*> arbitrary <*>
arbitrary <*> arbitrary <*> arbitrary <*> arbitrary <*>
arbitrary <*> arbitrary <*> arbitrary <*> arbitrary
arbitrary <*> arbitrary <*> arbitrary <*> arbitrary <*> arbitrary
"OP_CLUSTER_REDIST_CONF" -> pure OpCodes.OpClusterRedistConf
"OP_CLUSTER_ACTIVATE_MASTER_IP" ->
pure OpCodes.OpClusterActivateMasterIp
......
......@@ -837,6 +837,14 @@ class GanetiRapiClientTests(testutils.GanetiTestCase):
self.assertEqual([123, 124], self.client.GetJobs())
self.assertHandler(rlib2.R_2_jobs)
self.rapi.AddResponse('[ { "id": "123", "uri": "\\/2\\/jobs\\/123" },'
' { "id": "124", "uri": "\\/2\\/jobs\\/124" } ]')
self.assertEqual([{"id": "123", "uri": "/2/jobs/123"},
{"id": "124", "uri": "/2/jobs/124"}],
self.client.GetJobs(bulk=True))
self.assertHandler(rlib2.R_2_jobs)
self.assertBulk()
def testGetJobStatus(self):
self.rapi.AddResponse("{\"foo\": \"bar\"}")
self.assertEqual({"foo": "bar"}, self.client.GetJobStatus(1234))
......
......@@ -25,6 +25,9 @@ import os
import tempfile
import shutil
import errno
import grp
import pwd
import stat
from ganeti import constants
from ganeti import utils
......@@ -92,6 +95,43 @@ class TestWriteFile(testutils.GanetiTestCase):
keep_perms=utils.KP_IF_EXISTS)
self.assertFileGid(target, tgid)
class TestCanRead(testutils.GanetiTestCase):
def setUp(self):
testutils.GanetiTestCase.setUp(self)
self.tmpdir = tempfile.mkdtemp()
self.confdUid = pwd.getpwnam(constants.CONFD_USER).pw_uid
self.masterdUid = pwd.getpwnam(constants.MASTERD_USER).pw_uid
self.masterdGid = grp.getgrnam(constants.MASTERD_GROUP).gr_gid
def tearDown(self):
testutils.GanetiTestCase.tearDown(self)
if self.tmpdir:
shutil.rmtree(self.tmpdir)
def testUserCanRead(self):
target = utils.PathJoin(self.tmpdir, "target1")
f=open(target, "w")
f.close()
utils.EnforcePermission(target, 0400, uid=self.confdUid,
gid=self.masterdGid)
self.assertTrue(utils.CanRead(constants.CONFD_USER, target))
if constants.CONFD_USER != constants.MASTERD_USER:
self.assertFalse(utils.CanRead(constants.MASTERD_USER, target))
def testGroupCanRead(self):
target = utils.PathJoin(self.tmpdir, "target2")
f=open(target, "w")
f.close()
utils.EnforcePermission(target, 0040, uid=self.confdUid,
gid=self.masterdGid)
self.assertFalse(utils.CanRead(constants.CONFD_USER, target))
if constants.CONFD_USER != constants.MASTERD_USER:
self.assertTrue(utils.CanRead(constants.MASTERD_USER, target))
utils.EnforcePermission(target, 0040, uid=self.masterdUid+1,
gid=self.masterdGid)
self.assertTrue(utils.CanRead(constants.MASTERD_USER, target))
if __name__ == "__main__":
testutils.GanetiTestProgram()
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment