Commit a6c43c02 authored by Helga Velroyen's avatar Helga Velroyen
Browse files

Verify client certificates



This patch adds a step to 'gnt-cluster verify' to verify
the existence and validity of the nodes' client
certificates. Since this is a crucial point of the
security concept, the verification is very detailed with
expressive error messages and well tested by unit tests.
Signed-off-by: default avatarHelga Velroyen <helgav@google.com>
Reviewed-by: default avatarHrvoje Ribicic <riba@google.com>
parent b3cc1646
......@@ -930,6 +930,24 @@ def _VerifyNodeInfo(what, vm_capable, result, all_hvparams):
result[constants.NV_HVINFO] = hyper.GetNodeInfo(hvparams=hvparams)
def _VerifyClientCertificate(cert_file=pathutils.NODED_CLIENT_CERT_FILE):
"""Verify the existance and validity of the client SSL certificate.
"""
create_cert_cmd = "gnt-cluster renew-crypto --new-node-certificates"
if not os.path.exists(cert_file):
return (constants.CV_ERROR,
"The client certificate does not exist. Run '%s' to create"
"client certificates for all nodes." % create_cert_cmd)
(errcode, msg) = utils.VerifyCertificate(cert_file)
if errcode is not None:
return (errcode, msg)
else:
# if everything is fine, we return the digest to be compared to the config
return (None, utils.GetCertificateDigest(cert_filename=cert_file))
def VerifyNode(what, cluster_name, all_hvparams, node_groups, groups_cfg):
"""Verify the status of the local node.
......@@ -983,6 +1001,9 @@ def VerifyNode(what, cluster_name, all_hvparams, node_groups, groups_cfg):
dict((vcluster.MakeVirtualPath(key), value)
for (key, value) in fingerprints.items())
if constants.NV_CLIENT_CERT in what:
result[constants.NV_CLIENT_CERT] = _VerifyClientCertificate()
if constants.NV_NODELIST in what:
(nodes, bynode) = what[constants.NV_NODELIST]
......
......@@ -21,8 +21,6 @@
"""Logical units dealing with the cluster."""
import OpenSSL
import copy
import itertools
import logging
......@@ -1484,8 +1482,8 @@ class _VerifyErrors(object):
"""
ETYPE_FIELD = "code"
ETYPE_ERROR = "ERROR"
ETYPE_WARNING = "WARNING"
ETYPE_ERROR = constants.CV_ERROR
ETYPE_WARNING = constants.CV_WARNING
def _Error(self, ecode, item, msg, *args, **kwargs):
"""Format an error message.
......@@ -1529,39 +1527,6 @@ class _VerifyErrors(object):
self._Error(*args, **kwargs)
def _VerifyCertificate(filename):
"""Verifies a certificate for L{LUClusterVerifyConfig}.
@type filename: string
@param filename: Path to PEM file
"""
try:
cert = OpenSSL.crypto.load_certificate(OpenSSL.crypto.FILETYPE_PEM,
utils.ReadFile(filename))
except Exception, err: # pylint: disable=W0703
return (LUClusterVerifyConfig.ETYPE_ERROR,
"Failed to load X509 certificate %s: %s" % (filename, err))
(errcode, msg) = \
utils.VerifyX509Certificate(cert, constants.SSL_CERT_EXPIRATION_WARN,
constants.SSL_CERT_EXPIRATION_ERROR)
if msg:
fnamemsg = "While verifying %s: %s" % (filename, msg)
else:
fnamemsg = None
if errcode is None:
return (None, fnamemsg)
elif errcode == utils.CERT_WARNING:
return (LUClusterVerifyConfig.ETYPE_WARNING, fnamemsg)
elif errcode == utils.CERT_ERROR:
return (LUClusterVerifyConfig.ETYPE_ERROR, fnamemsg)
raise errors.ProgrammerError("Unhandled certificate error code %r" % errcode)
def _GetAllHypervisorParameters(cluster, instances):
"""Compute the set of all hypervisor parameters.
......@@ -1642,7 +1607,7 @@ class LUClusterVerifyConfig(NoHooksLU, _VerifyErrors):
feedback_fn("* Verifying cluster certificate files")
for cert_filename in pathutils.ALL_CERT_FILES:
(errcode, msg) = _VerifyCertificate(cert_filename)
(errcode, msg) = utils.VerifyCertificate(cert_filename)
self._ErrorIf(errcode, constants.CV_ECLUSTERCERT, None, msg, code=errcode)
self._ErrorIf(not utils.CanRead(constants.LUXID_USER,
......@@ -2328,6 +2293,86 @@ class LUClusterVerifyGroup(LogicalUnit, _VerifyErrors):
" should node %s fail (%dMiB needed, %dMiB available)",
self.cfg.GetNodeName(prinode), needed_mem, n_img.mfree)
def _VerifyClientCertificates(self, nodes, all_nvinfo):
"""Verifies the consistency of the client certificates.
This includes several aspects:
- the individual validation of all nodes' certificates
- the consistency of the master candidate certificate map
- the consistency of the master candidate certificate map with the
certificates that the master candidates are actually using.
@param nodes: the list of nodes to consider in this verification
@param all_nvinfo: the map of results of the verify_node call to
all nodes
"""
candidate_certs = self.cfg.GetClusterInfo().candidate_certs
if candidate_certs is None or len(candidate_certs) == 0:
self._ErrorIf(
True, constants.CV_ECLUSTERCLIENTCERT, None,
"The cluster's list of master candidate certificates is empty."
"If you just updated the cluster, please run"
" 'gnt-cluster renew-crypto --new-node-certificates'.")
return
self._ErrorIf(
len(candidate_certs) != len(set(candidate_certs.values())),
constants.CV_ECLUSTERCLIENTCERT, None,
"There are at least two master candidates configured to use the same"
" certificate.")
# collect the client certificate
for node in nodes:
if node.offline:
continue
nresult = all_nvinfo[node.uuid]
if nresult.fail_msg or not nresult.payload:
continue
(errcode, msg) = nresult.payload.get(constants.NV_CLIENT_CERT, None)
self._ErrorIf(
errcode is not None, constants.CV_ECLUSTERCLIENTCERT, None,
"Client certificate of node '%s' failed validation: %s (code '%s')",
node.uuid, msg, errcode)
if not errcode:
digest = msg
if node.master_candidate:
if node.uuid in candidate_certs:
self._ErrorIf(
digest != candidate_certs[node.uuid],
constants.CV_ECLUSTERCLIENTCERT, None,
"Client certificate digest of master candidate '%s' does not"
" match its entry in the cluster's map of master candidate"
" certificates. Expected: %s Got: %s", node.uuid,
digest, candidate_certs[node.uuid])
else:
self._ErrorIf(
True, constants.CV_ECLUSTERCLIENTCERT, None,
"The master candidate '%s' does not have an entry in the"
" map of candidate certificates.", node.uuid)
self._ErrorIf(
digest in candidate_certs.values(),
constants.CV_ECLUSTERCLIENTCERT, None,
"Master candidate '%s' is using a certificate of another node.",
node.uuid)
else:
self._ErrorIf(
node.uuid in candidate_certs,
constants.CV_ECLUSTERCLIENTCERT, None,
"Node '%s' is not a master candidate, but still listed in the"
" map of master candidate certificates.", node.uuid)
self._ErrorIf(
(node.uuid not in candidate_certs) and
(digest in candidate_certs.values()),
constants.CV_ECLUSTERCLIENTCERT, None,
"Node '%s' is not a master candidate and is incorrectly using a"
" certificate of another node which is master candidate.",
node.uuid)
def _VerifyFiles(self, nodes, master_node_uuid, all_nvinfo,
(files_all, files_opt, files_mc, files_vm)):
"""Verifies file checksums collected from all nodes.
......@@ -2371,7 +2416,7 @@ class LUClusterVerifyGroup(LogicalUnit, _VerifyErrors):
if nresult.fail_msg or not nresult.payload:
node_files = None
else:
fingerprints = nresult.payload.get(constants.NV_FILELIST, None)
fingerprints = nresult.payload.get(constants.NV_FILELIST, {})
node_files = dict((vcluster.LocalizeVirtualPath(key), value)
for (key, value) in fingerprints.items())
del fingerprints
......@@ -3008,6 +3053,7 @@ class LUClusterVerifyGroup(LogicalUnit, _VerifyErrors):
constants.NV_OSLIST: None,
constants.NV_VMNODES: self.cfg.GetNonVmCapableNodeList(),
constants.NV_USERSCRIPTS: user_scripts,
constants.NV_CLIENT_CERT: None,
}
if vg_name is not None:
......@@ -3132,6 +3178,7 @@ class LUClusterVerifyGroup(LogicalUnit, _VerifyErrors):
feedback_fn("* Verifying configuration file consistency")
self._VerifyClientCertificates(self.my_node_info.values(), all_nvinfo)
# If not all nodes are being checked, we need to make sure the master node
# and a non-checked vm_capable node are in the list.
absent_node_uuids = set(self.all_node_info).difference(self.my_node_info)
......
......@@ -28,6 +28,8 @@ import os
from ganeti.utils import io
from ganeti.utils import x509
from ganeti import constants
from ganeti import errors
from ganeti import pathutils
......@@ -110,3 +112,36 @@ def GenerateNewSslCert(new_cert, cert_filename, log_msg):
logging.debug(log_msg)
x509.GenerateSelfSignedSslCert(cert_filename)
def VerifyCertificate(filename):
"""Verifies a SSL certificate.
@type filename: string
@param filename: Path to PEM file
"""
try:
cert = OpenSSL.crypto.load_certificate(OpenSSL.crypto.FILETYPE_PEM,
io.ReadFile(filename))
except Exception, err: # pylint: disable=W0703
return (constants.CV_ERROR,
"Failed to load X509 certificate %s: %s" % (filename, err))
(errcode, msg) = \
x509.VerifyX509Certificate(cert, constants.SSL_CERT_EXPIRATION_WARN,
constants.SSL_CERT_EXPIRATION_ERROR)
if msg:
fnamemsg = "While verifying %s: %s" % (filename, msg)
else:
fnamemsg = None
if errcode is None:
return (None, fnamemsg)
elif errcode == x509.CERT_WARNING:
return (constants.CV_WARNING, fnamemsg)
elif errcode == x509.CERT_ERROR:
return (constants.CV_ERROR, fnamemsg)
raise errors.ProgrammerError("Unhandled certificate error code %r" % errcode)
......@@ -2684,6 +2684,14 @@ cvTnode = "node"
cvTinstance :: String
cvTinstance = "instance"
-- * Cluster Verify error levels
cvWarning :: String
cvWarning = "WARNING"
cvError :: String
cvError = "ERROR"
-- * Cluster Verify error codes and documentation
cvEclustercert :: (String, String, String)
......@@ -2692,6 +2700,12 @@ cvEclustercert =
Types.cVErrorCodeToRaw CvECLUSTERCERT,
"Cluster certificate files verification failure")
cvEclusterclientcert :: (String, String, String)
cvEclusterclientcert =
("cluster",
Types.cVErrorCodeToRaw CvECLUSTERCLIENTCERT,
"Cluster client certificate files verification failure")
cvEclustercfg :: (String, String, String)
cvEclustercfg =
("cluster",
......@@ -2965,6 +2979,9 @@ cvAllEcodesStrings =
nvBridges :: String
nvBridges = "bridges"
nvClientCert :: String
nvClientCert = "client-cert"
nvDrbdhelper :: String
nvDrbdhelper = "drbd-helper"
......
......@@ -362,6 +362,7 @@ $(THH.makeJSONInstance ''VerifyOptionalChecks)
$(THH.declareLADT ''String "CVErrorCode"
[ ("CvECLUSTERCFG", "ECLUSTERCFG")
, ("CvECLUSTERCERT", "ECLUSTERCERT")
, ("CvECLUSTERCLIENTCERT", "ECLUSTERCLIENTCERT")
, ("CvECLUSTERFILECHECK", "ECLUSTERFILECHECK")
, ("CvECLUSTERDANGLINGNODES", "ECLUSTERDANGLINGNODES")
, ("CvECLUSTERDANGLINGINST", "ECLUSTERDANGLINGINST")
......
......@@ -25,13 +25,9 @@
import OpenSSL
import copy
import unittest
import operator
import os
import tempfile
import shutil
from collections import defaultdict
from ganeti.cmdlib import cluster
from ganeti import constants
......@@ -49,29 +45,6 @@ from testsupport import *
import testutils
class TestCertVerification(testutils.GanetiTestCase):
def setUp(self):
testutils.GanetiTestCase.setUp(self)
self.tmpdir = tempfile.mkdtemp()
def tearDown(self):
shutil.rmtree(self.tmpdir)
def testVerifyCertificate(self):
cluster._VerifyCertificate(testutils.TestDataFilename("cert1.pem"))
nonexist_filename = os.path.join(self.tmpdir, "does-not-exist")
(errcode, msg) = cluster._VerifyCertificate(nonexist_filename)
self.assertEqual(errcode, cluster.LUClusterVerifyConfig.ETYPE_ERROR)
# Try to load non-certificate file
invalid_cert = testutils.TestDataFilename("bdev-net.txt")
(errcode, msg) = cluster._VerifyCertificate(invalid_cert)
self.assertEqual(errcode, cluster.LUClusterVerifyConfig.ETYPE_ERROR)
class TestClusterVerifySsh(unittest.TestCase):
def testMultipleGroups(self):
fn = cluster.LUClusterVerifyGroup._SelectSshCheckNodes
......@@ -1013,7 +986,7 @@ class TestLUClusterVerifyConfig(CmdlibTestCase):
.patch_object(OpenSSL.crypto, "load_certificate")
self._load_cert_mock = self._load_cert_patcher.start()
self._verify_cert_patcher = testutils \
.patch_object(utils, "VerifyX509Certificate")
.patch_object(utils, "VerifyCertificate")
self._verify_cert_mock = self._verify_cert_patcher.start()
self._read_file_patcher = testutils.patch_object(utils, "ReadFile")
self._read_file_mock = self._read_file_patcher.start()
......@@ -1037,7 +1010,6 @@ class TestLUClusterVerifyConfig(CmdlibTestCase):
self.cfg.AddNewInstance()
op = opcodes.OpClusterVerifyConfig()
result = self.ExecOpCode(op)
self.assertTrue(result)
def testDanglingNode(self):
......@@ -1113,6 +1085,140 @@ class TestLUClusterVerifyGroup(CmdlibTestCase):
self.ExecOpCode(op)
class TestLUClusterVerifyClientCerts(CmdlibTestCase):
def _AddNormalNode(self):
self.normalnode = copy.deepcopy(self.master)
self.normalnode.master_candidate = False
self.normalnode.uuid = "normal-node-uuid"
self.cfg.AddNode(self.normalnode, None)
def testVerifyMasterCandidate(self):
client_cert = "client-cert-digest"
self.cluster.candidate_certs = {self.master.uuid: client_cert}
self.rpc.call_node_verify.return_value = \
RpcResultsBuilder() \
.AddSuccessfulNode(self.master,
{constants.NV_CLIENT_CERT: (None, client_cert)}) \
.Build()
op = opcodes.OpClusterVerifyGroup(group_name="default", verbose=True)
self.ExecOpCode(op)
def testVerifyMasterCandidateInvalid(self):
client_cert = "client-cert-digest"
self.cluster.candidate_certs = {self.master.uuid: client_cert}
self.rpc.call_node_verify.return_value = \
RpcResultsBuilder() \
.AddSuccessfulNode(self.master,
{constants.NV_CLIENT_CERT: (666, "Invalid Certificate")}) \
.Build()
op = opcodes.OpClusterVerifyGroup(group_name="default", verbose=True)
self.ExecOpCode(op)
self.mcpu.assertLogContainsRegex("Client certificate")
self.mcpu.assertLogContainsRegex("failed validation")
def testVerifyNoMasterCandidateMap(self):
client_cert = "client-cert-digest"
self.cluster.candidate_certs = {}
self.rpc.call_node_verify.return_value = \
RpcResultsBuilder() \
.AddSuccessfulNode(self.master,
{constants.NV_CLIENT_CERT: (None, client_cert)}) \
.Build()
op = opcodes.OpClusterVerifyGroup(group_name="default", verbose=True)
self.ExecOpCode(op)
self.mcpu.assertLogContainsRegex(
"list of master candidate certificates is empty")
def testVerifyNoSharingMasterCandidates(self):
client_cert = "client-cert-digest"
self.cluster.candidate_certs = {
self.master.uuid: client_cert,
"some-other-master-candidate-uuid": client_cert}
self.rpc.call_node_verify.return_value = \
RpcResultsBuilder() \
.AddSuccessfulNode(self.master,
{constants.NV_CLIENT_CERT: (None, client_cert)}) \
.Build()
op = opcodes.OpClusterVerifyGroup(group_name="default", verbose=True)
self.ExecOpCode(op)
self.mcpu.assertLogContainsRegex(
"two master candidates configured to use the same")
def testVerifyMasterCandidateCertMismatch(self):
client_cert = "client-cert-digest"
self.cluster.candidate_certs = {self.master.uuid: "different-cert-digest"}
self.rpc.call_node_verify.return_value = \
RpcResultsBuilder() \
.AddSuccessfulNode(self.master,
{constants.NV_CLIENT_CERT: (None, client_cert)}) \
.Build()
op = opcodes.OpClusterVerifyGroup(group_name="default", verbose=True)
self.ExecOpCode(op)
self.mcpu.assertLogContainsRegex("does not match its entry")
def testVerifyMasterCandidateUnregistered(self):
client_cert = "client-cert-digest"
self.cluster.candidate_certs = {"other-node-uuid": "different-cert-digest"}
self.rpc.call_node_verify.return_value = \
RpcResultsBuilder() \
.AddSuccessfulNode(self.master,
{constants.NV_CLIENT_CERT: (None, client_cert)}) \
.Build()
op = opcodes.OpClusterVerifyGroup(group_name="default", verbose=True)
self.ExecOpCode(op)
self.mcpu.assertLogContainsRegex("does not have an entry")
def testVerifyMasterCandidateOtherNodesCert(self):
client_cert = "client-cert-digest"
self.cluster.candidate_certs = {"other-node-uuid": client_cert}
self.rpc.call_node_verify.return_value = \
RpcResultsBuilder() \
.AddSuccessfulNode(self.master,
{constants.NV_CLIENT_CERT: (None, client_cert)}) \
.Build()
op = opcodes.OpClusterVerifyGroup(group_name="default", verbose=True)
self.ExecOpCode(op)
self.mcpu.assertLogContainsRegex("using a certificate of another node")
def testNormalNodeStillInList(self):
self._AddNormalNode()
client_cert_master = "client-cert-digest-master"
client_cert_normal = "client-cert-digest-normal"
self.cluster.candidate_certs = {
self.normalnode.uuid: client_cert_normal,
self.master.uuid: client_cert_master}
self.rpc.call_node_verify.return_value = \
RpcResultsBuilder() \
.AddSuccessfulNode(self.normalnode,
{constants.NV_CLIENT_CERT: (None, client_cert_normal)}) \
.AddSuccessfulNode(self.master,
{constants.NV_CLIENT_CERT: (None, client_cert_master)}) \
.Build()
op = opcodes.OpClusterVerifyGroup(group_name="default", verbose=True)
self.ExecOpCode(op)
self.mcpu.assertLogContainsRegex("not a master candidate")
self.mcpu.assertLogContainsRegex("still listed")
def testNormalNodeStealingMasterCandidateCert(self):
self._AddNormalNode()
client_cert_master = "client-cert-digest-master"
self.cluster.candidate_certs = {
self.master.uuid: client_cert_master}
self.rpc.call_node_verify.return_value = \
RpcResultsBuilder() \
.AddSuccessfulNode(self.normalnode,
{constants.NV_CLIENT_CERT: (None, client_cert_master)}) \
.AddSuccessfulNode(self.master,
{constants.NV_CLIENT_CERT: (None, client_cert_master)}) \
.Build()
op = opcodes.OpClusterVerifyGroup(group_name="default", verbose=True)
self.ExecOpCode(op)
self.mcpu.assertLogContainsRegex("not a master candidate")
self.mcpu.assertLogContainsRegex(
"certificate of another node which is master candidate")
class TestLUClusterVerifyGroupMethods(CmdlibTestCase):
"""Base class for testing individual methods in LUClusterVerifyGroup.
......
......@@ -177,6 +177,29 @@ class TestNodeVerify(testutils.GanetiTestCase):
get_hv_fn=self._GetHypervisor)
self._mock_hv.Verify.assert_called_with(hvparams=hvparams)
@testutils.patch_object(utils, "VerifyCertificate")
def testVerifyClientCertificateSuccess(self, verif_cert):
# mock the underlying x509 verification because the test cert is expired
verif_cert.return_value = (None, None)
cert_file = testutils.TestDataFilename("cert2.pem")
(errcode, digest) = backend._VerifyClientCertificate(cert_file=cert_file)
self.assertEqual(None, errcode)
self.assertTrue(isinstance(digest, str))
@testutils.patch_object(utils, "VerifyCertificate")
def testVerifyClientCertificateFailed(self, verif_cert):
expected_errcode = 666
verif_cert.return_value = (expected_errcode,
"The devil created this certificate.")
cert_file = testutils.TestDataFilename("cert2.pem")
(errcode, digest) = backend._VerifyClientCertificate(cert_file=cert_file)
self.assertEqual(expected_errcode, errcode)
def testVerifyClientCertificateNoCert(self):
cert_file = testutils.TestDataFilename("cert-that-does-not-exist.pem")
(errcode, digest) = backend._VerifyClientCertificate(cert_file=cert_file)
self.assertEqual(constants.CV_ERROR, errcode)
def _DefRestrictedCmdOwner():
return (os.getuid(), os.getgid())
......
......@@ -22,8 +22,12 @@
"""Script for unittesting the ganeti.utils.storage module"""
import mock
import os
import shutil
import tempfile
import unittest
from ganeti import constants
from ganeti.utils import security
import testutils
......@@ -88,5 +92,28 @@ class TestGetCertificateDigest(testutils.GanetiTestCase):
self.assertFalse(digest1 == digest2)
class TestCertVerification(testutils.GanetiTestCase):
def setUp(self):
testutils.GanetiTestCase.setUp(self)
self.tmpdir = tempfile.mkdtemp()
def tearDown(self):
shutil.rmtree(self.tmpdir)
def testVerifyCertificate(self):
security.VerifyCertificate(testutils.TestDataFilename("cert1.pem"))
nonexist_filename = os.path.join(self.tmpdir, "does-not-exist")
(errcode, msg) = security.VerifyCertificate(nonexist_filename)
self.assertEqual(errcode, constants.CV_ERROR)
# Try to load non-certificate file
invalid_cert = testutils.TestDataFilename("bdev-net.txt")
(errcode, msg) = security.VerifyCertificate(invalid_cert)
self.assertEqual(errcode, constants.CV_ERROR)
if __name__ == "__main__":
testutils.GanetiTestProgram()
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment