Commit 560ef132 authored by Santi Raffa's avatar Santi Raffa Committed by Jose A. Lopes
Browse files

serializer: emit and encode Private values

For inbound data the simplest, safest thing to do is to traverse all
JSON right after encoding and search for private parameters by key.

This ensures that all consumers of this data get Private values
transparently and consistently; the serializing methods don't have to
worry about trying to understand what kind of JSON it is receiving.
It also minimizes the surface area of codebase that is exposed to the
values directly.

The downside is it adds a ~5% time overhead to all JSON decoding as
measured through:

$ time make hs-test-py_compat_types

As far as encoding to JSON is concerned, the serializer will redact
Private values unless told to do so explicitly (e.g., for tests).
Signed-off-by: default avatarSanti Raffa <>
Reviewed-by: default avatarJose A. Lopes <>
parent 4884f187
......@@ -120,7 +120,8 @@ def FormatRequest(method, args, version=None):
request[KEY_VERSION] = version
# Serialize the request
return serializer.DumpJson(request)
return serializer.DumpJson(request,
def CallRPCMethod(transport_cb, method, args, version=None):
......@@ -40,19 +40,25 @@ import simplejson
from ganeti import errors
from ganeti import utils
from ganeti import constants
_RE_EOLSP = re.compile("[ \t]+$", re.MULTILINE)
def DumpJson(data):
def DumpJson(data, private_encoder=None):
"""Serialize a given object.
@param data: the data to serialize
@return: the string representation of data
@param private_encoder: specify L{serializer.EncodeWithPrivateFields} if you
require the produced JSON to also contain private
parameters. Otherwise, they will encode to null.
encoded = simplejson.dumps(data)
if private_encoder is None:
# Do not leak private fields by default.
private_encoder = EncodeWithoutPrivateFields
encoded = simplejson.dumps(data, default=private_encoder)
txt = _RE_EOLSP.sub("", encoded)
if not txt.endswith("\n"):
......@@ -69,20 +75,64 @@ def LoadJson(txt):
@raise JSONDecodeError: if L{txt} is not a valid JSON document
return simplejson.loads(txt)
values = simplejson.loads(txt)
# Hunt and seek for Private fields and wrap them.
return values
def WrapPrivateValues(json):
"""Crawl a JSON decoded structure for private values and wrap them.
@param json: the json-decoded value to protect.
# This function used to be recursive. I use this list to avoid actual
# recursion, however, since this is a very high-traffic area.
todo = [json]
while todo:
data = todo.pop()
if isinstance(data, list): # Array
for item in data:
elif isinstance(data, dict): # Object
# This is kind of a kludge, but the only place where we know what should
# be protected is in ganeti.opcodes, and not in a way that is helpful to
# us, especially in such a high traffic method; on the other hand, the
# Haskell `py_compat_fields` test should complain whenever this check
# does not protect fields properly.
for field in data:
value = data[field]
if not field.endswith("_cluster"):
data[field] = PrivateDict(value)
for os in data[field]:
value[os] = PrivateDict(value[os])
else: # Values
def DumpSignedJson(data, key, salt=None, key_selector=None):
def DumpSignedJson(data, key, salt=None, key_selector=None,
"""Serialize a given object and authenticate it.
@param data: the data to serialize
@param key: shared hmac key
@param key_selector: name/id that identifies the key (in case there are
multiple keys in use, e.g. in a multi-cluster environment)
@param private_encoder: see L{DumpJson}
@return: the string representation of data signed by the hmac key
txt = DumpJson(data)
txt = DumpJson(data, private_encoder=private_encoder)
if salt is None:
salt = ""
signed_dict = {
......@@ -113,6 +163,9 @@ def LoadSignedJson(txt, key):
signed_dict = LoadJson(txt)
if not isinstance(signed_dict, dict):
raise errors.SignatureError("Invalid external message")
......@@ -319,3 +372,15 @@ class PrivateDict(dict):
for key in self:
returndict[key] = self[key].Get()
return returndict
def EncodeWithoutPrivateFields(obj):
if isinstance(obj, Private):
return None
raise TypeError(repr(obj) + " is not JSON serializable")
def EncodeWithPrivateFields(obj):
if isinstance(obj, Private):
return obj.Get()
raise TypeError(repr(obj) + " is not JSON serializable")
......@@ -4773,3 +4773,23 @@ instanceCommunicationNetwork = "ganeti:network:communication"
instanceCommunicationNicPrefix :: String
instanceCommunicationNicPrefix = "ganeti:communication:"
-- | Parameters that should be protected
-- Python does not have a type system and can't automatically infer what should
-- be the resulting type of a JSON request. As a result, it must rely on this
-- list of parameter names to protect values correctly.
-- Names ending in _cluster will be treated as dicts of dicts of private values.
-- Otherwise they are considered dicts of private values.
privateParametersBlacklist :: [String]
privateParametersBlacklist = [ "osparams_private"
, "osparams_secret"
, "osparams_private_cluster"
-- | Warn the user that the logging level is too low for production use.
debugModeConfidentialityWarning :: String
debugModeConfidentialityWarning =
"ALERT: %s started in debug mode.\n\
\ Private and secret parameters WILL be logged!\n"
......@@ -519,7 +519,10 @@ case_py_compat_types = do
\ op.Validate(True)\n\
\encoded = [(op.Summary(), op.__getstate__())\n\
\ for op in decoded]\n\
\print serializer.Dump(encoded)" serialized
\print serializer.Dump(\
\ encoded,\
\ private_encoder=serializer.EncodeWithPrivateFields)"
>>= checkPythonResult
let deserialised =
J.decode py_stdout::J.Result [(String, OpCodes.MetaOpCode)]
......@@ -229,7 +229,6 @@ class TestLUNodeAdd(CmdlibTestCase):
self.ExecOpCodeExpectOpPrereqError(op, "Readded node doesn't have the same"
" IP address configuration as before")
def testNodeHasSecondaryIpButNotMaster(self):
self.master.secondary_ip = self.master.primary_ip
......@@ -22,11 +22,13 @@
"""Script for unittesting the serializer module"""
import doctest
import unittest
from ganeti import serializer
from ganeti import errors
from ganeti import ht
from ganeti import objects
from ganeti import serializer
import testutils
......@@ -39,23 +41,28 @@ class TestSerializer(testutils.GanetiTestCase):
[1, 2, 3],
(1, 2, 3),
{ "1": 2, "foo": "bar", },
{"1": 2,
"foo": "bar"},
["abc", 1, 2, 3, 999,
"a1": ("Hello", "World"),
"a2": "This is only a test",
"a3": None,
"foo": "bar",
"osparams:": serializer.PrivateDict({
"foo": 5,
def _TestSerializer(self, dump_fn, load_fn):
_dump_fn = lambda data: dump_fn(
for data in self._TESTDATA:
self.assertEqualValues(load_fn(dump_fn(data)), data)
self.assertEqualValues(load_fn(_dump_fn(data)), data)
def testGeneric(self):
self._TestSerializer(serializer.Dump, serializer.Load)
......@@ -70,30 +77,35 @@ class TestSerializer(testutils.GanetiTestCase):
self._TestSigned(serializer.DumpSignedJson, serializer.LoadSignedJson)
def _TestSigned(self, dump_fn, load_fn):
_dump_fn = lambda *args, **kwargs: dump_fn(
for data in self._TESTDATA:
self.assertEqualValues(load_fn(dump_fn(data, "mykey"), "mykey"),
self.assertEqualValues(load_fn(_dump_fn(data, "mykey"), "mykey"),
(data, ""))
self.assertEqualValues(load_fn(dump_fn(data, "myprivatekey",
self.assertEqualValues(load_fn(_dump_fn(data, "myprivatekey",
(data, "mysalt"))
keydict = {
"mykey_id": "myprivatekey",
self.assertEqualValues(load_fn(dump_fn(data, "myprivatekey",
self.assertEqualValues(load_fn(_dump_fn(data, "myprivatekey",
(data, "mysalt"))
self.assertRaises(errors.SignatureError, load_fn,
dump_fn(data, "myprivatekey",
_dump_fn(data, "myprivatekey",
self.assertRaises(errors.SignatureError, load_fn,
dump_fn("test", "myprivatekey"),
_dump_fn("test", "myprivatekey"),
self.assertRaises(errors.SignatureError, load_fn,
......@@ -131,5 +143,83 @@ class TestLoadAndVerifyJson(unittest.TestCase):
self.assertEqual(serializer.LoadAndVerifyJson("\"Foo\"", ht.TAny), "Foo")
class TestPrivate(unittest.TestCase):
def testEquality(self):
pDict = serializer.PrivateDict()
pDict["bar"] = "egg"
nDict = {"bar": "egg"}
self.assertEqual(pDict, nDict, "PrivateDict-dict equality failure")
def testPrivateDictUnprivate(self):
pDict = serializer.PrivateDict()
pDict["bar"] = "egg"
uDict = pDict.Unprivate()
nDict = {"bar": "egg"}
self.assertEquals(type(uDict), dict,
"PrivateDict.Unprivate() did not return a dict")
self.assertEqual(pDict, uDict, "PrivateDict.Unprivate() equality failure")
self.assertEqual(nDict, uDict, "PrivateDict.Unprivate() failed to return")
def testAttributeTransparency(self):
class Dummy(object):
dummy = Dummy() = "egg"
pDummy = serializer.Private(dummy)
self.assertEqual(, "egg", "Failed to access attribute of Private")
def testCallTransparency(self):
foo = serializer.Private("egg")
self.assertEqual(foo.upper(), "EGG", "Failed to call Private instance")
def testFillDict(self):
pDict = serializer.PrivateDict()
pDict["bar"] = "egg"
self.assertEqual(pDict, objects.FillDict({}, pDict))
def testLeak(self):
pDict = serializer.PrivateDict()
pDict["bar"] = "egg"
self.assertNotIn("egg", str(pDict), "Value leaked in str(PrivateDict)")
self.assertNotIn("egg", repr(pDict), "Value leaked in repr(PrivateDict)")
self.assertNotIn("egg", "{0}".format(pDict),
"Value leaked in PrivateDict.__format__")
self.assertNotIn("egg", serializer.Dump(pDict),
"Value leaked in serializer.Dump(PrivateDict)")
def testProperAccess(self):
pDict = serializer.PrivateDict()
pDict["bar"] = "egg"
self.assertIs("egg", pDict["bar"].Get(),
"Value not returned by Private.Get()")
self.assertIs("egg", pDict.GetPrivate("bar"),
"Value not returned by Private.GetPrivate()")
self.assertIs("egg", pDict.Unprivate()["bar"],
"Value not returned by PrivateDict.Unprivate()")
def testDictGet(self):
self.assertIs("tar", serializer.PrivateDict().GetPrivate("bar", "tar"),
"Private.GetPrivate() did not handle the default case")
def testZeronessPrivate(self):
"Private of non-empty string is false")
self.assertFalse(serializer.Private(""), "Private empty string is true")
class TestCheckDoctests(unittest.TestCase):
def testCheckSerializer(self):
results = doctest.testmod(serializer)
self.assertEquals(results.failed, 0, "Doctest failures detected")
if __name__ == "__main__":
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment