Commit a182a3ed authored by Michael Hanselmann's avatar Michael Hanselmann
Browse files

serializer: Remove JSON indentation and dict key sorting



Serializing to JSON using “simplejson” is significantly slower when
indentation and/or sorting of dictionary keys is used. In simplejson 1.x
the difference isn't that big, but with simplejson 2.x the difference
can be up to a factor of 7.5. The reason is that the latter no longer
uses C functions when sorting or indentation is used.

With this patch we revert everything to simplejson's defaults, which
should provide us with the best performance available.
Signed-off-by: default avatarMichael Hanselmann <hansmi@google.com>
Reviewed-by: default avatarIustin Pop <iustin@google.com>
parent 0f63f1f9
......@@ -200,7 +200,7 @@ class StatusFile:
self._data.mtime = time.time()
utils.WriteFile(self._path,
data=serializer.DumpJson(self._data.ToDict(), indent=True),
data=serializer.DumpJson(self._data.ToDict()),
mode=0400)
......
......@@ -185,9 +185,8 @@ class QmpMessage:
return QmpMessage(data)
def __str__(self):
# The protocol expects the JSON object to be sent as a single
# line, hence the need for indent=False.
return serializer.DumpJson(self.data, indent=False)
# The protocol expects the JSON object to be sent as a single line.
return serializer.DumpJson(self.data)
def __eq__(self, other):
# When comparing two QmpMessages, we are interested in comparing
......
......@@ -2248,7 +2248,7 @@ class JobQueue(object):
assert job.writable, "Can't update read-only job"
filename = self._GetJobPath(job.id)
data = serializer.DumpJson(job.Serialize(), indent=False)
data = serializer.DumpJson(job.Serialize())
logging.debug("Writing job %s to %s", job.id, filename)
self._UpdateJobQueueFile(filename, data, replicate)
......
......@@ -343,7 +343,7 @@ def FormatRequest(method, args, version=None):
request[KEY_VERSION] = version
# Serialize the request
return serializer.DumpJson(request, indent=False)
return serializer.DumpJson(request)
def CallLuxiMethod(transport_cb, method, args, version=None):
......
......@@ -442,8 +442,7 @@ class _RpcClientBase:
read_timeout = timeout
body = serializer.DumpJson(map(self._encoder,
zip(map(compat.snd, argdefs), args)),
indent=False)
zip(map(compat.snd, argdefs), args)))
result = self._proc(node_list, procedure, body, read_timeout=read_timeout)
......
......@@ -42,49 +42,19 @@ from ganeti import errors
from ganeti import utils
_JSON_INDENT = 2
_RE_EOLSP = re.compile("[ \t]+$", re.MULTILINE)
def _GetJsonDumpers(_encoder_class=simplejson.JSONEncoder):
"""Returns two JSON functions to serialize data.
@rtype: (callable, callable)
@return: The function to generate a compact form of JSON and another one to
generate a more readable, indented form of JSON (if supported)
"""
plain_encoder = _encoder_class(sort_keys=True)
# Check whether the simplejson module supports indentation
try:
indent_encoder = _encoder_class(indent=_JSON_INDENT, sort_keys=True)
except TypeError:
# Indentation not supported
indent_encoder = plain_encoder
return (plain_encoder.encode, indent_encoder.encode)
(_DumpJson, _DumpJsonIndent) = _GetJsonDumpers()
def DumpJson(data, indent=True):
def DumpJson(data):
"""Serialize a given object.
@param data: the data to serialize
@param indent: whether to indent output (depends on simplejson version)
@return: the string representation of data
"""
if indent:
fn = _DumpJsonIndent
else:
fn = _DumpJson
encoded = simplejson.dumps(data)
txt = _RE_EOLSP.sub("", fn(data))
txt = _RE_EOLSP.sub("", encoded)
if not txt.endswith("\n"):
txt += "\n"
......@@ -112,7 +82,7 @@ def DumpSignedJson(data, key, salt=None, key_selector=None):
@return: the string representation of data signed by the hmac key
"""
txt = DumpJson(data, indent=False)
txt = DumpJson(data)
if salt is None:
salt = ""
signed_dict = {
......@@ -127,7 +97,7 @@ def DumpSignedJson(data, key, salt=None, key_selector=None):
signed_dict["hmac"] = utils.Sha1Hmac(key, txt, salt=salt + key_selector)
return DumpJson(signed_dict, indent=False)
return DumpJson(signed_dict)
def LoadSignedJson(txt, key):
......
......@@ -169,7 +169,7 @@ class NodeHttpServer(http.server.HttpServer):
logging.exception("Error in RPC call")
result = (False, "Error while executing backend function: %s" % str(err))
return serializer.DumpJson(result, indent=False)
return serializer.DumpJson(result)
# the new block devices --------------------------
......
......@@ -79,7 +79,7 @@ class JsonErrorRequestExecutor(http.server.HttpServerRequestExecutor):
@return: the body of the message
"""
return serializer.DumpJson(values, indent=True)
return serializer.DumpJson(values)
class RemoteApiHttpServer(http.auth.HttpServerRequestAuthentication,
......
......@@ -101,7 +101,7 @@ class QmpStub(threading.Thread):
conn.close()
def encode_string(self, message):
return (serializer.DumpJson(message, indent=False) +
return (serializer.DumpJson(message) +
hv_kvm.QmpConnection._MESSAGE_END_TOKEN)
......
......@@ -52,10 +52,9 @@ class TestSerializer(testutils.GanetiTestCase):
]
def _TestSerializer(self, dump_fn, load_fn):
for indent in [True, False]:
for data in self._TESTDATA:
self.failUnless(dump_fn(data, indent=indent).endswith("\n"))
self.assertEqualValues(load_fn(dump_fn(data, indent=indent)), data)
for data in self._TESTDATA:
self.failUnless(dump_fn(data).endswith("\n"))
self.assertEqualValues(load_fn(dump_fn(data)), data)
def testGeneric(self):
self._TestSerializer(serializer.Dump, serializer.Load)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment