From b18dd0195773aebcf3249bacfd6b9c4440c22960 Mon Sep 17 00:00:00 2001 From: Iustin Pop <iustin@google.com> Date: Fri, 5 Dec 2008 10:12:58 +0000 Subject: [PATCH] http: use slicing instead of string modification The combination of the current buffer splitting method and (4KB) buffer size is very inefficient when writing big amounts of data. Just walking over a 16 megabyte string using a 4K buffer takes (on a random computer) 1m06s, whereas using slices will decrease this to 0.080s, and slicing with 32 KB size decreases this to 0.073s. This means that uploading a big config file (it nears 1MB for big clusters) will take more and more time per the number of nodes, since it needs lots of slicing. I happened upon this by accidentally setting all nodes as master candidates, at which point just uploading the config file to all nodes took 40s. Applying the patch decreases this to 15s (this probably can still be optimized). The patch also removes a duplicate constant (the one actually used is in http/client.py), and changes the receive buffer size to use the same constant. Reviewed-by: imsnah --- lib/http/__init__.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/lib/http/__init__.py b/lib/http/__init__.py index 55076b8b6..e8a355da6 100644 --- a/lib/http/__init__.py +++ b/lib/http/__init__.py @@ -36,8 +36,6 @@ from ganeti import serializer from ganeti import utils -HTTP_CLIENT_THREADS = 10 - HTTP_GANETI_VERSION = "Ganeti %s" % constants.RELEASE_VERSION HTTP_OK = 200 @@ -71,6 +69,8 @@ _SSL_UNEXPECTED_EOF = "Unexpected EOF" SOCKOP_RECV, SOCKOP_SHUTDOWN) = range(3) +# send/receive quantum +SOCK_BUF_SIZE = 32768 class HttpError(Exception): """Internal exception for HTTP errors. @@ -469,17 +469,20 @@ class HttpMessageWriter(object): buf = self._FormatMessage() poller = select.poll() - while buf: - # Send only 4 KB at a time - data = buf[:4096] + + pos = 0 + end = len(buf) + while pos < end: + # Send only SOCK_BUF_SIZE bytes at a time + data = buf[pos:pos+SOCK_BUF_SIZE] sent = SocketOperation(poller, sock, SOCKOP_SEND, data, write_timeout) # Remove sent bytes - buf = buf[sent:] + pos += sent - assert not buf, "Message wasn't sent completely" + assert pos == end, "Message wasn't sent completely" def _PrepareMessage(self): """Prepares the HTTP message by setting mandatory headers. @@ -555,7 +558,7 @@ class HttpMessageReader(object): buf = "" eof = False while self.parser_status != self.PS_COMPLETE: - data = SocketOperation(self.poller, sock, SOCKOP_RECV, 4096, + data = SocketOperation(self.poller, sock, SOCKOP_RECV, SOCK_BUF_SIZE, read_timeout) if data: -- GitLab