Commit 2a7c3583 authored by Michael Hanselmann's avatar Michael Hanselmann

RAPI client: Switch to pycURL

Currently the RAPI client uses the urllib2 and httplib modules from
Python's standard library. They're used with pyOpenSSL in a very fragile
way, and there are known issues when receiving large responses from a RAPI

By switching to PycURL we leverage the power and stability of the
widely-used curl library (libcurl). This brings us much more flexibility
than before, and timeouts were easily implemented (something that would
have involved a lot of work with the built-in modules).

There's one small drawback: Programs using libcurl have to call
curl_global_init(3) (available as pycurl.global_init) while exactly one
thread is running (e.g. before other threads) and are supposed to call
curl_global_cleanup(3) (available as pycurl.global_cleanup) upon exiting.
See the manpages for details. A decorator is provided to simplify this.

Unittests for the new code are provided, increasing the test coverage of
the RAPI client from 74% to 89%.
Signed-off-by: default avatarMichael Hanselmann <>
Reviewed-by: default avatarGuido Trotter <>
Reviewed-by: default avatarIustin Pop <>
parent b939de46
......@@ -29,6 +29,7 @@ Before installing, please verify that you have the following programs:
- `simplejson Python module <>`_
- `pyparsing Python module <>`_
- `pyinotify Python module <>`_
- `PycURL Python module <>`_
- `socat <>`_
These programs are supplied as part of most Linux distributions, so
......@@ -39,7 +40,8 @@ packages, except for DRBD and Xen::
$ apt-get install lvm2 ssh bridge-utils iproute iputils-arping \
python python-pyopenssl openssl python-pyparsing \
python-simplejson python-pyinotify socat
python-simplejson python-pyinotify python-pycurl \
If you want to build from source, please see doc/devnotes.rst for more
......@@ -610,10 +610,9 @@ def IsRapiResponding(hostname):
@return: Whether RAPI is working properly
ssl_config = rapi.client.CertAuthorityVerify(constants.RAPI_CERT_FILE)
rapi_client = \
curl_config = rapi.client.GenericCurlConfig(cafile=constants.RAPI_CERT_FILE)
rapi_client = rapi.client.GanetiRapiClient(hostname,
master_version = rapi_client.GetVersion()
except rapi.client.CertificateError, err:
......@@ -646,6 +645,7 @@ def ParseOptions():
return options, args
def main():
"""Main function.
......@@ -19,20 +19,30 @@
# 02110-1301, USA.
"""Ganeti RAPI client."""
"""Ganeti RAPI client.
@attention: To use the RAPI client, the application B{must} call
C{pycurl.global_init} during initialization and
C{pycurl.global_cleanup} before exiting the process. This is very
important in multi-threaded programs. See curl_global_init(3) and
curl_global_cleanup(3) for details. The decorator L{UsesRapiClient}
can be used.
# No Ganeti-specific modules should be imported. The RAPI client is supposed to
# be standalone.
import sys
import httplib
import urllib2
import logging
import simplejson
import socket
import urllib
import OpenSSL
import distutils.version
import threading
import pycurl
from cStringIO import StringIO
except ImportError:
from StringIO import StringIO
......@@ -61,6 +71,19 @@ NODE_ROLE_REGULAR = "regular"
_REQ_DATA_VERSION_FIELD = "__version__"
_INST_CREATE_REQV1 = "instance-create-reqv1"
# Older pycURL versions don't have all error constants
except AttributeError:
_CURL_SSL_CERT_ERRORS = frozenset([
class Error(Exception):
"""Base error class for this module.
......@@ -85,239 +108,123 @@ class GanetiApiError(Error):
self.code = code
def FormatX509Name(x509_name):
"""Formats an X509 name.
@type x509_name: OpenSSL.crypto.X509Name
def UsesRapiClient(fn):
"""Decorator for code using RAPI client to initialize pycURL.
def wrapper(*args, **kwargs):
# curl_global_init(3) and curl_global_cleanup(3) must be called with only
# one thread running. This check is just a safety measure -- it doesn't
# cover all cases.
assert threading.activeCount() == 1, \
"Found active threads when initializing pycURL"
# Only supported in pyOpenSSL 0.7 and above
get_components_fn = x509_name.get_components
except AttributeError:
return repr(x509_name)
return "".join("/%s=%s" % (name, value)
for name, value in get_components_fn())
class CertAuthorityVerify:
"""Certificate verificator for SSL context.
Configures SSL context to verify server's certificate.
_PYOPENSSL_VERSION = OpenSSL.__version__
def __init__(self, cafile=None, capath=None, use_default_verify_paths=False):
"""Initializes this class.
return fn(*args, **kwargs)
return wrapper
def GenericCurlConfig(verbose=False, use_signal=False,
use_curl_cabundle=False, cafile=None, capath=None,
proxy=None, verify_hostname=False,
connect_timeout=None, timeout=None,
"""Curl configuration function generator.
@type verbose: bool
@param verbose: Whether to set cURL to verbose mode
@type use_signal: bool
@param use_signal: Whether to allow cURL to use signals
@type use_curl_cabundle: bool
@param use_curl_cabundle: Whether to use cURL's default CA bundle
@type cafile: string
@param cafile: In which file we can find the certificates
@type capath: string
@param capath: In which directory we can find the certificates
@type use_default_verify_paths: bool
@param use_default_verify_paths: Whether the platform provided CA
certificates are to be used for
verification purposes
@type proxy: string
@param proxy: Proxy to use, None for default behaviour and empty string for
disabling proxies (see curl_easy_setopt(3))
@type verify_hostname: bool
@param verify_hostname: Whether to verify the remote peer certificate's
@type connect_timeout: number
@param connect_timeout: Timeout for establishing connection in seconds
@type timeout: number
@param timeout: Timeout for complete transfer in seconds (see
self._cafile = cafile
self._capath = capath
self._use_default_verify_paths = use_default_verify_paths
if self._capath is not None and not self._SUPPORT_CAPATH:
raise Error(("PyOpenSSL %s has no support for a CA directory,"
" version %s or above is required") %
if self._use_default_verify_paths and not self._SUPPORT_DEFVFYPATHS:
raise Error(("PyOpenSSL %s has no support for using default verification"
" paths, version %s or above is required") %
if use_curl_cabundle and (cafile or capath):
raise Error("Can not use default CA bundle when CA file or path is set")
def _VerifySslCertCb(logger, _, cert, errnum, errdepth, ok):
"""Callback for SSL certificate verification.
def _ConfigCurl(curl, logger):
"""Configures a cURL object
@param logger: Logging object
@type curl: pycurl.Curl
@param curl: cURL object
if ok:
log_fn = logger.debug
logger.debug("Using cURL version %s", pycurl.version)
# pycurl.version_info returns a tuple with information about the used
# version of libcurl. Item 5 is the SSL library linked to it.
# e.g.: (3, '7.18.0', 463360, 'x86_64-pc-linux-gnu', 1581, 'GnuTLS/2.0.4',
# 0, '', ...)
sslver = _pycurl_version_fn()[5]
if not sslver:
raise Error("No SSL support in cURL")
lcsslver = sslver.lower()
if lcsslver.startswith("openssl/"):
elif lcsslver.startswith("gnutls/"):
if capath:
raise Error("cURL linked against GnuTLS has no support for a"
" CA path (%s)" % (pycurl.version, ))
log_fn = logger.error
log_fn("Verifying SSL certificate at depth %s, subject '%s', issuer '%s'",
errdepth, FormatX509Name(cert.get_subject()),
if not ok:
# Only supported in pyOpenSSL 0.7 and above
# pylint: disable-msg=E1101
fn = OpenSSL.crypto.X509_verify_cert_error_string
except AttributeError:
errmsg = ""
raise NotImplementedError("cURL uses unsupported SSL version '%s'" %
curl.setopt(pycurl.VERBOSE, verbose)
curl.setopt(pycurl.NOSIGNAL, not use_signal)
# Whether to verify remote peer's CN
if verify_hostname:
# curl_easy_setopt(3): "When CURLOPT_SSL_VERIFYHOST is 2, that
# certificate must indicate that the server is the server to which you
# meant to connect, or the connection fails. [...] When the value is 1,
# the certificate must contain a Common Name field, but it doesn't matter
# what name it says. [...]"
curl.setopt(pycurl.SSL_VERIFYHOST, 2)
errmsg = ":%s" % fn(errnum)
logger.error("verify error:num=%s%s", errnum, errmsg)
return ok
def __call__(self, ctx, logger):
"""Configures an SSL context to verify certificates.
@type ctx: OpenSSL.SSL.Context
@param ctx: SSL context
if self._use_default_verify_paths:
if self._cafile or self._capath:
ctx.load_verify_locations(self._cafile, self._capath)
curl.setopt(pycurl.SSL_VERIFYHOST, 0)
if cafile or capath or use_curl_cabundle:
# Require certificates to be checked
curl.setopt(pycurl.SSL_VERIFYPEER, True)
if cafile:
curl.setopt(pycurl.CAINFO, str(cafile))
if capath:
curl.setopt(pycurl.CAPATH, str(capath))
# Not changing anything for using default CA bundle
# Disable SSL certificate verification
curl.setopt(pycurl.SSL_VERIFYPEER, False)
lambda conn, cert, errnum, errdepth, ok: \
self._VerifySslCertCb(logger, conn, cert,
errnum, errdepth, ok))
class _HTTPSConnectionOpenSSL(httplib.HTTPSConnection):
"""HTTPS Connection handler that verifies the SSL certificate.
# Python before version 2.6 had its own httplib.FakeSocket wrapper for
# sockets
_SUPPORT_FAKESOCKET = (sys.hexversion < 0x2060000)
def __init__(self, *args, **kwargs):
"""Initializes this class.
if proxy is not None:
curl.setopt(pycurl.PROXY, str(proxy))
httplib.HTTPSConnection.__init__(self, *args, **kwargs)
self._logger = None
self._config_ssl_verification = None
def Setup(self, logger, config_ssl_verification):
"""Sets the SSL verification config function.
@param logger: Logging object
@type config_ssl_verification: callable
assert self._logger is None
assert self._config_ssl_verification is None
self._logger = logger
self._config_ssl_verification = config_ssl_verification
def connect(self):
"""Connect to the server specified when the object was created.
This ensures that SSL certificates are verified.
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
# Timeouts
if connect_timeout is not None:
curl.setopt(pycurl.CONNECTTIMEOUT, connect_timeout)
if timeout is not None:
curl.setopt(pycurl.TIMEOUT, timeout)
ctx = OpenSSL.SSL.Context(OpenSSL.SSL.TLSv1_METHOD)
if self._config_ssl_verification:
self._config_ssl_verification(ctx, self._logger)
ssl = OpenSSL.SSL.Connection(ctx, sock)
ssl.connect((, self.port))
self.sock = httplib.FakeSocket(sock, ssl)
self.sock = _SslSocketWrapper(ssl)
class _SslSocketWrapper(object):
def __init__(self, sock):
"""Initializes this class.
self._sock = sock
def __getattr__(self, name):
"""Forward everything to underlying socket.
return getattr(self._sock, name)
def makefile(self, mode, bufsize):
"""Fake makefile method.
makefile() on normal file descriptors uses dup2(2), which doesn't work with
SSL sockets and therefore is not implemented by pyOpenSSL. This fake method
works with the httplib module, but might not work for other modules.
# pylint: disable-msg=W0212
return socket._fileobject(self._sock, mode, bufsize)
class _HTTPSHandler(urllib2.HTTPSHandler):
def __init__(self, logger, config_ssl_verification):
"""Initializes this class.
@param logger: Logging object
@type config_ssl_verification: callable
@param config_ssl_verification: Function to configure SSL context for
certificate verification
self._logger = logger
self._config_ssl_verification = config_ssl_verification
def _CreateHttpsConnection(self, *args, **kwargs):
"""Wrapper around L{_HTTPSConnectionOpenSSL} to add SSL verification.
This wrapper is necessary provide a compatible API to urllib2.
conn = _HTTPSConnectionOpenSSL(*args, **kwargs)
conn.Setup(self._logger, self._config_ssl_verification)
return conn
def https_open(self, req):
"""Creates HTTPS connection.
Called by urllib2.
return self.do_open(self._CreateHttpsConnection, req)
class _RapiRequest(urllib2.Request):
def __init__(self, method, url, headers, data):
"""Initializes this class.
urllib2.Request.__init__(self, url, data=data, headers=headers)
self._method = method
def get_method(self):
"""Returns the HTTP request method.
return self._method
return _ConfigCurl
class GanetiRapiClient(object):
......@@ -328,10 +235,9 @@ class GanetiRapiClient(object):
_json_encoder = simplejson.JSONEncoder(sort_keys=True)
def __init__(self, host, port=GANETI_RAPI_PORT,
username=None, password=None,
config_ssl_verification=None, ignore_proxy=False,
username=None, password=None, logger=logging,
curl_config_fn=None, curl=None):
"""Initializes this class.
@type host: string
@param host: the ganeti cluster master to interact with
......@@ -341,11 +247,8 @@ class GanetiRapiClient(object):
@param username: the username to connect with
@type password: string
@param password: the password to connect with
@type config_ssl_verification: callable
@param config_ssl_verification: Function to configure SSL context for
certificate verification
@type ignore_proxy: bool
@param ignore_proxy: Whether to ignore proxy settings
@type curl_config_fn: callable
@param curl_config_fn: Function to configure C{pycurl.Curl} object
@param logger: Logging object
......@@ -355,25 +258,37 @@ class GanetiRapiClient(object):
self._base_url = "https://%s:%s" % (host, port)
handlers = [_HTTPSHandler(self._logger, config_ssl_verification)]
# Create pycURL object if not supplied
if not curl:
curl = pycurl.Curl()
# Default cURL settings
curl.setopt(pycurl.VERBOSE, False)
curl.setopt(pycurl.FOLLOWLOCATION, False)
curl.setopt(pycurl.MAXREDIRS, 5)
curl.setopt(pycurl.NOSIGNAL, True)
curl.setopt(pycurl.USERAGENT, self.USER_AGENT)
curl.setopt(pycurl.SSL_VERIFYHOST, 0)
curl.setopt(pycurl.SSL_VERIFYPEER, False)
curl.setopt(pycurl.HTTPHEADER, [
"Accept: %s" % HTTP_APP_JSON,
"Content-type: %s" % HTTP_APP_JSON,
# Setup authentication
if username is not None:
pwmgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
pwmgr.add_password(None, self._base_url, username, password)
if password is None:
raise Error("Password not specified")
curl.setopt(pycurl.HTTPAUTH, pycurl.HTTPAUTH_BASIC)
curl.setopt(pycurl.USERPWD, str("%s:%s" % (username, password)))
elif password:
raise Error("Specified password without username")
if ignore_proxy:
self._http = urllib2.build_opener(*handlers) # pylint: disable-msg=W0142
# Call external configuration function
if curl_config_fn:
curl_config_fn(curl, logger)
self._headers = {
"Accept": HTTP_APP_JSON,
"Content-type": HTTP_APP_JSON,
"User-Agent": self.USER_AGENT,
self._curl = curl
def _EncodeQuery(query):
......@@ -427,10 +342,12 @@ class GanetiRapiClient(object):
assert path.startswith("/")
curl = self._curl
if content:
encoded_content = self._json_encoder.encode(content)
encoded_content = None
encoded_content = ""
# Build URL
urlparts = [self._base_url, path]
......@@ -440,30 +357,43 @@ class GanetiRapiClient(object):
url = "".join(urlparts)
self._logger.debug("Sending request %s %s to %s:%s"
" (headers=%r, content=%r)",
method, url, self._host, self._port, self._headers,
self._logger.debug("Sending request %s %s to %s:%s (content=%r)",
method, url, self._host, self._port, encoded_content)
# Buffer for response
encoded_resp_body = StringIO()
req = _RapiRequest(method, url, self._headers, encoded_content)
# Configure cURL
curl.setopt(pycurl.CUSTOMREQUEST, str(method))
curl.setopt(pycurl.URL, str(url))
curl.setopt(pycurl.POSTFIELDS, str(encoded_content))
curl.setopt(pycurl.WRITEFUNCTION, encoded_resp_body.write)
resp =
encoded_response_content =
except (OpenSSL.SSL.Error, OpenSSL.crypto.Error), err:
raise CertificateError("SSL issue: %s (%r)" % (err, err))
except urllib2.HTTPError, err:
raise GanetiApiError(str(err), code=err.code)
except urllib2.URLError, err:
raise GanetiApiError(str(err))
# Send request and wait for response
except pycurl.error, err:
if err.args[0] in _CURL_SSL_CERT_ERRORS:
raise CertificateError("SSL certificate error %s" % err)
if encoded_response_content:
response_content = simplejson.loads(encoded_response_content)
raise GanetiApiError(str(err))
# Reset settings to not keep references to large objects in memory
# between requests
curl.setopt(pycurl.POSTFIELDS, "")
curl.setopt(pycurl.WRITEFUNCTION, lambda _: None)
# Get HTTP response code
http_code = curl.getinfo(pycurl.RESPONSE_CODE)
# Was anything written to the response buffer?
if encoded_resp_body.tell():
response_content = simplejson.loads(encoded_resp_body.getvalue())
response_content = None
# TODO: Are there other status codes that are valid? (redirect?)
if resp.code != HTTP_OK:
if http_code != HTTP_OK:
if isinstance(response_content, dict):
msg = ("%s %s: %s" %
......@@ -472,7 +402,7 @@ class GanetiRapiClient(object):
msg = str(response_content)
raise GanetiApiError(msg, code=resp.code)
raise GanetiApiError(msg, code=http_code)
return response_content
......@@ -39,6 +39,9 @@ import qa_tags
import qa_utils
from ganeti import utils
from ganeti import rapi
import ganeti.rapi.client
def RunTest(fn, *args):
......@@ -269,6 +272,7 @@ def RunHardwareFailureTests(instance, pnode, snode):
instance, pnode, snode)
def main():
"""Main program.
......@@ -72,13 +72,13 @@ def Setup(username, password):
port = qa_config.get("rapi-port", default=constants.DEFAULT_RAPI_PORT)
cfg_ssl = rapi.client.CertAuthorityVerify(
cfg_curl = rapi.client.GenericCurlConfig(,
_rapi_client = rapi.client.GanetiRapiClient(master["primary"], port=port,