Merge branch 'hotfix-0.13.5'

Changelog

@@ -5,6 +5,17 @@ Unified Changelog file for Kamaki versions >= 0.13
 
 .. _Changelog-0.13:
 
+v0.13.5
+=======
+
+Bug Fixes
+=========
+* Raise SSLError when connecting with SSL on OSX and no valid SSL
+    credential files exist)
+* Use a new logging level for verbosity. This prevents the client
+    loggers in INFO mode and above from outputting the detailed
+    connection information.
+
 v0.13.4
 =======
 

kamaki/cli/__init__.py

@@ -45,7 +45,7 @@ from kamaki.cli.utils import (
 from kamaki.cli.errors import CLIError, CLICmdSpecError
 from kamaki.cli import logger
 from kamaki.clients.astakos import CachedAstakosClient
-from kamaki.clients import ClientError, KamakiSSLError
+from kamaki.clients import ClientError, KamakiSSLError, DEBUGV
 from kamaki.clients.utils import https, escape_ctrl_chars
 
 
@@ -175,9 +175,9 @@ def _setup_logging(debug=False, verbose=False):
         logger.add_stream_logger('kamaki.clients.recv', logging.DEBUG, rfmt)
         logger.add_stream_logger(__name__, logging.DEBUG)
     elif verbose:
-        logger.add_stream_logger('kamaki.clients.send', logging.INFO, sfmt)
-        logger.add_stream_logger('kamaki.clients.recv', logging.INFO, rfmt)
-        logger.add_stream_logger(__name__, logging.INFO)
+        logger.add_stream_logger('kamaki.clients.send', DEBUGV, sfmt)
+        logger.add_stream_logger('kamaki.clients.recv', DEBUGV, rfmt)
+        logger.add_stream_logger(__name__, DEBUGV)
     # else:
     #     logger.add_stream_logger(__name__, logging.WARNING)
     global kloger

kamaki/clients/__init__.py

@@ -38,20 +38,21 @@ from time import time
 from httplib import ResponseNotReady, HTTPException
 from time import sleep
 from random import random
-from logging import getLogger
+import logging
 import ssl
 
 from kamaki.clients.utils import https
-
 from kamaki.clients import utils
 
 
 TIMEOUT = 60.0   # seconds
 HTTP_METHODS = ['GET', 'POST', 'PUT', 'HEAD', 'DELETE', 'COPY', 'MOVE']
+DEBUGV = logging.DEBUG + 1
 
-log = getLogger(__name__)
-sendlog = getLogger('%s.send' % __name__)
-recvlog = getLogger('%s.recv' % __name__)
+logging.addLevelName(DEBUGV, 'DEBUGV')
+log = logging.getLogger(__name__)
+sendlog = logging.getLogger('%s.send' % __name__)
+recvlog = logging.getLogger('%s.recv' % __name__)
 
 
 def _encode(v):
@@ -154,19 +155,19 @@ class RequestManager(Logged):
 
     def dump_log(self):
         plog = ('\t[%s]' % self) if self.LOG_PID else ''
-        sendlog.info('%s %s://%s%s%s' % (
+        sendlog.log(DEBUGV, '%s %s://%s%s%s' % (
             self.method, self.scheme, self.netloc, self.path, plog))
         for key, val in self.headers.items():
             if key.lower() in ('x-auth-token', ) and not self.LOG_TOKEN:
                 self._token, val = val, '...'
-            sendlog.info('  %s: %s%s' % (key, val, plog))
+            sendlog.log(DEBUGV, '  %s: %s%s' % (key, val, plog))
         if self.data:
-            sendlog.info('data size: %s%s' % (len(self.data), plog))
+            sendlog.log(DEBUGV, 'data size: %s%s' % (len(self.data), plog))
             if self.LOG_DATA:
-                sendlog.info(utils.escape_ctrl_chars(self.data.replace(
+                sendlog.log(DEBUGV, utils.escape_ctrl_chars(self.data.replace(
                     self._token, '...') if self._token else self.data))
         else:
-            sendlog.info('data size: 0%s' % plog)
+            sendlog.log(DEBUGV, 'data size: 0%s' % plog)
 
     def _encode_headers(self):
         headers = dict()
@@ -193,7 +194,7 @@ class RequestManager(Logged):
                 url=self.path.encode('utf-8'),
                 headers=self.headers,
                 body=self.data)
-            sendlog.info('')
+            sendlog.log(DEBUGV, '')
             keep_trying = TIMEOUT
             while keep_trying > 0:
                 try:
@@ -206,7 +207,7 @@ class RequestManager(Logged):
             raise KamakiSSLError('SSL Connection error (%s)' % ssle)
         plog = ('\t[%s]' % self) if self.LOG_PID else ''
         logmsg = 'Kamaki Timeout %s %s%s' % (self.method, self.path, plog)
-        recvlog.debug(logmsg)
+        recvlog.log(DEBUGV, logmsg)
         raise ClientError('HTTPResponse takes too long - kamaki timeout')
 
     @property
@@ -272,13 +273,14 @@ class ResponseManager(Logged):
                     r = self.request.perform(connection)
                     plog = ''
                     if self.LOG_PID:
-                        recvlog.info('\n%s <-- %s <-- [req: %s]\n' % (
+                        recvlog.log(DEBUGV, '\n%s <-- %s <-- [req: %s]\n' % (
                             self, r, self.request))
                         plog = '\t[%s]' % self
                     self._request_performed = True
                     self._status_code, self._status = r.status, unquote(
                         r.reason)
-                    recvlog.info(
+                    recvlog.log(
+                        DEBUGV,
                         '%d %s%s' % (self.status_code, self.status, plog))
                     self._headers = dict()
 
@@ -287,16 +289,16 @@ class ResponseManager(Logged):
                     for k, v in r_headers:
                         self._headers[k] = unquote(v).decode('utf-8') if (
                             k.lower()) in enc_headers else v
-                        recvlog.info('  %s: %s%s' % (k, v, plog))
+                        recvlog.log(DEBUGV, '  %s: %s%s' % (k, v, plog))
                     self._content = r.read()
-                    recvlog.info('data size: %s%s' % (
+                    recvlog.log(DEBUGV, 'data size: %s%s' % (
                         len(self._content) if self._content else 0, plog))
                     if self.LOG_DATA and self._content:
                         data = '%s%s' % (self._content, plog)
                         data = utils.escape_ctrl_chars(data)
                         if self._token:
                             data = data.replace(self._token, '...')
-                        recvlog.info(data)
+                        recvlog.log(DEBUGV, data)
                 break
             except Exception as err:
                 if isinstance(err, HTTPException):
@@ -306,8 +308,8 @@ class ResponseManager(Logged):
                                 self.request.url, retries, type(err), err))
                 else:
                     from traceback import format_stack
-                    recvlog.debug(
-                        '\n'.join(['%s' % type(err)] + format_stack()))
+                    recvlog.log(
+                        DEBUGV, '\n'.join(['%s' % type(err)] + format_stack()))
                     raise
 
     @property
@@ -492,7 +494,7 @@ class Client(Logged):
                 else:
                     results[key] = thread.value
             flying = unfinished
-        sendlog.info('- - - wait for threads to finish')
+        sendlog.debug('- - - wait for threads to finish')
         for key, thread in flying.items():
             if thread.isAlive():
                 thread.join()
@@ -537,7 +539,8 @@ class Client(Logged):
             if data:
                 headers.setdefault('Content-Length', '%s' % len(data))
             plog = ('\t[%s]' % self) if self.LOG_PID else ''
-            sendlog.debug('\n\nCMT %s@%s%s', method, self.endpoint_url, plog)
+            sendlog.log(
+                DEBUGV, '\n\nCMT %s@%s%s', method, self.endpoint_url, plog)
             req = RequestManager(
                 method, self.endpoint_url, path,
                 data=data, headers=headers, params=params)

kamaki/clients/astakos/test.py

@@ -91,8 +91,8 @@ class AstakosClient(TestCase):
                 (['some url', 'some token', 'other', 'params'], {}),
                 (['some url', 'some token', 'other params'], (dict(k='v'))),
                 (['some url', 'some token'], (dict(k1='v1', k2='v2'))),
-                (['some url', ], (dict(k1='v1', k2='v2', token='some token'))),
-            ):
+                (['some url', ], (dict(k1='v1', k2='v2', token='some token')))
+                ):
             astakos.AstakosClient(*args, **kwargs)
             url, token = args.pop(0), kwargs.pop('token', None) or args.pop(0)
             self.assertTrue(
@@ -139,7 +139,7 @@ class LoggedAstakosClient(TestCase):
     def tearDown(self):
         FR.headers = {}
 
-    @patch('kamaki.clients.recvlog.info', return_value='recvlog info')
+    @patch('kamaki.clients.recvlog.debug', return_value='recvlog info')
     def test__dump_response(self, recvlog_info):
         for headers, status, message, data, LOG_DATA, LOG_TOKEN in product(
                 (

kamaki/clients/pithos/__init__.py

@@ -37,14 +37,17 @@ from os import fstat
 from hashlib import new as newhashlib
 from time import time
 from StringIO import StringIO
+from logging import getLogger
 
 from binascii import hexlify
 
-from kamaki.clients import SilentEvent, sendlog
+from kamaki.clients import SilentEvent
 from kamaki.clients.pithos.rest_api import PithosRestClient
 from kamaki.clients.storage import ClientError
 from kamaki.clients.utils import path4url, filter_in, readall
 
+LOG = getLogger(__name__)
+
 
 def _pithos_hash(block, blockhash):
     h = newhashlib(blockhash)
@@ -457,14 +460,14 @@ class PithosClient(PithosRestClient):
                 try:
                     upload_gen.next()
                 except:
-                    sendlog.debug('Progress bar failure')
+                    LOG.debug('Progress bar failure')
                     break
         else:
             upload_gen = None
 
         retries = 7
         while retries:
-            sendlog.info('%s blocks missing' % len(missing))
+            LOG.debug('%s blocks missing' % len(missing))
             num_of_blocks = len(missing)
             missing = self._upload_missing_blocks(
                 missing, hmap, f, upload_gen)
@@ -613,7 +616,7 @@ class PithosClient(PithosRestClient):
             if missing:
                 raise ClientError('%s blocks failed to upload' % len(missing))
         except KeyboardInterrupt:
-            sendlog.info('- - - wait for threads to finish')
+            LOG.debug('- - - wait for threads to finish')
             for thread in activethreads():
                 thread.join()
             raise
@@ -908,7 +911,7 @@ class PithosClient(PithosRestClient):
                     flying.pop(runid)
             return ''.join(ret)
         except KeyboardInterrupt:
-            sendlog.info('- - - wait for threads to finish')
+            LOG.debug('- - - wait for threads to finish')
             for thread in activethreads():
                 thread.join()
 
@@ -1360,7 +1363,7 @@ class PithosClient(PithosRestClient):
                     self._cb_next()
                 flying = unfinished
         except KeyboardInterrupt:
-            sendlog.info('- - - wait for threads to finish')
+            LOG.debug('- - - wait for threads to finish')
             for thread in activethreads():
                 thread.join()
         finally:

kamaki/clients/utils/https.py

-# Copyright 2014 GRNET S.A. All rights reserved.
+# Copyright 2014-2015 GRNET S.A. All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or
 # without modification, are permitted provided that the following
@@ -35,6 +35,7 @@ import logging
 import httplib
 import socket
 import ssl
+import os.path
 from objpool import http
 
 log = logging.getLogger(__name__)
@@ -44,6 +45,10 @@ class SSLUnicodeError(ssl.SSLError):
     """SSL module cannot handle unicode file names"""
 
 
+class SSLCredentialsMissing(ssl.SSLError):
+    """Missing credentials for SSL authentication"""
+
+
 class HTTPSClientAuthConnection(httplib.HTTPSConnection):
     """HTTPS connection, with full client-based SSL Authentication support"""
 
@@ -55,7 +60,8 @@ class HTTPSClientAuthConnection(httplib.HTTPSConnection):
             :param ignore_ssl: flag (default: False)
         """
         try:
-            self.ca_file = str(kwargs.pop('ca_file', self.ca_file))
+            self.ca_file = str(
+                kwargs.pop('ca_file', self.ca_file) or '') or None
         except UnicodeError as ue:
             raise SSLUnicodeError(0, SSLUnicodeError.__doc__, ue)
 
@@ -91,6 +97,21 @@ class HTTPSClientAuthConnection(httplib.HTTPSConnection):
                     ca_certs=self.ca_file, cert_reqs=ssl.CERT_REQUIRED)
         except UnicodeError as ue:
             raise SSLUnicodeError(0, SSLUnicodeError.__doc__, ue)
+        except IOError as ioe:
+            # In OSX, a wrong SSL credential file may raise an IOError with
+            # errno of 2, instead of an SSL error
+            # Wrap it in SSL and raise it
+            if getattr(ioe, 'errno', None) == 2:
+                files = self.key_file, self.cert_file, self.ca_file
+                if not any(files):
+                    raise SSLCredentialsMissing(
+                        'No SSL cred. files provided (IOError:%s)' % ioe)
+                for f in files:
+                    if f and not os.path.exists(f):
+                        raise SSLCredentialsMissing(
+                            'SSL cred. file %s does not exist (IOError:%s)' % (
+                                f, ioe))
+            raise
 
 
 http.HTTPConnectionPool._scheme_to_class['https'] = HTTPSClientAuthConnection

kamaki/version.py

 
-__version__ = "0.13.4"
+__version__ = "0.13.5"
 __version_vcs_info__ = {
-    'branch': 'hotfix-0.13.4',
-    'revid': '9a1c9ba',
-    'revno': 2503}
+    'branch': 'hotfix-0.13.5',
+    'revid': 'dfe824d',
+    'revno': 2509}
 __version_user_email__ = "saxtouri@admin.grnet.gr"
 __version_user_name__ = "Stavros Sachtouris"

version

-0.13.4
+0.13.5