Skip to content

GitLab

K
kamaki
Commit 58fef81d authored by Nikos Skalkotos's avatar Nikos Skalkotos
Browse files
Options

Merge branch 'hotfix-0.13.5'

parents 43312fd2 187afeff
Hide whitespace changes
Inline Side-by-side
Showing with 79 additions and 41 deletions
Changelog
View file @ 58fef81d
......@@ -5,6 +5,17 @@ Unified Changelog file for Kamaki versions >= 0.13
.. _Changelog-0.13:
v0.13.5
=======
Bug Fixes
=========
* Raise SSLError when connecting with SSL on OSX and no valid SSL
credential files exist)
* Use a new logging level for verbosity. This prevents the client
loggers in INFO mode and above from outputting the detailed
connection information.
v0.13.4
=======
......
kamaki/cli/__init__.py
View file @ 58fef81d
......@@ -45,7 +45,7 @@ from kamaki.cli.utils import (
from kamaki.cli.errors import CLIError, CLICmdSpecError
from kamaki.cli import logger
from kamaki.clients.astakos import CachedAstakosClient
from kamaki.clients import ClientError, KamakiSSLError
from kamaki.clients import ClientError, KamakiSSLError, DEBUGV
from kamaki.clients.utils import https, escape_ctrl_chars
......@@ -175,9 +175,9 @@ def _setup_logging(debug=False, verbose=False):
logger.add_stream_logger('kamaki.clients.recv', logging.DEBUG, rfmt)
logger.add_stream_logger(__name__, logging.DEBUG)
elif verbose:
logger.add_stream_logger('kamaki.clients.send', logging.INFO, sfmt)
logger.add_stream_logger('kamaki.clients.recv', logging.INFO, rfmt)
logger.add_stream_logger(__name__, logging.INFO)
logger.add_stream_logger('kamaki.clients.send', DEBUGV, sfmt)
logger.add_stream_logger('kamaki.clients.recv', DEBUGV, rfmt)
logger.add_stream_logger(__name__, DEBUGV)
# else:
# logger.add_stream_logger(__name__, logging.WARNING)
global kloger
......
kamaki/clients/__init__.py
View file @ 58fef81d
......@@ -38,20 +38,21 @@ from time import time
from httplib import ResponseNotReady, HTTPException
from time import sleep
from random import random
from logging import getLogger
import logging
import ssl
from kamaki.clients.utils import https
from kamaki.clients import utils
TIMEOUT = 60.0 # seconds
HTTP_METHODS = ['GET', 'POST', 'PUT', 'HEAD', 'DELETE', 'COPY', 'MOVE']
DEBUGV = logging.DEBUG + 1
log = getLogger(__name__)
sendlog = getLogger('%s.send' % __name__)
recvlog = getLogger('%s.recv' % __name__)
logging.addLevelName(DEBUGV, 'DEBUGV')
log = logging.getLogger(__name__)
sendlog = logging.getLogger('%s.send' % __name__)
recvlog = logging.getLogger('%s.recv' % __name__)
def _encode(v):
......@@ -154,19 +155,19 @@ class RequestManager(Logged):
def dump_log(self):
plog = ('\t[%s]' % self) if self.LOG_PID else ''
sendlog.info('%s %s://%s%s%s' % (
sendlog.log(DEBUGV, '%s %s://%s%s%s' % (
self.method, self.scheme, self.netloc, self.path, plog))
for key, val in self.headers.items():
if key.lower() in ('x-auth-token', ) and not self.LOG_TOKEN:
self._token, val = val, '...'
sendlog.info(' %s: %s%s' % (key, val, plog))
sendlog.log(DEBUGV, ' %s: %s%s' % (key, val, plog))
if self.data:
sendlog.info('data size: %s%s' % (len(self.data), plog))
sendlog.log(DEBUGV, 'data size: %s%s' % (len(self.data), plog))
if self.LOG_DATA:
sendlog.info(utils.escape_ctrl_chars(self.data.replace(
sendlog.log(DEBUGV, utils.escape_ctrl_chars(self.data.replace(
self._token, '...') if self._token else self.data))
else:
sendlog.info('data size: 0%s' % plog)
sendlog.log(DEBUGV, 'data size: 0%s' % plog)
def _encode_headers(self):
headers = dict()
......@@ -193,7 +194,7 @@ class RequestManager(Logged):
url=self.path.encode('utf-8'),
headers=self.headers,
body=self.data)
sendlog.info('')
sendlog.log(DEBUGV, '')
keep_trying = TIMEOUT
while keep_trying > 0:
try:
......@@ -206,7 +207,7 @@ class RequestManager(Logged):
raise KamakiSSLError('SSL Connection error (%s)' % ssle)
plog = ('\t[%s]' % self) if self.LOG_PID else ''
logmsg = 'Kamaki Timeout %s %s%s' % (self.method, self.path, plog)
recvlog.debug(logmsg)
recvlog.log(DEBUGV, logmsg)
raise ClientError('HTTPResponse takes too long - kamaki timeout')
@property
......@@ -272,13 +273,14 @@ class ResponseManager(Logged):
r = self.request.perform(connection)
plog = ''
if self.LOG_PID:
recvlog.info('\n%s <-- %s <-- [req: %s]\n' % (
recvlog.log(DEBUGV, '\n%s <-- %s <-- [req: %s]\n' % (
self, r, self.request))
plog = '\t[%s]' % self
self._request_performed = True
self._status_code, self._status = r.status, unquote(
r.reason)
recvlog.info(
recvlog.log(
DEBUGV,
'%d %s%s' % (self.status_code, self.status, plog))
self._headers = dict()
......@@ -287,16 +289,16 @@ class ResponseManager(Logged):
for k, v in r_headers:
self._headers[k] = unquote(v).decode('utf-8') if (
k.lower()) in enc_headers else v
recvlog.info(' %s: %s%s' % (k, v, plog))
recvlog.log(DEBUGV, ' %s: %s%s' % (k, v, plog))
self._content = r.read()
recvlog.info('data size: %s%s' % (
recvlog.log(DEBUGV, 'data size: %s%s' % (
len(self._content) if self._content else 0, plog))
if self.LOG_DATA and self._content:
data = '%s%s' % (self._content, plog)
data = utils.escape_ctrl_chars(data)
if self._token:
data = data.replace(self._token, '...')
recvlog.info(data)
recvlog.log(DEBUGV, data)
break
except Exception as err:
if isinstance(err, HTTPException):
......@@ -306,8 +308,8 @@ class ResponseManager(Logged):
self.request.url, retries, type(err), err))
else:
from traceback import format_stack
recvlog.debug(
'\n'.join(['%s' % type(err)] + format_stack()))
recvlog.log(
DEBUGV, '\n'.join(['%s' % type(err)] + format_stack()))
raise
@property
......@@ -492,7 +494,7 @@ class Client(Logged):
else:
results[key] = thread.value
flying = unfinished
sendlog.info('- - - wait for threads to finish')
sendlog.debug('- - - wait for threads to finish')
for key, thread in flying.items():
if thread.isAlive():
thread.join()
......@@ -537,7 +539,8 @@ class Client(Logged):
if data:
headers.setdefault('Content-Length', '%s' % len(data))
plog = ('\t[%s]' % self) if self.LOG_PID else ''
sendlog.debug('\n\nCMT %s@%s%s', method, self.endpoint_url, plog)
sendlog.log(
DEBUGV, '\n\nCMT %s@%s%s', method, self.endpoint_url, plog)
req = RequestManager(
method, self.endpoint_url, path,
data=data, headers=headers, params=params)
......
kamaki/clients/astakos/test.py
View file @ 58fef81d
......@@ -91,8 +91,8 @@ class AstakosClient(TestCase):
(['some url', 'some token', 'other', 'params'], {}),
(['some url', 'some token', 'other params'], (dict(k='v'))),
(['some url', 'some token'], (dict(k1='v1', k2='v2'))),
(['some url', ], (dict(k1='v1', k2='v2', token='some token'))),
):
(['some url', ], (dict(k1='v1', k2='v2', token='some token')))
):
astakos.AstakosClient(*args, **kwargs)
url, token = args.pop(0), kwargs.pop('token', None) or args.pop(0)
self.assertTrue(
......@@ -139,7 +139,7 @@ class LoggedAstakosClient(TestCase):
def tearDown(self):
FR.headers = {}
@patch('kamaki.clients.recvlog.info', return_value='recvlog info')
@patch('kamaki.clients.recvlog.debug', return_value='recvlog info')
def test__dump_response(self, recvlog_info):
for headers, status, message, data, LOG_DATA, LOG_TOKEN in product(
(
......
kamaki/clients/pithos/__init__.py
View file @ 58fef81d
......@@ -37,14 +37,17 @@ from os import fstat
from hashlib import new as newhashlib
from time import time
from StringIO import StringIO
from logging import getLogger
from binascii import hexlify
from kamaki.clients import SilentEvent, sendlog
from kamaki.clients import SilentEvent
from kamaki.clients.pithos.rest_api import PithosRestClient
from kamaki.clients.storage import ClientError
from kamaki.clients.utils import path4url, filter_in, readall
LOG = getLogger(__name__)
def _pithos_hash(block, blockhash):
h = newhashlib(blockhash)
......@@ -457,14 +460,14 @@ class PithosClient(PithosRestClient):
try:
upload_gen.next()
except:
sendlog.debug('Progress bar failure')
LOG.debug('Progress bar failure')
break
else:
upload_gen = None
retries = 7
while retries:
sendlog.info('%s blocks missing' % len(missing))
LOG.debug('%s blocks missing' % len(missing))
num_of_blocks = len(missing)
missing = self._upload_missing_blocks(
missing, hmap, f, upload_gen)
......@@ -613,7 +616,7 @@ class PithosClient(PithosRestClient):
if missing:
raise ClientError('%s blocks failed to upload' % len(missing))
except KeyboardInterrupt:
sendlog.info('- - - wait for threads to finish')
LOG.debug('- - - wait for threads to finish')
for thread in activethreads():
thread.join()
raise
......@@ -908,7 +911,7 @@ class PithosClient(PithosRestClient):
flying.pop(runid)
return ''.join(ret)
except KeyboardInterrupt:
sendlog.info('- - - wait for threads to finish')
LOG.debug('- - - wait for threads to finish')
for thread in activethreads():
thread.join()
......@@ -1360,7 +1363,7 @@ class PithosClient(PithosRestClient):
self._cb_next()
flying = unfinished
except KeyboardInterrupt:
sendlog.info('- - - wait for threads to finish')
LOG.debug('- - - wait for threads to finish')
for thread in activethreads():
thread.join()
finally:
......
kamaki/clients/utils/https.py
View file @ 58fef81d
# Copyright 2014 GRNET S.A. All rights reserved.
# Copyright 2014-2015 GRNET S.A. All rights reserved.
#
# Redistribution and use in source and binary forms, with or
# without modification, are permitted provided that the following
......@@ -35,6 +35,7 @@ import logging
import httplib
import socket
import ssl
import os.path
from objpool import http
log = logging.getLogger(__name__)
......@@ -44,6 +45,10 @@ class SSLUnicodeError(ssl.SSLError):
"""SSL module cannot handle unicode file names"""
class SSLCredentialsMissing(ssl.SSLError):
"""Missing credentials for SSL authentication"""
class HTTPSClientAuthConnection(httplib.HTTPSConnection):
"""HTTPS connection, with full client-based SSL Authentication support"""
......@@ -55,7 +60,8 @@ class HTTPSClientAuthConnection(httplib.HTTPSConnection):
:param ignore_ssl: flag (default: False)
"""
try:
self.ca_file = str(kwargs.pop('ca_file', self.ca_file))
self.ca_file = str(
kwargs.pop('ca_file', self.ca_file) or '') or None
except UnicodeError as ue:
raise SSLUnicodeError(0, SSLUnicodeError.__doc__, ue)
......@@ -91,6 +97,21 @@ class HTTPSClientAuthConnection(httplib.HTTPSConnection):
ca_certs=self.ca_file, cert_reqs=ssl.CERT_REQUIRED)
except UnicodeError as ue:
raise SSLUnicodeError(0, SSLUnicodeError.__doc__, ue)
except IOError as ioe:
# In OSX, a wrong SSL credential file may raise an IOError with
# errno of 2, instead of an SSL error
# Wrap it in SSL and raise it
if getattr(ioe, 'errno', None) == 2:
files = self.key_file, self.cert_file, self.ca_file
if not any(files):
raise SSLCredentialsMissing(
'No SSL cred. files provided (IOError:%s)' % ioe)
for f in files:
if f and not os.path.exists(f):
raise SSLCredentialsMissing(
'SSL cred. file %s does not exist (IOError:%s)' % (
f, ioe))
raise
http.HTTPConnectionPool._scheme_to_class['https'] = HTTPSClientAuthConnection
......
kamaki/version.py
View file @ 58fef81d
__version__ = "0.13.4"
__version__ = "0.13.5"
__version_vcs_info__ = {
'branch': 'hotfix-0.13.4',
'revid': '9a1c9ba',
'revno': 2503}
'branch': 'hotfix-0.13.5',
'revid': 'dfe824d',
'revno': 2509}
__version_user_email__ = "saxtouri@admin.grnet.gr"
__version_user_name__ = "Stavros Sachtouris"
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment