Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
itminedu
kamaki
Commits
58fef81d
Commit
58fef81d
authored
Sep 24, 2015
by
Nikos Skalkotos
Browse files
Merge branch 'hotfix-0.13.5'
parents
43312fd2
187afeff
Changes
8
Hide whitespace changes
Inline
Side-by-side
Changelog
View file @
58fef81d
...
...
@@ -5,6 +5,17 @@ Unified Changelog file for Kamaki versions >= 0.13
.. _Changelog-0.13:
v0.13.5
=======
Bug Fixes
=========
* Raise SSLError when connecting with SSL on OSX and no valid SSL
credential files exist)
* Use a new logging level for verbosity. This prevents the client
loggers in INFO mode and above from outputting the detailed
connection information.
v0.13.4
=======
...
...
kamaki/cli/__init__.py
View file @
58fef81d
...
...
@@ -45,7 +45,7 @@ from kamaki.cli.utils import (
from
kamaki.cli.errors
import
CLIError
,
CLICmdSpecError
from
kamaki.cli
import
logger
from
kamaki.clients.astakos
import
CachedAstakosClient
from
kamaki.clients
import
ClientError
,
KamakiSSLError
from
kamaki.clients
import
ClientError
,
KamakiSSLError
,
DEBUGV
from
kamaki.clients.utils
import
https
,
escape_ctrl_chars
...
...
@@ -175,9 +175,9 @@ def _setup_logging(debug=False, verbose=False):
logger
.
add_stream_logger
(
'kamaki.clients.recv'
,
logging
.
DEBUG
,
rfmt
)
logger
.
add_stream_logger
(
__name__
,
logging
.
DEBUG
)
elif
verbose
:
logger
.
add_stream_logger
(
'kamaki.clients.send'
,
logging
.
INFO
,
sfmt
)
logger
.
add_stream_logger
(
'kamaki.clients.recv'
,
logging
.
INFO
,
rfmt
)
logger
.
add_stream_logger
(
__name__
,
logging
.
INFO
)
logger
.
add_stream_logger
(
'kamaki.clients.send'
,
DEBUGV
,
sfmt
)
logger
.
add_stream_logger
(
'kamaki.clients.recv'
,
DEBUGV
,
rfmt
)
logger
.
add_stream_logger
(
__name__
,
DEBUGV
)
# else:
# logger.add_stream_logger(__name__, logging.WARNING)
global
kloger
...
...
kamaki/clients/__init__.py
View file @
58fef81d
...
...
@@ -38,20 +38,21 @@ from time import time
from
httplib
import
ResponseNotReady
,
HTTPException
from
time
import
sleep
from
random
import
random
from
logging
import
getLogger
import
logging
import
ssl
from
kamaki.clients.utils
import
https
from
kamaki.clients
import
utils
TIMEOUT
=
60.0
# seconds
HTTP_METHODS
=
[
'GET'
,
'POST'
,
'PUT'
,
'HEAD'
,
'DELETE'
,
'COPY'
,
'MOVE'
]
DEBUGV
=
logging
.
DEBUG
+
1
log
=
getLogger
(
__name__
)
sendlog
=
getLogger
(
'%s.send'
%
__name__
)
recvlog
=
getLogger
(
'%s.recv'
%
__name__
)
logging
.
addLevelName
(
DEBUGV
,
'DEBUGV'
)
log
=
logging
.
getLogger
(
__name__
)
sendlog
=
logging
.
getLogger
(
'%s.send'
%
__name__
)
recvlog
=
logging
.
getLogger
(
'%s.recv'
%
__name__
)
def
_encode
(
v
):
...
...
@@ -154,19 +155,19 @@ class RequestManager(Logged):
def
dump_log
(
self
):
plog
=
(
'
\t
[%s]'
%
self
)
if
self
.
LOG_PID
else
''
sendlog
.
info
(
'%s %s://%s%s%s'
%
(
sendlog
.
log
(
DEBUGV
,
'%s %s://%s%s%s'
%
(
self
.
method
,
self
.
scheme
,
self
.
netloc
,
self
.
path
,
plog
))
for
key
,
val
in
self
.
headers
.
items
():
if
key
.
lower
()
in
(
'x-auth-token'
,
)
and
not
self
.
LOG_TOKEN
:
self
.
_token
,
val
=
val
,
'...'
sendlog
.
info
(
' %s: %s%s'
%
(
key
,
val
,
plog
))
sendlog
.
log
(
DEBUGV
,
' %s: %s%s'
%
(
key
,
val
,
plog
))
if
self
.
data
:
sendlog
.
info
(
'data size: %s%s'
%
(
len
(
self
.
data
),
plog
))
sendlog
.
log
(
DEBUGV
,
'data size: %s%s'
%
(
len
(
self
.
data
),
plog
))
if
self
.
LOG_DATA
:
sendlog
.
info
(
utils
.
escape_ctrl_chars
(
self
.
data
.
replace
(
sendlog
.
log
(
DEBUGV
,
utils
.
escape_ctrl_chars
(
self
.
data
.
replace
(
self
.
_token
,
'...'
)
if
self
.
_token
else
self
.
data
))
else
:
sendlog
.
info
(
'data size: 0%s'
%
plog
)
sendlog
.
log
(
DEBUGV
,
'data size: 0%s'
%
plog
)
def
_encode_headers
(
self
):
headers
=
dict
()
...
...
@@ -193,7 +194,7 @@ class RequestManager(Logged):
url
=
self
.
path
.
encode
(
'utf-8'
),
headers
=
self
.
headers
,
body
=
self
.
data
)
sendlog
.
info
(
''
)
sendlog
.
log
(
DEBUGV
,
''
)
keep_trying
=
TIMEOUT
while
keep_trying
>
0
:
try
:
...
...
@@ -206,7 +207,7 @@ class RequestManager(Logged):
raise
KamakiSSLError
(
'SSL Connection error (%s)'
%
ssle
)
plog
=
(
'
\t
[%s]'
%
self
)
if
self
.
LOG_PID
else
''
logmsg
=
'Kamaki Timeout %s %s%s'
%
(
self
.
method
,
self
.
path
,
plog
)
recvlog
.
debug
(
logmsg
)
recvlog
.
log
(
DEBUGV
,
logmsg
)
raise
ClientError
(
'HTTPResponse takes too long - kamaki timeout'
)
@
property
...
...
@@ -272,13 +273,14 @@ class ResponseManager(Logged):
r
=
self
.
request
.
perform
(
connection
)
plog
=
''
if
self
.
LOG_PID
:
recvlog
.
info
(
'
\n
%s <-- %s <-- [req: %s]
\n
'
%
(
recvlog
.
log
(
DEBUGV
,
'
\n
%s <-- %s <-- [req: %s]
\n
'
%
(
self
,
r
,
self
.
request
))
plog
=
'
\t
[%s]'
%
self
self
.
_request_performed
=
True
self
.
_status_code
,
self
.
_status
=
r
.
status
,
unquote
(
r
.
reason
)
recvlog
.
info
(
recvlog
.
log
(
DEBUGV
,
'%d %s%s'
%
(
self
.
status_code
,
self
.
status
,
plog
))
self
.
_headers
=
dict
()
...
...
@@ -287,16 +289,16 @@ class ResponseManager(Logged):
for
k
,
v
in
r_headers
:
self
.
_headers
[
k
]
=
unquote
(
v
).
decode
(
'utf-8'
)
if
(
k
.
lower
())
in
enc_headers
else
v
recvlog
.
info
(
' %s: %s%s'
%
(
k
,
v
,
plog
))
recvlog
.
log
(
DEBUGV
,
' %s: %s%s'
%
(
k
,
v
,
plog
))
self
.
_content
=
r
.
read
()
recvlog
.
info
(
'data size: %s%s'
%
(
recvlog
.
log
(
DEBUGV
,
'data size: %s%s'
%
(
len
(
self
.
_content
)
if
self
.
_content
else
0
,
plog
))
if
self
.
LOG_DATA
and
self
.
_content
:
data
=
'%s%s'
%
(
self
.
_content
,
plog
)
data
=
utils
.
escape_ctrl_chars
(
data
)
if
self
.
_token
:
data
=
data
.
replace
(
self
.
_token
,
'...'
)
recvlog
.
info
(
data
)
recvlog
.
log
(
DEBUGV
,
data
)
break
except
Exception
as
err
:
if
isinstance
(
err
,
HTTPException
):
...
...
@@ -306,8 +308,8 @@ class ResponseManager(Logged):
self
.
request
.
url
,
retries
,
type
(
err
),
err
))
else
:
from
traceback
import
format_stack
recvlog
.
debu
g
(
'
\n
'
.
join
([
'%s'
%
type
(
err
)]
+
format_stack
()))
recvlog
.
lo
g
(
DEBUGV
,
'
\n
'
.
join
([
'%s'
%
type
(
err
)]
+
format_stack
()))
raise
@
property
...
...
@@ -492,7 +494,7 @@ class Client(Logged):
else
:
results
[
key
]
=
thread
.
value
flying
=
unfinished
sendlog
.
info
(
'- - - wait for threads to finish'
)
sendlog
.
debug
(
'- - - wait for threads to finish'
)
for
key
,
thread
in
flying
.
items
():
if
thread
.
isAlive
():
thread
.
join
()
...
...
@@ -537,7 +539,8 @@ class Client(Logged):
if
data
:
headers
.
setdefault
(
'Content-Length'
,
'%s'
%
len
(
data
))
plog
=
(
'
\t
[%s]'
%
self
)
if
self
.
LOG_PID
else
''
sendlog
.
debug
(
'
\n\n
CMT %s@%s%s'
,
method
,
self
.
endpoint_url
,
plog
)
sendlog
.
log
(
DEBUGV
,
'
\n\n
CMT %s@%s%s'
,
method
,
self
.
endpoint_url
,
plog
)
req
=
RequestManager
(
method
,
self
.
endpoint_url
,
path
,
data
=
data
,
headers
=
headers
,
params
=
params
)
...
...
kamaki/clients/astakos/test.py
View file @
58fef81d
...
...
@@ -91,8 +91,8 @@ class AstakosClient(TestCase):
([
'some url'
,
'some token'
,
'other'
,
'params'
],
{}),
([
'some url'
,
'some token'
,
'other params'
],
(
dict
(
k
=
'v'
))),
([
'some url'
,
'some token'
],
(
dict
(
k1
=
'v1'
,
k2
=
'v2'
))),
([
'some url'
,
],
(
dict
(
k1
=
'v1'
,
k2
=
'v2'
,
token
=
'some token'
)))
,
):
([
'some url'
,
],
(
dict
(
k1
=
'v1'
,
k2
=
'v2'
,
token
=
'some token'
)))
):
astakos
.
AstakosClient
(
*
args
,
**
kwargs
)
url
,
token
=
args
.
pop
(
0
),
kwargs
.
pop
(
'token'
,
None
)
or
args
.
pop
(
0
)
self
.
assertTrue
(
...
...
@@ -139,7 +139,7 @@ class LoggedAstakosClient(TestCase):
def
tearDown
(
self
):
FR
.
headers
=
{}
@
patch
(
'kamaki.clients.recvlog.
info
'
,
return_value
=
'recvlog info'
)
@
patch
(
'kamaki.clients.recvlog.
debug
'
,
return_value
=
'recvlog info'
)
def
test__dump_response
(
self
,
recvlog_info
):
for
headers
,
status
,
message
,
data
,
LOG_DATA
,
LOG_TOKEN
in
product
(
(
...
...
kamaki/clients/pithos/__init__.py
View file @
58fef81d
...
...
@@ -37,14 +37,17 @@ from os import fstat
from
hashlib
import
new
as
newhashlib
from
time
import
time
from
StringIO
import
StringIO
from
logging
import
getLogger
from
binascii
import
hexlify
from
kamaki.clients
import
SilentEvent
,
sendlog
from
kamaki.clients
import
SilentEvent
from
kamaki.clients.pithos.rest_api
import
PithosRestClient
from
kamaki.clients.storage
import
ClientError
from
kamaki.clients.utils
import
path4url
,
filter_in
,
readall
LOG
=
getLogger
(
__name__
)
def
_pithos_hash
(
block
,
blockhash
):
h
=
newhashlib
(
blockhash
)
...
...
@@ -457,14 +460,14 @@ class PithosClient(PithosRestClient):
try
:
upload_gen
.
next
()
except
:
sendlog
.
debug
(
'Progress bar failure'
)
LOG
.
debug
(
'Progress bar failure'
)
break
else
:
upload_gen
=
None
retries
=
7
while
retries
:
sendlog
.
info
(
'%s blocks missing'
%
len
(
missing
))
LOG
.
debug
(
'%s blocks missing'
%
len
(
missing
))
num_of_blocks
=
len
(
missing
)
missing
=
self
.
_upload_missing_blocks
(
missing
,
hmap
,
f
,
upload_gen
)
...
...
@@ -613,7 +616,7 @@ class PithosClient(PithosRestClient):
if
missing
:
raise
ClientError
(
'%s blocks failed to upload'
%
len
(
missing
))
except
KeyboardInterrupt
:
sendlog
.
info
(
'- - - wait for threads to finish'
)
LOG
.
debug
(
'- - - wait for threads to finish'
)
for
thread
in
activethreads
():
thread
.
join
()
raise
...
...
@@ -908,7 +911,7 @@ class PithosClient(PithosRestClient):
flying
.
pop
(
runid
)
return
''
.
join
(
ret
)
except
KeyboardInterrupt
:
sendlog
.
info
(
'- - - wait for threads to finish'
)
LOG
.
debug
(
'- - - wait for threads to finish'
)
for
thread
in
activethreads
():
thread
.
join
()
...
...
@@ -1360,7 +1363,7 @@ class PithosClient(PithosRestClient):
self
.
_cb_next
()
flying
=
unfinished
except
KeyboardInterrupt
:
sendlog
.
info
(
'- - - wait for threads to finish'
)
LOG
.
debug
(
'- - - wait for threads to finish'
)
for
thread
in
activethreads
():
thread
.
join
()
finally
:
...
...
kamaki/clients/utils/https.py
View file @
58fef81d
# Copyright 2014 GRNET S.A. All rights reserved.
# Copyright 2014
-2015
GRNET S.A. All rights reserved.
#
# Redistribution and use in source and binary forms, with or
# without modification, are permitted provided that the following
...
...
@@ -35,6 +35,7 @@ import logging
import
httplib
import
socket
import
ssl
import
os.path
from
objpool
import
http
log
=
logging
.
getLogger
(
__name__
)
...
...
@@ -44,6 +45,10 @@ class SSLUnicodeError(ssl.SSLError):
"""SSL module cannot handle unicode file names"""
class
SSLCredentialsMissing
(
ssl
.
SSLError
):
"""Missing credentials for SSL authentication"""
class
HTTPSClientAuthConnection
(
httplib
.
HTTPSConnection
):
"""HTTPS connection, with full client-based SSL Authentication support"""
...
...
@@ -55,7 +60,8 @@ class HTTPSClientAuthConnection(httplib.HTTPSConnection):
:param ignore_ssl: flag (default: False)
"""
try
:
self
.
ca_file
=
str
(
kwargs
.
pop
(
'ca_file'
,
self
.
ca_file
))
self
.
ca_file
=
str
(
kwargs
.
pop
(
'ca_file'
,
self
.
ca_file
)
or
''
)
or
None
except
UnicodeError
as
ue
:
raise
SSLUnicodeError
(
0
,
SSLUnicodeError
.
__doc__
,
ue
)
...
...
@@ -91,6 +97,21 @@ class HTTPSClientAuthConnection(httplib.HTTPSConnection):
ca_certs
=
self
.
ca_file
,
cert_reqs
=
ssl
.
CERT_REQUIRED
)
except
UnicodeError
as
ue
:
raise
SSLUnicodeError
(
0
,
SSLUnicodeError
.
__doc__
,
ue
)
except
IOError
as
ioe
:
# In OSX, a wrong SSL credential file may raise an IOError with
# errno of 2, instead of an SSL error
# Wrap it in SSL and raise it
if
getattr
(
ioe
,
'errno'
,
None
)
==
2
:
files
=
self
.
key_file
,
self
.
cert_file
,
self
.
ca_file
if
not
any
(
files
):
raise
SSLCredentialsMissing
(
'No SSL cred. files provided (IOError:%s)'
%
ioe
)
for
f
in
files
:
if
f
and
not
os
.
path
.
exists
(
f
):
raise
SSLCredentialsMissing
(
'SSL cred. file %s does not exist (IOError:%s)'
%
(
f
,
ioe
))
raise
http
.
HTTPConnectionPool
.
_scheme_to_class
[
'https'
]
=
HTTPSClientAuthConnection
...
...
kamaki/version.py
View file @
58fef81d
__version__
=
"0.13.
4
"
__version__
=
"0.13.
5
"
__version_vcs_info__
=
{
'branch'
:
'hotfix-0.13.
4
'
,
'revid'
:
'
9a1c9ba
'
,
'revno'
:
250
3
}
'branch'
:
'hotfix-0.13.
5
'
,
'revid'
:
'
dfe824d
'
,
'revno'
:
250
9
}
__version_user_email__
=
"saxtouri@admin.grnet.gr"
__version_user_name__
=
"Stavros Sachtouris"
version
View file @
58fef81d
0.13.
4
0.13.
5
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment