Commit b5b9534a authored by Stavros Sachtouris's avatar Stavros Sachtouris
Browse files

Merge branch 'develop' into feature-logging

parents b3bb083f 6287a048
......@@ -20,11 +20,11 @@ from sys import path, stderr
import os
try:
from objpool import http
http
from objpool.http import PooledHTTPConnection
PooledHTTPConnection
except ImportError:
stderr.write("`objpool` package is required to build kamaki docs.\n")
#exit()
raise
path.insert(0, os.path.join(os.path.abspath(os.path.dirname(__file__)), '..'))
......
......@@ -121,16 +121,10 @@ utils
The clients API
---------------
Imports
^^^^^^^
.. toctree::
connection
Modules list
^^^^^^^^^^^^
compute_rest_api
compute ReST API
^^^^^^^^^^^^^^^^
.. automodule:: kamaki.clients.compute.rest_api
......@@ -148,6 +142,14 @@ compute
:undoc-members:
cyclades ReST API
^^^^^^^^^^^^^^^^^
.. automodule:: kamaki.clients.cyclades.rest_api
:members:
:show-inheritance:
:undoc-members:
cyclades
^^^^^^^^
......@@ -165,24 +167,22 @@ storage
:show-inheritance:
:undoc-members:
pithos_rest_api
^^^^^^^^^^^^^^^
pithos
^^^^^^
.. automodule:: kamaki.clients.pithos
.. automodule:: kamaki.clients.pithos.rest_api
:members:
:show-inheritance:
:undoc-members:
pithos_rest_api
^^^^^^^^^^^^^^^
pithos
^^^^^^
.. automodule:: kamaki.clients.pithos.rest_api
.. automodule:: kamaki.clients.pithos
:members:
:show-inheritance:
:undoc-members:
image
^^^^^
......
Connection
==========
An http connection package with connection pooling.
Since version 0.6 it is safe to use threaded connections.
The Connection package uses httplib, standard python threads and a connection pooling mechanism, which is imported from the *objpool* package.
.. automodule:: kamaki.clients.connection
:members:
:show-inheritance:
:undoc-members:
Extending kamaki.clients
========================
By default, kamaki clients are REST clients (they manage HTTP requests and responses to communicate with services). This is achieved by importing the connection module, which is an httplib wrapper.
Connection
----------
The connection module features an error handling and logging system, a lazy response mechanism, a pooling mechanism as well as concurrency control for thread-demanding client functions (e.g. store upload).
By default, kamaki clients are REST clients (they manage HTTP requests and responses to communicate with services).
How to build a client
---------------------
......
......@@ -10,7 +10,7 @@ Kamaki project documentation
./kamaki is a simple, yet intuitive, multipurpose, interactive command-line tool and client API for managing clouds.
As a develpment API is an initial implementation of the OpenStack Compute API, v1.1, with custom extensions specific to the `Synnefo IaaS <http://synnefo.org/>`_ cloud management software.
As a develpment API is an initial implementation of an OpenStack inspired API designed for the `Synnefo IaaS <http://www.synnefo.org/>`_ cloud management software.
./kamaki is open source and released under a 2-clause BSD License.
......
......@@ -156,8 +156,30 @@ The [global] group is treated by kamaki as a generic group for arbitrary options
* history.file <history file path>
the path of a simple file for inter-session kamaki history. Make sure kamaki is executed in a context where this file is accessible for reading and writing. Kamaki automatically creates the file if it doesn't exist
Hidden features
^^^^^^^^^^^^^^^
Additional features
^^^^^^^^^^^^^^^^^^^
Richer connection logs
""""""""""""""""""""""
Kamaki logs down the http requests and responses in /var/log/kamaki/clients.log (make sure it is accessible). The request and response data and user authentication information is excluded from the logs be default. The former may render the logs unreadable and the later are sensitive information. Users my activate data and / or token logging my setting the global options log_data and log_token respectively::
$ kamaki config set log_data on
$ kamaki config set log_token on
Either or both of these options may be switched off either by setting them to ``off`` or by deleting them.
$ kamaki config set log_data off
$ kamaki config delete log_token
Set custom thread limit
"""""""""""""""""""""""
Some operations (e.g. download and upload) may use threaded http connections for better performance. Kamaki.clients utilizes a sophisticated mechanism for dynamically adjusting the number of simultaneous threads running, but users may wish to enforce their own upper thread limit. In that case, the max_threads option may be set to the configuration file::
$ kamaki config set max_threads 3
If the value is not a positive integer, kamaki will ignore it and a warning message will be logged.
The livetest suite
""""""""""""""""""
......
......@@ -17,7 +17,7 @@ It is essential for users to get a valid configuration token that works with for
Example 1.1: Set user token to myt0k3n==
$ kamaki set token myt0k3n==
$ kamaki config set token myt0k3n==
Shell vs one-command
--------------------
......
......@@ -54,7 +54,20 @@ class _command_init(object):
self.client.LOG_TOKEN, self.client.LOG_DATA = (
self['config'].get('global', 'log_token') == 'on',
self['config'].get('global', 'log_data') == 'on')
except:
except Exception as e:
sendlog.warning('Failed to read custom log settings: %s' % e)
sendlog.warning('\tdefaults for token and data logging are off')
pass
def _update_max_threads(self):
try:
max_threads = int(self['config'].get('global', 'max_threads'))
assert max_threads > 0
self.client.MAX_THREADS = max_threads
except Exception as e:
sendlog.warning('Failed to read custom thread settings: %s' % e)
sendlog.warning(
'\tdefault for max threads is %s' % self.client.MAX_THREADS)
pass
def _safe_progress_bar(self, msg, arg='progress_bar'):
......
......@@ -52,6 +52,7 @@ class _astakos_init(_command_init):
or self.config.get('global', 'url')
self.client = AstakosClient(base_url=base_url, token=token)
self._update_low_level_log()
self._update_max_threads()
def main(self):
self._run()
......
......@@ -73,6 +73,7 @@ class _init_cyclades(_command_init):
or self.config.get('global', 'url')
self.client = CycladesClient(base_url=base_url, token=token)
self._update_low_level_log()
self._update_max_threads()
def main(self):
self._run()
......
......@@ -230,7 +230,7 @@ class cyclades(object):
except ClientError as ce:
if network_id and ce.status == 400:
msg = 'Network with id %s does not exist' % network_id,
raiseCLIError(ce, msg, details=self.about_network_id)
raiseCLIError(ce, msg, details=this.about_network_id)
elif network_id or ce.status == 421:
msg = 'Network with id %s is in use' % network_id,
raiseCLIError(ce, msg, details=[
......
......@@ -60,6 +60,7 @@ class _init_image(_command_init):
or self.config.get('global', 'url')
self.client = ImageClient(base_url=base_url, token=token)
self._update_low_level_log()
self._update_max_threads()
def main(self):
self._run()
......@@ -201,7 +202,8 @@ class image_register(_init_image):
if self['update']:
self.client.reregister(location, name, params, properties)
else:
self.client.register(name, location, params, properties)
r = self.client.register(name, location, params, properties)
print_dict(r)
def main(self, name, location):
super(self.__class__, self)._run()
......
......@@ -174,6 +174,7 @@ class _pithos_init(_command_init):
account=self.account,
container=self.container)
self._update_low_level_log()
self._update_max_threads()
def main(self):
self._run()
......@@ -330,7 +331,6 @@ class store_list(_store_container_command):
'format to parse until data (default: d/m/Y H:M:S )',
'--format'),
shared=FlagArgument('show only shared', '--shared'),
public=FlagArgument('show only public', '--public'),
more=FlagArgument(
'output results in pages (-n to set items per page, default 10)',
'--more'),
......@@ -1312,16 +1312,22 @@ class store_download(_store_container_command):
if_modified_since=self['if_modified_since'],
if_unmodified_since=self['if_unmodified_since'])
except KeyboardInterrupt:
from threading import enumerate as activethreads
stdout.write('\nFinishing active threads ')
for thread in activethreads():
from threading import activeCount, enumerate as activethreads
timeout = 0.5
while activeCount() > 1:
stdout.write('\nCancel %s threads: ' % (activeCount() - 1))
stdout.flush()
try:
thread.join()
stdout.write('.')
except RuntimeError:
continue
print('\ndownload canceled by user')
for thread in activethreads():
try:
thread.join(timeout)
stdout.write('.' if thread.isAlive() else '*')
except RuntimeError:
continue
finally:
stdout.flush()
timeout += 0.1
print('\nDownload canceled by user')
if local_path is not None:
print('to resume, re-run with --resume')
except Exception:
......
......@@ -57,7 +57,10 @@ DEFAULTS = {
'global': {
'colors': 'off',
'account': '',
'token': ''
'token': '',
'log_token': 'off',
'log_data': 'off',
'max_threads': 7
},
'config': {
'cli': 'config_cli',
......
......@@ -404,7 +404,7 @@ class Client(object):
if data:
headers.setdefault('Content-Length', '%s' % len(data))
sendlog.debug('COMMIT %s @ %s\t[%s]', method, self.base_url, self)
sendlog.debug('\n\nCMT %s@%s\t[%s]', method, self.base_url, self)
req = RequestManager(
method, self.base_url, path,
data=data, headers=headers, params=params)
......
......@@ -80,11 +80,10 @@ class ComputeClient(ComputeRestClient):
'imageRef': image_id}}
image = self.get_image_details(image_id)
img_meta = image['metadata']['values']
metadata = {}
for key in ('os', 'users'):
try:
metadata[key] = img_meta[key]
metadata[key] = image['metadata']['values'][key]
except KeyError:
pass
if metadata:
......
......@@ -200,7 +200,7 @@ class CycladesClient(CycladesRestClient):
if err.status == 421:
err.details = [
'Network may be still connected to at least one server']
raise err
raise
def connect_server(self, server_id, network_id):
""" Connect a server to a network
......
......@@ -32,7 +32,7 @@
# or implied, of GRNET S.A.
from kamaki.clients import Client, ClientError
from kamaki.clients.utils import path4url
from kamaki.clients.utils import path4url, filter_in
class ImageClient(Client):
......@@ -109,6 +109,8 @@ class ImageClient(Client):
disc_format, container_format, size, checksum, is_public, owner
:param properties: (dict) image properties (X-Image-Meta-Property)
:returns: (dict) details of the created image
"""
path = path4url('images') + '/'
self.set_header('X-Image-Meta-Name', name)
......@@ -124,7 +126,8 @@ class ImageClient(Client):
for key, val in properties.items():
async_headers['x-image-meta-property-%s' % key] = val
self.post(path, success=200, async_headers=async_headers)
r = self.post(path, success=200, async_headers=async_headers)
return filter_in(r.headers, 'X-Image-')
def list_members(self, image_id):
"""
......
......@@ -35,6 +35,22 @@ from mock import patch, call
from unittest import TestCase
from itertools import product
example_image_headers = {
'x-image-meta-id': '3edd4d15-41b4-4a39-9601-015ef56b3bb3',
'x-image-meta-checksum': 'df23837c30889252c0aed80b6f770a53a86',
'x-image-meta-container-format': 'bare',
'x-image-meta-location': 'pithos://a13528163db/con/obj_13.0',
'x-image-meta-disk-format': 'diskdump',
'x-image-meta-is-public': 'True',
'x-image-meta-status': 'available',
'x-image-meta-deleted-at': '',
'x-image-meta-updated-at': '2013-04-11 15:22:39',
'x-image-meta-created-at': '2013-04-11 15:22:37',
'x-image-meta-owner': 'a13529bb3c3db',
'x-image-meta-size': '1073741824',
'x-image-meta-name': 'img_1365686546.0',
'extraheaders': 'should be ignored'
}
example_images = [
{
"status": "available",
......@@ -185,6 +201,7 @@ class ImageClient(TestCase):
@patch('%s.post' % image_pkg, return_value=FR())
def test_register(self, post, SH):
img0 = example_images_detailed[0]
FR.headers = example_image_headers
img0_location = img0['location']
img0_name = 'A new img0 name'
prfx = 'x-image-meta-'
......@@ -206,9 +223,12 @@ class ImageClient(TestCase):
async_headers['%s%s' % (prfx, k)] = args[i]
props['%s%s' % (proprfx, args[i])] = k
async_headers.update(props)
self.client.register(
r = self.client.register(
img0_name, img0_location,
params=params, properties=props)
expectedict = dict(example_image_headers)
expectedict.pop('extraheaders')
self.assert_dicts_are_equal(expectedict, r)
self.assertEqual(
post.mock_calls[-1],
call('/images/', async_headers=async_headers, success=200))
......
......@@ -39,6 +39,12 @@ from kamaki.clients.image import ImageClient
from kamaki.clients import ClientError
IMGMETA = set([
'id', 'name', 'checksum', 'container-format', 'location', 'disk-format',
'is-public', 'status', 'deleted-at', 'updated-at', 'created-at', 'owner',
'size'])
class Image(livetest.Generic):
def setUp(self):
self.now = time.mktime(time.gmtime())
......@@ -49,6 +55,7 @@ class Image(livetest.Generic):
cyclades_url = self['compute', 'url']
self.cyclades = CycladesClient(cyclades_url, self['token'])
self._imglist = {}
self._imgdetails = {}
def test_000(self):
self._prepare_img()
......@@ -73,12 +80,13 @@ class Image(livetest.Generic):
print('\t- ok')
f.close()
self.client.register(
r = self.client.register(
self.imgname,
self.location,
params=dict(is_public=True))
img = self._get_img_by_name(self.imgname)
self._imglist[self.imgname] = img
self._imglist[self.imgname] = dict(
name=r['x-image-meta-name'], id=r['x-image-meta-id'])
self._imgdetails[self.imgname] = r
def tearDown(self):
for img in self._imglist.values():
......@@ -200,6 +208,9 @@ class Image(livetest.Generic):
self.assertTrue(self._imglist)
for img in self._imglist.values():
self.assertTrue(img is not None)
r = set(self._imgdetails[img['name']].keys())
self.assertTrue(
r.issubset(['x-image-meta-%s' % k for k in IMGMETA]))
def test_set_members(self):
"""Test set_members"""
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment