Merge branch 'feature-threads' into develop

docs/conf.py

@@ -20,11 +20,11 @@ from sys import path, stderr
 import os
 
 try:
-    from objpool import http
-    http
+    from objpool.http import PooledHTTPConnection
+    PooledHTTPConnection
 except ImportError:
     stderr.write("`objpool` package is required to build kamaki docs.\n")
-    #exit()
+    raise
 
 path.insert(0, os.path.join(os.path.abspath(os.path.dirname(__file__)), '..'))
 

docs/developers/code.rst

@@ -121,16 +121,10 @@ utils
 The clients API
 ---------------
 
-Imports
-^^^^^^^
-
-.. toctree::
-    connection
-
 Modules list
 ^^^^^^^^^^^^
 
-compute_rest_api
+compute ReST API
 ^^^^^^^^^^^^^^^^
 
 .. automodule:: kamaki.clients.compute.rest_api
@@ -148,6 +142,14 @@ compute
     :undoc-members:
 
 
+cyclades ReST API
+^^^^^^^^^^^^^^^^^
+
+.. automodule:: kamaki.clients.cyclades.rest_api
+    :members:
+    :show-inheritance:
+    :undoc-members:
+
 cyclades
 ^^^^^^^^
 
@@ -165,24 +167,22 @@ storage
     :show-inheritance:
     :undoc-members:
 
+pithos_rest_api
+^^^^^^^^^^^^^^^
 
-pithos
-^^^^^^
-
-.. automodule:: kamaki.clients.pithos
+.. automodule:: kamaki.clients.pithos.rest_api
     :members:
     :show-inheritance:
     :undoc-members:
 
-pithos_rest_api
-^^^^^^^^^^^^^^^
+pithos
+^^^^^^
 
-.. automodule:: kamaki.clients.pithos.rest_api
+.. automodule:: kamaki.clients.pithos
     :members:
     :show-inheritance:
     :undoc-members:
 
-
 image
 ^^^^^
 

docs/developers/connection.rst

-Connection
-==========
-
-An http connection package with connection pooling.
-
-Since version 0.6 it is safe to use threaded connections.
-
-The Connection package uses httplib, standard python threads and a connection pooling mechanism, which is imported from the *objpool* package.
-
-.. automodule:: kamaki.clients.connection
-    :members:
-    :show-inheritance:
-    :undoc-members:

docs/developers/extending-clients-api.rst

 Extending kamaki.clients
 ========================
 
-By default, kamaki clients are REST clients (they manage HTTP requests and responses to communicate with services). This is achieved by importing the connection module, which is an httplib wrapper.
-
-Connection
-----------
-
-The connection module features an error handling and logging system, a lazy response mechanism, a pooling mechanism as well as concurrency control for thread-demanding client functions (e.g. store upload).
+By default, kamaki clients are REST clients (they manage HTTP requests and responses to communicate with services).
 
 How to build a client
 ---------------------

docs/index.rst

@@ -10,7 +10,7 @@ Kamaki project documentation
 
 ./kamaki is a simple, yet intuitive, multipurpose, interactive command-line tool and client API for managing clouds.
 
-As a develpment API is an initial implementation of the OpenStack Compute API, v1.1, with custom extensions specific to the `Synnefo IaaS <http://synnefo.org/>`_ cloud management software.
+As a develpment API is an initial implementation of an OpenStack inspired API designed for the `Synnefo IaaS <http://www.synnefo.org/>`_ cloud management software.
 
 ./kamaki is open source and released under a 2-clause BSD License.
 

docs/setup.rst

@@ -156,8 +156,30 @@ The [global] group is treated by kamaki as a generic group for arbitrary options
 * history.file <history file path>
     the path of a simple file for inter-session kamaki history. Make sure kamaki is executed in a context where this file is accessible for reading and writing. Kamaki automatically creates the file if it doesn't exist
 
-Hidden features
-^^^^^^^^^^^^^^^
+Additional features
+^^^^^^^^^^^^^^^^^^^
+
+Richer connection logs
+""""""""""""""""""""""
+
+Kamaki logs down the http requests and responses in /var/log/kamaki/clients.log (make sure it is accessible). The request and response data and user authentication information is excluded from the logs be default. The former may render the logs unreadable and the later are sensitive information. Users my activate data and / or token logging my setting the global options log_data and log_token respectively::
+
+    $ kamaki config set log_data on
+    $ kamaki config set log_token on
+
+Either or both of these options may be switched off either by setting them to ``off`` or by deleting them.
+
+    $ kamaki config set log_data off
+    $ kamaki config delete log_token
+
+Set custom thread limit
+"""""""""""""""""""""""
+
+Some operations (e.g. download and upload) may use threaded http connections for better performance. Kamaki.clients utilizes a sophisticated mechanism for dynamically adjusting the number of simultaneous threads running, but users may wish to enforce their own upper thread limit. In that case, the max_threads option may be set to the configuration file::
+
+    $ kamaki config set max_threads 3
+
+If the value is not a positive integer, kamaki will ignore it and a warning message will be logged.
 
 The livetest suite
 """"""""""""""""""

kamaki/cli/commands/__init__.py

@@ -54,7 +54,20 @@ class _command_init(object):
             self.client.LOG_TOKEN, self.client.LOG_DATA = (
                 self['config'].get('global', 'log_token') == 'on',
                 self['config'].get('global', 'log_data') == 'on')
-        except:
+        except Exception as e:
+            sendlog.warning('Failed to read custom log settings: %s' % e)
+            sendlog.warning('\tdefaults for token and data logging are off')
+            pass
+
+    def _update_max_threads(self):
+        try:
+            max_threads = int(self['config'].get('global', 'max_threads'))
+            assert max_threads > 0
+            self.client.MAX_THREADS = max_threads
+        except Exception as e:
+            sendlog.warning('Failed to read custom thread settings: %s' % e)
+            sendlog.warning(
+                '\tdefault for max threads is %s' % self.client.MAX_THREADS)
             pass
 
     def _safe_progress_bar(self, msg, arg='progress_bar'):

kamaki/cli/commands/astakos_cli.py

@@ -52,6 +52,7 @@ class _astakos_init(_command_init):
             or self.config.get('global', 'url')
         self.client = AstakosClient(base_url=base_url, token=token)
         self._update_low_level_log()
+        self._update_max_threads()
 
     def main(self):
         self._run()

kamaki/cli/commands/cyclades_cli.py

@@ -73,6 +73,7 @@ class _init_cyclades(_command_init):
             or self.config.get('global', 'url')
         self.client = CycladesClient(base_url=base_url, token=token)
         self._update_low_level_log()
+        self._update_max_threads()
 
     def main(self):
         self._run()

kamaki/cli/commands/image_cli.py

@@ -60,6 +60,7 @@ class _init_image(_command_init):
             or self.config.get('global', 'url')
         self.client = ImageClient(base_url=base_url, token=token)
         self._update_low_level_log()
+        self._update_max_threads()
 
     def main(self):
         self._run()

kamaki/cli/commands/pithos_cli.py

@@ -174,6 +174,7 @@ class _pithos_init(_command_init):
             account=self.account,
             container=self.container)
         self._update_low_level_log()
+        self._update_max_threads()
 
     def main(self):
         self._run()

kamaki/cli/config.py

@@ -57,7 +57,10 @@ DEFAULTS = {
     'global': {
         'colors': 'off',
         'account':  '',
-        'token': ''
+        'token': '',
+        'log_token': 'off',
+        'log_data': 'off',
+        'max_threads': 7
     },
     'config': {
         'cli': 'config_cli',