Merge branch 'develop' into feature-windows

docs/conf.py

@@ -19,13 +19,13 @@
 from sys import path, stderr
 import os
 
-path.append('/home/saxtouri/src/synnefo/snf-common')
+path.append('/home/saxtouri/src/objpool')
 try:
-    from synnefo import lib
-    lib
+    from objpool import http
+    http
 except ImportError:
-    stderr.write("`snf-common` package is required to build kamaki docs.\n")
-    exit()
+    stderr.write("`objpool` package is required to build kamaki docs.\n")
+    #exit()
 
 path.append(os.path.join(os.path.abspath(os.path.dirname(__file__)), '..'))
 

docs/developers/connection.rst

@@ -3,9 +3,11 @@ Connection
 
 An http connection package with connection pooling.
 
-In version 0.6 and on it is safe to use threaded connections.
+Since version 0.6 it is safe to use threaded connections.
 
-It uses httplib and GRNet Synnefo snf-common packages. Pooling parameters are configures in the external snf-common package.
+The Connection package uses httplib, standard python threads and a connection pooling mechanism.
+
+.. note:: in versions 0.6.0 to 0.6.1 the GRNet Synnefo *snf-common* package is used for its connection pooling module. Since version 0.6.2 the underlying pooling mechanism is packed in a new GRNet package called *objpool*, which is now used instead of snf-common.
 
 .. automodule:: kamaki.clients.connection
     :members:

docs/developers/extending-clients-api.rst

 Extending kamaki.clients
 ========================
 
-By default, kamaki clients are REST clients (they manage HTTP requests and responses to communicate with services). This is achieved by importing the connection module, which is an httplib rapper.
+By default, kamaki clients are REST clients (they manage HTTP requests and responses to communicate with services). This is achieved by importing the connection module, which is an httplib wrapper.
 
 Connection
 ----------

docs/installation.rst

@@ -40,15 +40,28 @@ With virtualenv users can setup kamaki and synnefo services in a sandbox environ
 
 A more detailed example of using virtual env can be found at the `snf-image-creator setup guide <http://docs.dev.grnet.gr/snf-image-creator/latest/install.html#python-virtual-environment>`_
 
-2. Install snf-common
-"""""""""""""""""""""
+2. Install objpool (was: snf-common)
+""""""""""""""""""""""""""""""""""""""""""
 
-Package snf-common is part of the synnefo project and is a kamaki dependency since version 0.6.0.
+.. note:: **versions 0.6.0 - 0.6.1**
+
+    Package snf-common is part of the synnefo project and used to be a kamaki dependency in versions from 0.6.0 to 0.6.1 to provide a connection pooling mechanism. Users who still run 0.6.0 or 0.6.1 may need to manually install the snf-common package:
+
+    .. code-block:: console
+
+        $ git clone http://code.grnet.gr/git/synnefo
+        $ cd synnefo/snf-common
+        $ ./setup build install
+        $ cd -
+
+**Version 0.6.2 and on:**
+
+Since 0.6.2, kamaki is based on objpool (hence the snf-common dependency is now obsolete). The objpool package is easy to install from source, even on windows platforms:
 
 .. code-block:: console
 
-    $ git clone http://code.grnet.gr/git/synnefo
-    $ cd synnefo/snf-common
+    $ git clone http://code.grnet.gr/git/objpool
+    $ cd objpool
     $ ./setup build install
     $ cd -
 
@@ -103,7 +116,13 @@ The following steps describe a command-line approach, but any graphic package ma
 3. Install kamaki
 """""""""""""""""
 
-Since version 0.6.0, the package snf-common (available at synnefo apt repository) will be automatically installed as a dependency.
+.. note:: **versions 0.6.0 - 0.6.1:**
+
+    The *snf-common* package (available at synnefo apt repository) will be automatically installed as a dependency.
+
+.. note:: **versions 0.6.2 and on:**
+
+    Since version 0.6.2, *objpool* replaces *snf-common*. The objpool package is also available at synnefo repository and is automatically installed as a dependency. The *snf-common* dependency is removed.
 
 .. code-block:: console
 

kamaki/cli/__init__.py

@@ -36,8 +36,7 @@ from sys import argv, exit, stdout
 from os.path import basename
 from inspect import getargspec
 
-from kamaki.cli.argument import _arguments, parse_known_args, init_parser,\
-    update_arguments
+from kamaki.cli.argument import ArgumentParseManager
 from kamaki.cli.history import History
 from kamaki.cli.utils import print_dict, print_list, red, magenta, yellow
 from kamaki.cli.errors import CLIError
@@ -277,18 +276,19 @@ def _print_subcommands_help(cmd):
 
 def _update_parser_help(parser, cmd):
     global _best_match
-    parser.prog = parser.prog.split('<')[0]
-    parser.prog += ' '.join(_best_match)
+    parser.syntax = parser.syntax.split('<')[0]
+    parser.syntax += ' '.join(_best_match)
 
     if cmd.is_command:
         cls = cmd.get_class()
-        parser.prog += ' ' + cls.syntax
-        arguments = cls().arguments
-        update_arguments(parser, arguments)
+        parser.syntax += ' ' + cls.syntax
+        parser.update_arguments(cls().arguments)
+        # arguments = cls().arguments
+        # update_arguments(parser, arguments)
     else:
-        parser.prog += ' <...>'
+        parser.syntax += ' <...>'
     if cmd.has_description:
-        parser.description = cmd.help
+        parser.parser.description = cmd.help
 
 
 def _print_error_message(cli_err):
@@ -329,41 +329,39 @@ def _exec_cmd(instance, cmd_args, help_method):
     return 1
 
 
-def set_command_param(param, value):
-    if param == 'prefix':
-        pos = 0
-    elif param == 'descedants_depth':
-        pos = 1
-    else:
-        return
+def set_command_params(parameters):
+    """Add a parameters list to a command
+
+    :param paramters: (list of str) a list of parameters
+    """
     global command
     def_params = list(command.func_defaults)
-    def_params[pos] = value
+    def_params[0] = parameters
     command.func_defaults = tuple(def_params)
 
 
-def one_cmd(parser, unparsed, arguments):
-    group = get_command_group(list(unparsed), arguments)
+#def one_cmd(parser, unparsed, arguments):
+def one_cmd(parser):
+    group = get_command_group(list(parser.unparsed), parser.arguments)
     if not group:
-        parser.print_help()
-        _groups_help(arguments)
+        parser.parser.print_help()
+        _groups_help(parser.arguments)
         exit(0)
 
-    set_command_param(
-        'prefix',
-        [term for term in unparsed if not term.startswith('-')]
-    )
+    nonargs = [term for term in parser.unparsed if not term.startswith('-')]
+    set_command_params(nonargs)
+
     global _best_match
     _best_match = []
 
-    spec_module = _load_spec_module(group, arguments, '_commands')
+    spec_module = _load_spec_module(group, parser.arguments, '_commands')
 
     cmd_tree = _get_cmd_tree_from_spec(group, spec_module._commands)
 
     if _best_match:
         cmd = cmd_tree.get_command('_'.join(_best_match))
     else:
-        cmd = _get_best_match_from_cmd_tree(cmd_tree, unparsed)
+        cmd = _get_best_match_from_cmd_tree(cmd_tree, parser.unparsed)
         _best_match = cmd.path.split('_')
     if cmd is None:
         if _debug or _verbose:
@@ -373,16 +371,17 @@ def one_cmd(parser, unparsed, arguments):
     _update_parser_help(parser, cmd)
 
     if _help or not cmd.is_command:
-        parser.print_help()
+        parser.parser.print_help()
         _print_subcommands_help(cmd)
         exit(0)
 
     cls = cmd.get_class()
-    executable = cls(arguments)
-    parsed, unparsed = parse_known_args(parser, executable.arguments)
+    executable = cls(parser.arguments)
+    parser.update_arguments(executable.arguments)
+    #parsed, unparsed = parse_known_args(parser, executable.arguments)
     for term in _best_match:
-        unparsed.remove(term)
-    _exec_cmd(executable, unparsed, parser.print_help)
+        parser.unparsed.remove(term)
+    _exec_cmd(executable, parser.unparsed, parser.parser.print_help)
 
 
 def _load_all_commands(cmd_tree, arguments):
@@ -402,33 +401,33 @@ def _load_all_commands(cmd_tree, arguments):
                 break
 
 
-def run_shell(exe_string, arguments):
+def run_shell(exe_string, parser):
     from command_shell import _init_shell
-    shell = _init_shell(exe_string, arguments)
-    _load_all_commands(shell.cmd_tree, arguments)
-    shell.run(arguments)
+    shell = _init_shell(exe_string, parser)
+    _load_all_commands(shell.cmd_tree, parser.arguments)
+    shell.run(parser)
 
 
 def main():
     try:
         exe = basename(argv[0])
-        parser = init_parser(exe, _arguments)
-        parsed, unparsed = parse_known_args(parser, _arguments)
+        parser = ArgumentParseManager(exe)
 
-        if _arguments['version'].value:
+        if parser.arguments['version'].value:
             exit(0)
 
-        _init_session(_arguments)
+        _init_session(parser.arguments)
 
-        if unparsed:
-            _history = History(_arguments['config'].get('history', 'file'))
+        if parser.unparsed:
+            _history = History(
+                parser.arguments['config'].get('history', 'file'))
             _history.add(' '.join([exe] + argv[1:]))
-            one_cmd(parser, unparsed, _arguments)
+            one_cmd(parser)
         elif _help:
-            parser.print_help()
-            _groups_help(_arguments)
+            parser.parser.print_help()
+            _groups_help(parser.arguments)
         else:
-            run_shell(exe, _arguments)
+            run_shell(exe, parser)
     except CLIError as err:
         if _debug:
             raise err

kamaki/cli/argument.py

@@ -38,7 +38,8 @@ from kamaki.cli.utils import split_input
 from argparse import ArgumentParser, ArgumentError
 
 try:
-    from progress.bar import IncrementalBar
+    from progress.bar import FillingCirclesBar as KamakiProgressBar
+    #  IncrementalBar
 except ImportError:
     # progress not installed - pls, pip install progress
     pass
@@ -285,7 +286,7 @@ class ProgressBarArgument(FlagArgument):
         self.suffix = '%(percent)d%%'
         super(ProgressBarArgument, self).__init__(help, parsed_name, default)
         try:
-            IncrementalBar
+            KamakiProgressBar
         except NameError:
             print('Warning: no progress bar functionality')
 
@@ -303,12 +304,13 @@ class ProgressBarArgument(FlagArgument):
         if self.value:
             return None
         try:
-            self.bar = IncrementalBar()
+            self.bar = KamakiProgressBar()
         except NameError:
             self.value = None
             return self.value
         self.bar.message = message.ljust(message_len)
         self.bar.suffix = '%(percent)d%% - %(eta)ds'
+        self.bar.start()
 
         def progress_gen(n):
             for i in self.bar.iter(range(int(n))):
@@ -351,23 +353,102 @@ Mechanism:
 """
 
 
-def init_parser(exe, arguments):
-    """Create and initialize an ArgumentParser object"""
+class ArgumentParseManager(object):
+    """Manage (initialize and update) an ArgumentParser object"""
+
     parser = ArgumentParser(add_help=False)
-    parser.prog = '%s <cmd_group> [<cmd_subbroup> ...] <cmd>' % exe
-    update_arguments(parser, arguments)
-    return parser
-
-
-def parse_known_args(parser, arguments=None):
-    """Fill in arguments from user input"""
-    parsed, unparsed = parser.parse_known_args()
-    for name, arg in arguments.items():
-        arg.value = getattr(parsed, name, arg.default)
-    newparsed = []
-    for term in unparsed:
-        newparsed += split_input(' \'%s\' ' % term)
-    return parsed, newparsed
+    _arguments = {}
+    _parser_modified = False
+    _parsed = None
+    _unparsed = None
+
+    def __init__(self, exe, arguments=None):
+        """
+        :param exe: (str) the basic command (e.g. 'kamaki')
+
+        :param arguments: (dict) if given, overrides the global _argument as
+            the parsers arguments specification
+        """
+        self.syntax = '%s <cmd_group> [<cmd_subbroup> ...] <cmd>' % exe
+        if arguments:
+            self.arguments = arguments
+        else:
+            global _arguments
+            self.arguments = _arguments
+        self.parse()
+
+    @property
+    def syntax(self):
+        """The command syntax (useful for help messages, descriptions, etc)"""
+        return self.parser.prog
+
+    @syntax.setter
+    def syntax(self, new_syntax):
+        self.parser.prog = new_syntax
+
+    @property
+    def arguments(self):
+        """(dict) arguments the parser should be aware of"""
+        return self._arguments
+
+    @arguments.setter
+    def arguments(self, new_arguments):
+        if new_arguments:
+            assert isinstance(new_arguments, dict)
+        self._arguments = new_arguments
+        self.update_parser()
+
+    @property
+    def parsed(self):
+        """(Namespace) parser-matched terms"""
+        if self._parser_modified:
+            self.parse()
+        return self._parsed
+
+    @property
+    def unparsed(self):
+        """(list) parser-unmatched terms"""
+        if self._parser_modified:
+            self.parse()
+        return self._unparsed
+
+    def update_parser(self, arguments=None):
+        """Load argument specifications to parser
+
+        :param arguments: if not given, update self.arguments instead
+        """
+        if not arguments:
+            arguments = self._arguments
+
+        for name, arg in arguments.items():
+            try:
+                arg.update_parser(self.parser, name)
+                self._parser_modified = True
+            except ArgumentError:
+                pass
+
+    def update_arguments(self, new_arguments):
+        """Add to / update existing arguments
+
+        :param new_arguments: (dict)
+        """
+        if new_arguments:
+            assert isinstance(new_arguments, dict)
+            self._arguments.update(new_arguments)
+            self.update_parser()
+
+    def parse(self, new_args=None):
+        """Do parse user input"""
+        if new_args:
+            self._parsed, unparsed = self.parser.parse_known_args(new_args)
+        else:
+            self._parsed, unparsed = self.parser.parse_known_args()
+        for name, arg in self.arguments.items():
+            arg.value = getattr(self._parsed, name, arg.default)
+        self._unparsed = []
+        for term in unparsed:
+            self._unparsed += split_input(' \'%s\' ' % term)
+        self._parser_modified = False
 
 
 def update_arguments(parser, arguments):

kamaki/cli/command_shell.py

@@ -34,19 +34,16 @@
 from cmd import Cmd
 from os import popen
 from sys import stdout
-from argparse import ArgumentParser
 
 from kamaki.cli import _exec_cmd, _print_error_message
-from kamaki.cli.argument import update_arguments
+from kamaki.cli.argument import ArgumentParseManager
 from kamaki.cli.utils import print_dict, split_input
 from kamaki.cli.history import History
 from kamaki.cli.errors import CLIError
 
 
-def _init_shell(exe_string, arguments):
-    arguments.pop('version', None)
-    arguments.pop('options', None)
-    arguments.pop('history', None)
+def _init_shell(exe_string, parser):
+    parser.arguments.pop('version', None)
     shell = Shell()
     shell.set_prompt(exe_string)
     from kamaki import __version__ as version
@@ -64,9 +61,9 @@ class Shell(Cmd):
     _suffix = ']:'
     cmd_tree = None
     _history = None
-    _arguments = None
     _context_stack = []
     _prompt_stack = []
+    _parser = None
 
     undoc_header = 'interactive shell commands:'
 
@@ -143,7 +140,7 @@ class Shell(Cmd):
 
     def _register_command(self, cmd_path):
         cmd = self.cmd_tree.get_command(cmd_path)
-        arguments = self._arguments
+        arguments = self._parser.arguments
 
         def do_method(new_context, line):
             """ Template for all cmd.Cmd methods of the form do_<cmd name>
@@ -152,27 +149,31 @@ class Shell(Cmd):
                 even if cmd_term_term is not a terminal path
             """
             subcmd, cmd_args = cmd.parse_out(split_input(line))
-            if self._history:
-                self._history.add(' '.join([cmd.path.replace('_', ' '), line]))
-            cmd_parser = ArgumentParser(cmd.name, add_help=False)
-            cmd_parser.description = subcmd.help
+            self._history.add(' '.join([cmd.path.replace('_', ' '), line]))
+            cmd_parser = ArgumentParseManager(
+                cmd.name,
+                dict(self._parser.arguments))
+
+            cmd_parser.parser.description = subcmd.help
 
             # exec command or change context
             if subcmd.is_command:  # exec command
                 cls = subcmd.get_class()
-                instance = cls(dict(arguments))
-                cmd_parser.prog = '%s %s' % (cmd_parser.prog.replace('_', ' '),
-                    cls.syntax)
-                update_arguments(cmd_parser, instance.arguments)
+                instance = cls(dict(cmd_parser.arguments))
+                cmd_parser.syntax = '%s %s' % (
+                    subcmd.path.replace('_', ' '), cls.syntax)
+                cmd_parser.update_arguments(instance.arguments)
                 if '-h' in cmd_args or '--help' in cmd_args:
-                    cmd_parser.print_help()
+                    cmd_parser.parser.print_help()
                     return
-                parsed, unparsed = cmd_parser.parse_known_args(cmd_args)
+                cmd_parser.parse(cmd_args)
 
                 for name, arg in instance.arguments.items():
-                    arg.value = getattr(parsed, name, arg.default)
+                    arg.value = getattr(cmd_parser.parsed, name, arg.default)
                 try:
-                    _exec_cmd(instance, unparsed, cmd_parser.print_help)
+                    _exec_cmd(instance,
+                        cmd_parser.unparsed,
+                        cmd_parser.parser.print_help)
                 except CLIError as err:
                     _print_error_message(err)
             elif ('-h' in cmd_args or '--help' in cmd_args) \
@@ -225,9 +226,10 @@ class Shell(Cmd):
         hdr = tmp_partition[0].strip()
         return '%s commands:' % hdr
 
-    def run(self, arguments, path=''):
-        self._history = History(arguments['config'].get('history', 'file'))
-        self._arguments = arguments
+    def run(self, parser, path=''):
+        self._parser = parser
+        self._history = History(
+            parser.arguments['config'].get('history', 'file'))
         if path:
             cmd = self.cmd_tree.get_command(path)
             intro = cmd.path.replace('_', ' ')

kamaki/cli/commands/cyclades_cli.py

@@ -439,9 +439,9 @@ class server_wait(_init_cyclades):
             progress_bar.finish()
             raiseCLIError(err)
         if new_mode:
-            print('\nServer %s is now in %s mode' % (server_id, new_mode))
+            print('Server %s is now in %s mode' % (server_id, new_mode))
         else:
-            print('\nTime out')
+            print('Time out')
 
 
 @command(flavor_cmds)

kamaki/clients/connection/__init__.py

@@ -33,6 +33,9 @@
 
 
 class HTTPResponse(object):
+    """An abstract HTTP Response object to handle a performed HTTPRequest.
+    Subclass implementation required
+    """
 
     def __init__(self, request=None, prefetched=False):
         self.request = request
@@ -47,11 +50,12 @@ class HTTPResponse(object):
 
     def release(self):
         """Release the connection.
-        Use this after finished using the response"""
+        """
         raise NotImplementedError
 
     @property
     def prefetched(self):
+        """flag to avoid downloading more than nessecary"""
         return self._prefetched
 
     @prefetched.setter
@@ -60,6 +64,7 @@ class HTTPResponse(object):
 
     @property
     def content(self):
+        """(binary) request response content (data)"""
         self._get_response()
         return self._content
 
@@ -69,6 +74,7 @@ class HTTPResponse(object):
 
     @property
     def text(self):
+        """(str)"""
         self._get_response()
         return self._text
 
@@ -78,6 +84,7 @@ class HTTPResponse(object):
 
     @property
     def json(self):
+        """(dict)"""
         self._get_response()
         return self._json
 
@@ -87,6 +94,7 @@ class HTTPResponse(object):
 
     @property
     def headers(self):
+        """(dict)"""
         self._get_response()
         return self._headers
 
@@ -96,6 +104,7 @@ class HTTPResponse(object):
 
     @property
     def status_code(self):
+        """(int) optional"""
         self._get_response()
         return self._status_code
 
@@ -105,6 +114,7 @@ class HTTPResponse(object):
 
     @property
     def status(self):
+        """(str) useful in server error responses"""
         self._get_response()
         return self._status
 
@@ -114,6 +124,7 @@ class HTTPResponse(object):
 
     @property
     def request(self):
+        """(HTTPConnection) the source of this response object"""
         return self._request
 
     @request.setter
@@ -122,6 +133,8 @@ class HTTPResponse(object):
 
 
 class HTTPConnection(object):
+    """An abstract HTTP Connection mechanism. Subclass implementation required
+    """
 
     def __init__(self, method=None, url=None, params={}, headers={}):
         self.headers = headers

kamaki/clients/connection/kamakicon.py

@@ -71,8 +71,11 @@ class KamakiHTTPResponse(HTTPResponse):
 
     @property
     def text(self):
+        """
+        :returns: (str) content
+        """
         self._get_response()
-        return self._content
+        return unicode(self._content)
 
     @text.setter
     def test(self, v):
@@ -80,6 +83,11 @@ class KamakiHTTPResponse(HTTPResponse):