Commit d56d6f9f authored by Dionysis Zindros's avatar Dionysis Zindros
Browse files

Merge branch 'develop' into feature-windows

parents 7bc8c011 ee328362
......@@ -19,13 +19,13 @@
from sys import path, stderr
import os
from synnefo import lib
from objpool import http
except ImportError:
stderr.write("`snf-common` package is required to build kamaki docs.\n")
stderr.write("`objpool` package is required to build kamaki docs.\n")
path.append(os.path.join(os.path.abspath(os.path.dirname(__file__)), '..'))
......@@ -3,9 +3,11 @@ Connection
An http connection package with connection pooling.
In version 0.6 and on it is safe to use threaded connections.
Since version 0.6 it is safe to use threaded connections.
It uses httplib and GRNet Synnefo snf-common packages. Pooling parameters are configures in the external snf-common package.
The Connection package uses httplib, standard python threads and a connection pooling mechanism.
.. note:: in versions 0.6.0 to 0.6.1 the GRNet Synnefo *snf-common* package is used for its connection pooling module. Since version 0.6.2 the underlying pooling mechanism is packed in a new GRNet package called *objpool*, which is now used instead of snf-common.
.. automodule:: kamaki.clients.connection
Extending kamaki.clients
By default, kamaki clients are REST clients (they manage HTTP requests and responses to communicate with services). This is achieved by importing the connection module, which is an httplib rapper.
By default, kamaki clients are REST clients (they manage HTTP requests and responses to communicate with services). This is achieved by importing the connection module, which is an httplib wrapper.
......@@ -40,15 +40,28 @@ With virtualenv users can setup kamaki and synnefo services in a sandbox environ
A more detailed example of using virtual env can be found at the `snf-image-creator setup guide <>`_
2. Install snf-common
2. Install objpool (was: snf-common)
Package snf-common is part of the synnefo project and is a kamaki dependency since version 0.6.0.
.. note:: **versions 0.6.0 - 0.6.1**
Package snf-common is part of the synnefo project and used to be a kamaki dependency in versions from 0.6.0 to 0.6.1 to provide a connection pooling mechanism. Users who still run 0.6.0 or 0.6.1 may need to manually install the snf-common package:
.. code-block:: console
$ git clone
$ cd synnefo/snf-common
$ ./setup build install
$ cd -
**Version 0.6.2 and on:**
Since 0.6.2, kamaki is based on objpool (hence the snf-common dependency is now obsolete). The objpool package is easy to install from source, even on windows platforms:
.. code-block:: console
$ git clone
$ cd synnefo/snf-common
$ git clone
$ cd objpool
$ ./setup build install
$ cd -
......@@ -103,7 +116,13 @@ The following steps describe a command-line approach, but any graphic package ma
3. Install kamaki
Since version 0.6.0, the package snf-common (available at synnefo apt repository) will be automatically installed as a dependency.
.. note:: **versions 0.6.0 - 0.6.1:**
The *snf-common* package (available at synnefo apt repository) will be automatically installed as a dependency.
.. note:: **versions 0.6.2 and on:**
Since version 0.6.2, *objpool* replaces *snf-common*. The objpool package is also available at synnefo repository and is automatically installed as a dependency. The *snf-common* dependency is removed.
.. code-block:: console
......@@ -36,8 +36,7 @@ from sys import argv, exit, stdout
from os.path import basename
from inspect import getargspec
from kamaki.cli.argument import _arguments, parse_known_args, init_parser,\
from kamaki.cli.argument import ArgumentParseManager
from kamaki.cli.history import History
from kamaki.cli.utils import print_dict, print_list, red, magenta, yellow
from kamaki.cli.errors import CLIError
......@@ -277,18 +276,19 @@ def _print_subcommands_help(cmd):
def _update_parser_help(parser, cmd):
global _best_match
parser.prog = parser.prog.split('<')[0]
parser.prog += ' '.join(_best_match)
parser.syntax = parser.syntax.split('<')[0]
parser.syntax += ' '.join(_best_match)
if cmd.is_command:
cls = cmd.get_class()
parser.prog += ' ' + cls.syntax
arguments = cls().arguments
update_arguments(parser, arguments)
parser.syntax += ' ' + cls.syntax
# arguments = cls().arguments
# update_arguments(parser, arguments)
parser.prog += ' <...>'
parser.syntax += ' <...>'
if cmd.has_description:
parser.description =
parser.parser.description =
def _print_error_message(cli_err):
......@@ -329,41 +329,39 @@ def _exec_cmd(instance, cmd_args, help_method):
return 1
def set_command_param(param, value):
if param == 'prefix':
pos = 0
elif param == 'descedants_depth':
pos = 1
def set_command_params(parameters):
"""Add a parameters list to a command
:param paramters: (list of str) a list of parameters
global command
def_params = list(command.func_defaults)
def_params[pos] = value
def_params[0] = parameters
command.func_defaults = tuple(def_params)
def one_cmd(parser, unparsed, arguments):
group = get_command_group(list(unparsed), arguments)
#def one_cmd(parser, unparsed, arguments):
def one_cmd(parser):
group = get_command_group(list(parser.unparsed), parser.arguments)
if not group:
[term for term in unparsed if not term.startswith('-')]
nonargs = [term for term in parser.unparsed if not term.startswith('-')]
global _best_match
_best_match = []
spec_module = _load_spec_module(group, arguments, '_commands')
spec_module = _load_spec_module(group, parser.arguments, '_commands')
cmd_tree = _get_cmd_tree_from_spec(group, spec_module._commands)
if _best_match:
cmd = cmd_tree.get_command('_'.join(_best_match))
cmd = _get_best_match_from_cmd_tree(cmd_tree, unparsed)
cmd = _get_best_match_from_cmd_tree(cmd_tree, parser.unparsed)
_best_match = cmd.path.split('_')
if cmd is None:
if _debug or _verbose:
......@@ -373,16 +371,17 @@ def one_cmd(parser, unparsed, arguments):
_update_parser_help(parser, cmd)
if _help or not cmd.is_command:
cls = cmd.get_class()
executable = cls(arguments)
parsed, unparsed = parse_known_args(parser, executable.arguments)
executable = cls(parser.arguments)
#parsed, unparsed = parse_known_args(parser, executable.arguments)
for term in _best_match:
_exec_cmd(executable, unparsed, parser.print_help)
_exec_cmd(executable, parser.unparsed, parser.parser.print_help)
def _load_all_commands(cmd_tree, arguments):
......@@ -402,33 +401,33 @@ def _load_all_commands(cmd_tree, arguments):
def run_shell(exe_string, arguments):
def run_shell(exe_string, parser):
from command_shell import _init_shell
shell = _init_shell(exe_string, arguments)
_load_all_commands(shell.cmd_tree, arguments)
shell = _init_shell(exe_string, parser)
_load_all_commands(shell.cmd_tree, parser.arguments)
def main():
exe = basename(argv[0])
parser = init_parser(exe, _arguments)
parsed, unparsed = parse_known_args(parser, _arguments)
parser = ArgumentParseManager(exe)
if _arguments['version'].value:
if parser.arguments['version'].value:
if unparsed:
_history = History(_arguments['config'].get('history', 'file'))
if parser.unparsed:
_history = History(
parser.arguments['config'].get('history', 'file'))
_history.add(' '.join([exe] + argv[1:]))
one_cmd(parser, unparsed, _arguments)
elif _help:
run_shell(exe, _arguments)
run_shell(exe, parser)
except CLIError as err:
if _debug:
raise err
......@@ -38,7 +38,8 @@ from kamaki.cli.utils import split_input
from argparse import ArgumentParser, ArgumentError
from import IncrementalBar
from import FillingCirclesBar as KamakiProgressBar
# IncrementalBar
except ImportError:
# progress not installed - pls, pip install progress
......@@ -285,7 +286,7 @@ class ProgressBarArgument(FlagArgument):
self.suffix = '%(percent)d%%'
super(ProgressBarArgument, self).__init__(help, parsed_name, default)
except NameError:
print('Warning: no progress bar functionality')
......@@ -303,12 +304,13 @@ class ProgressBarArgument(FlagArgument):
if self.value:
return None
try: = IncrementalBar() = KamakiProgressBar()
except NameError:
self.value = None
return self.value = message.ljust(message_len) = '%(percent)d%% - %(eta)ds'
def progress_gen(n):
for i in
......@@ -351,23 +353,102 @@ Mechanism:
def init_parser(exe, arguments):
"""Create and initialize an ArgumentParser object"""
class ArgumentParseManager(object):
"""Manage (initialize and update) an ArgumentParser object"""
parser = ArgumentParser(add_help=False)
parser.prog = '%s <cmd_group> [<cmd_subbroup> ...] <cmd>' % exe
update_arguments(parser, arguments)
return parser
def parse_known_args(parser, arguments=None):
"""Fill in arguments from user input"""
parsed, unparsed = parser.parse_known_args()
for name, arg in arguments.items():
arg.value = getattr(parsed, name, arg.default)
newparsed = []
for term in unparsed:
newparsed += split_input(' \'%s\' ' % term)
return parsed, newparsed
_arguments = {}
_parser_modified = False
_parsed = None
_unparsed = None
def __init__(self, exe, arguments=None):
:param exe: (str) the basic command (e.g. 'kamaki')
:param arguments: (dict) if given, overrides the global _argument as
the parsers arguments specification
self.syntax = '%s <cmd_group> [<cmd_subbroup> ...] <cmd>' % exe
if arguments:
self.arguments = arguments
global _arguments
self.arguments = _arguments
def syntax(self):
"""The command syntax (useful for help messages, descriptions, etc)"""
return self.parser.prog
def syntax(self, new_syntax):
self.parser.prog = new_syntax
def arguments(self):
"""(dict) arguments the parser should be aware of"""
return self._arguments
def arguments(self, new_arguments):
if new_arguments:
assert isinstance(new_arguments, dict)
self._arguments = new_arguments
def parsed(self):
"""(Namespace) parser-matched terms"""
if self._parser_modified:
return self._parsed
def unparsed(self):
"""(list) parser-unmatched terms"""
if self._parser_modified:
return self._unparsed
def update_parser(self, arguments=None):
"""Load argument specifications to parser
:param arguments: if not given, update self.arguments instead
if not arguments:
arguments = self._arguments
for name, arg in arguments.items():
arg.update_parser(self.parser, name)
self._parser_modified = True
except ArgumentError:
def update_arguments(self, new_arguments):
"""Add to / update existing arguments
:param new_arguments: (dict)
if new_arguments:
assert isinstance(new_arguments, dict)
def parse(self, new_args=None):
"""Do parse user input"""
if new_args:
self._parsed, unparsed = self.parser.parse_known_args(new_args)
self._parsed, unparsed = self.parser.parse_known_args()
for name, arg in self.arguments.items():
arg.value = getattr(self._parsed, name, arg.default)
self._unparsed = []
for term in unparsed:
self._unparsed += split_input(' \'%s\' ' % term)
self._parser_modified = False
def update_arguments(parser, arguments):
......@@ -34,19 +34,16 @@
from cmd import Cmd
from os import popen
from sys import stdout
from argparse import ArgumentParser
from kamaki.cli import _exec_cmd, _print_error_message
from kamaki.cli.argument import update_arguments
from kamaki.cli.argument import ArgumentParseManager
from kamaki.cli.utils import print_dict, split_input
from kamaki.cli.history import History
from kamaki.cli.errors import CLIError
def _init_shell(exe_string, arguments):
arguments.pop('version', None)
arguments.pop('options', None)
arguments.pop('history', None)
def _init_shell(exe_string, parser):
parser.arguments.pop('version', None)
shell = Shell()
from kamaki import __version__ as version
......@@ -64,9 +61,9 @@ class Shell(Cmd):
_suffix = ']:'
cmd_tree = None
_history = None
_arguments = None
_context_stack = []
_prompt_stack = []
_parser = None
undoc_header = 'interactive shell commands:'
......@@ -143,7 +140,7 @@ class Shell(Cmd):
def _register_command(self, cmd_path):
cmd = self.cmd_tree.get_command(cmd_path)
arguments = self._arguments
arguments = self._parser.arguments
def do_method(new_context, line):
""" Template for all cmd.Cmd methods of the form do_<cmd name>
......@@ -152,27 +149,31 @@ class Shell(Cmd):
even if cmd_term_term is not a terminal path
subcmd, cmd_args = cmd.parse_out(split_input(line))
if self._history:
self._history.add(' '.join([cmd.path.replace('_', ' '), line]))
cmd_parser = ArgumentParser(, add_help=False)
cmd_parser.description =
self._history.add(' '.join([cmd.path.replace('_', ' '), line]))
cmd_parser = ArgumentParseManager(,
cmd_parser.parser.description =
# exec command or change context
if subcmd.is_command: # exec command
cls = subcmd.get_class()
instance = cls(dict(arguments))
cmd_parser.prog = '%s %s' % (cmd_parser.prog.replace('_', ' '),
update_arguments(cmd_parser, instance.arguments)
instance = cls(dict(cmd_parser.arguments))
cmd_parser.syntax = '%s %s' % (
subcmd.path.replace('_', ' '), cls.syntax)
if '-h' in cmd_args or '--help' in cmd_args:
parsed, unparsed = cmd_parser.parse_known_args(cmd_args)
for name, arg in instance.arguments.items():
arg.value = getattr(parsed, name, arg.default)
arg.value = getattr(cmd_parser.parsed, name, arg.default)
_exec_cmd(instance, unparsed, cmd_parser.print_help)
except CLIError as err:
elif ('-h' in cmd_args or '--help' in cmd_args) \
......@@ -225,9 +226,10 @@ class Shell(Cmd):
hdr = tmp_partition[0].strip()
return '%s commands:' % hdr
def run(self, arguments, path=''):
self._history = History(arguments['config'].get('history', 'file'))
self._arguments = arguments
def run(self, parser, path=''):
self._parser = parser
self._history = History(
parser.arguments['config'].get('history', 'file'))
if path:
cmd = self.cmd_tree.get_command(path)
intro = cmd.path.replace('_', ' ')
......@@ -439,9 +439,9 @@ class server_wait(_init_cyclades):
if new_mode:
print('\nServer %s is now in %s mode' % (server_id, new_mode))
print('Server %s is now in %s mode' % (server_id, new_mode))
print('\nTime out')
print('Time out')
......@@ -33,6 +33,9 @@
class HTTPResponse(object):
"""An abstract HTTP Response object to handle a performed HTTPRequest.
Subclass implementation required
def __init__(self, request=None, prefetched=False):
self.request = request
......@@ -47,11 +50,12 @@ class HTTPResponse(object):
def release(self):
"""Release the connection.
Use this after finished using the response"""
raise NotImplementedError
def prefetched(self):
"""flag to avoid downloading more than nessecary"""
return self._prefetched
......@@ -60,6 +64,7 @@ class HTTPResponse(object):
def content(self):
"""(binary) request response content (data)"""
return self._content
......@@ -69,6 +74,7 @@ class HTTPResponse(object):
def text(self):
return self._text
......@@ -78,6 +84,7 @@ class HTTPResponse(object):
def json(self):
return self._json
......@@ -87,6 +94,7 @@ class HTTPResponse(object):
def headers(self):
return self._headers
......@@ -96,6 +104,7 @@ class HTTPResponse(object):
def status_code(self):
"""(int) optional"""
return self._status_code
......@@ -105,6 +114,7 @@ class HTTPResponse(object):
def status(self):
"""(str) useful in server error responses"""
return self._status
......@@ -114,6 +124,7 @@ class HTTPResponse(object):
def request(self):
"""(HTTPConnection) the source of this response object"""
return self._request
......@@ -122,6 +133,8 @@ class HTTPResponse(object):
class HTTPConnection(object):
"""An abstract HTTP Connection mechanism. Subclass implementation required
def __init__(self, method=None, url=None, params={}, headers={}):
self.headers = headers
......@@ -71,8 +71,11 @@ class KamakiHTTPResponse(HTTPResponse):
def text(self):
:returns: (str) content
return self._content
return unicode(self._content)
def test(self, v):
......@@ -80,6 +83,11 @@ class KamakiHTTPResponse(HTTPResponse):