Merge branch 'master' into feature-quota

Conflicts:
	docs/installation.rst

Changelog

+CHANGELOG
+
+1. Clients API relies on objpool instead of snf-common for connection pooling
+2. MS Windows compatiblity:
+    * adjust pithos interface
+    * adjust pithos client local fs handling
+    * progress bar configuration not to rely on unicode characters
+    * adjust hidden Clients API unittests
+3. Update documentation:
+    * rearange text for easier setup
+    * document undocumented methods in Clients API and cli auxiliary methods
+    * MS Windows setup guide
+4. Improve interface output:
+    * detailed description mechanism
+    * uniformity between shell and one-command
+    * various command-specific improvements
+    * new print_list, print_dict and print_item methods have:
+        - better indentation 
+        - enumeration flag
+        - redundancy of presented information flag
+    * mechanism for more descriptive syntax errors and arguments help
+5. Improve error handling:
+    * concrete mechanism for client error propagation
+    * catch some unhandled errors
+    * mechanism for detailed error descriptions
+    * context-specific errors
+    * showcase error handling in store_list
+6. CLI Code restructure:
+    * Introduce ArgumentParseManager class to manage argument parsing
+    * Introduce a seperate one_command package for handling one_command
+    * clean-up main CLI code (move interface-specific methods to their pckgs)
+7. New history capabilities:
+    * allow command ranges
+    * allow backward counting
+    * in-shell sequensial script-like execution of previous commands
+8. Minor new features:
+    * store_publish returns publication url
+    * store_list can list with prefixes instead of full path
+    * hidden optional testing of quoted text and cli.utils.print_* methods
+    * shell prompt easy to modify
+9. Bugfixes:
+    * Shell does not repeat previous command in case of error
+    * handle ValueErrors and KeyErrors
+    * config set values are effective immidiately
+    * unwanted argument inheritance in shell, removed

docs/conf.py

@@ -7,7 +7,7 @@
 # dir.
 #
 # Note that not all possible configuration values are present in this
-# autogenerated file.
+# auto-generated file.
 #
 # All configuration values have a default; values that are commented out
 # serve to show the default.
@@ -19,7 +19,6 @@
 from sys import path, stderr
 import os
 
-path.append('/home/saxtouri/src/objpool')
 try:
     from objpool import http
     http
@@ -27,7 +26,7 @@ except ImportError:
     stderr.write("`objpool` package is required to build kamaki docs.\n")
     #exit()
 
-path.append(os.path.join(os.path.abspath(os.path.dirname(__file__)), '..'))
+path.insert(0, os.path.join(os.path.abspath(os.path.dirname(__file__)), '..'))
 
 # -- General configuration ---------------------------------------------------
 
@@ -61,7 +60,7 @@ copyright = u'2012, GRNET'
 # The short X.Y version.
 version = '0.6'
 # The full version, including alpha/beta/rc tags.
-release = '0.6.1'
+release = '0.6.2'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

docs/installation.rst

@@ -6,11 +6,74 @@ This guide describes the standard installation process for kamaki, with the aspi
 
 * Kamaki repository: `http://code.grnet.gr/git/kamaki <http://code.grnet.gr/git/kamaki>`_
 
-* Synnefo Linux packages: `http://apt.dev.grnet.gr <http://apt.dev.grnet.gr>`_
+* Synnefo Linux packages: `http://apt.dev.grnet.gr <http://apt.dev.grnet.gr>`_, `http://apt2.dev.grnet.gr <http://apt2.dev.grnet.gr>`_
 
 Linux and Unix-like enviroments
 -------------------------------
 
+Ubuntu and Debian packages
+^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+The following steps describe a command-line approach, but any graphic package manager can be used instead.
+
+Add the following to apt sources list
+"""""""""""""""""""""""""""""""""""""
+
+As root, append one of the following to */etc/apt/sources.list*:
+
+* Debian Sid (kamaki 0.6.2)::
+
+    deb http://apt.dev.grnet.gr/ sid main
+
+* Debian Stable (kamaki 0.6.1)::
+
+    deb http://apt.dev.grnet.gr/ squeeze main
+    deb http://apt2.dev.grnet.gr stable/
+
+* Ubuntu (kamaki 0.6.1)::
+
+    deb http://apt.dev.grnet.gr/ precise main
+
+
+Update
+""""""
+
+.. note:: make sure the GPG public key for the GRNET dev team is added:
+
+    .. code-block:: console
+
+        $ curl https://dev.grnet.gr/files/apt-grnetdev.pub|apt-key add -
+
+    otherwise *apt-get update* will produce GPG warnings.
+
+.. code-block:: console
+
+    $ sudo apt-get update
+
+
+Install kamaki
+""""""""""""""
+
+.. note:: **versions 0.6.0 - 0.6.1:**
+
+    The *snf-common* package (available at synnefo apt repository) will be automatically installed as a dependency.
+
+.. note:: **versions 0.6.2 and on:**
+
+    Since version 0.6.2, *objpool* replaces *snf-common*. The objpool package is also available at synnefo repository and is automatically installed as a dependency. The *snf-common* dependency is removed.
+
+.. code-block:: console
+
+    $ sudo apt-get install kamaki
+
+Install ansicolors and/or progress (Optional)
+"""""""""""""""""""""""""""""""""""""""""""""
+
+.. code-block:: console
+
+    $ sudo apt-get install python-ansicolors
+    $ sudo apt-get install python-progress
+
 .. _installing-from-source-ref:
 
 Installing from source (git repos.)
@@ -28,8 +91,8 @@ Optional:
 
  * VirtualEnv (python-virtualenv) [http://www.virtualenv.org]
 
-1. Setup a virtual enviroment (optional)
-""""""""""""""""""""""""""""""""""""""""
+Setup a virtual enviroment (optional)
+"""""""""""""""""""""""""""""""""""""
 
 With virtualenv users can setup kamaki and synnefo services in a sandbox environment.
 
@@ -40,8 +103,8 @@ With virtualenv users can setup kamaki and synnefo services in a sandbox environ
 
 A more detailed example of using virtual env can be found at the `snf-image-creator setup guide <http://docs.dev.grnet.gr/snf-image-creator/latest/install.html#python-virtual-environment>`_
 
-2. Install objpool (was: snf-common)
-""""""""""""""""""""""""""""""""""""""""""
+Install objpool (was: snf-common)
+"""""""""""""""""""""""""""""""""
 
 .. note:: **versions 0.6.0 - 0.6.1**
 
@@ -65,8 +128,8 @@ Since 0.6.2, kamaki is based on objpool (hence the snf-common dependency is now
     $ ./setup build install
     $ cd -
 
-3. Install kamaki
-"""""""""""""""""
+Install kamaki
+""""""""""""""
 
 .. code-block:: console
 
@@ -74,8 +137,8 @@ Since 0.6.2, kamaki is based on objpool (hence the snf-common dependency is now
     $ cd kamaki
     $ ./setup build install
 
-4. Install progress and/or ansicolors (optional)
-""""""""""""""""""""""""""""""""""""""""""""""""
+Install progress and/or ansicolors (optional)
+"""""""""""""""""""""""""""""""""""""""""""""
 
 progress: command-line progress bars (in some commands)
 
@@ -86,66 +149,102 @@ ansicolors: color kamaki output (can switched on and off in `setup <setup.html>`
     $ pip install progress
     $ pip install ansicolors
 
-Ubuntu and Debian packages
-^^^^^^^^^^^^^^^^^^^^^^^^^^
+Mac OS X
+--------
 
-The following steps describe a command-line approach, but any graphic package manager can be used instead.
+Kamaki can be installed on Mac OS X systems from source, by following the steps at :ref:`installing-from-source-ref`.
 
-1. Add the following to apt sources list
-""""""""""""""""""""""""""""""""""""""""
+Windows
+-------
 
-* Debian::
+Since version 0.6.2 kamaki can run on Windows, either on standard Windows console, or inside an improved command line shell. The present guide presents a tested method for using kamaki in windows
 
-    deb http://apt.dev.grnet.gr/ sid main
+Requirements
+^^^^^^^^^^^^
 
-* Ubuntu::
+* Python 2.7 or better (`Official versions <http://www.python.org/getit>`_)
 
-    deb http://apt.dev.grnet.gr/ precise main
+* Git (download `windows version <http://git-scm.com/download/win>`_)
 
-2. Update
-"""""""""
+* Setuptools (`Official versions and workarounds <http://pypi.python.org/pypi/setuptools>`_)
 
-.. code-block:: console
+Installation from source
+^^^^^^^^^^^^^^^^^^^^^^^^
 
-    $ sudo apt-get update
+Install python
+""""""""""""""
 
-.. note:: Don't forget to get the GPG public key for the GRNET dev team:
+Download and run the Windows installer from `here <http://www.python.org/getit>`_
 
-    .. code-block:: console
+Users should pick the installer that fits their windows version and architecture.
 
-        $ curl https://dev.grnet.gr/files/apt-grnetdev.pub|apt-key add -
+Add python to windows path
+""""""""""""""""""""""""""
 
-    otherwise *apt-get update* will produce GPG warnings.
+The following will allow users to run python and python scripts from command line.
 
-3. Install kamaki
-"""""""""""""""""
+* Select **System** from the Control Panel, select the **Advanced** tab, the **Environment Variables** button and then find the **PATH** (user or system) and **edit**
 
-.. note:: **versions 0.6.0 - 0.6.1:**
+* Without removing existing values, append the following to PATH::
 
-    The *snf-common* package (available at synnefo apt repository) will be automatically installed as a dependency.
+    C:\Python;C:\Python\Scripts
 
-.. note:: **versions 0.6.2 and on:**
+.. note:: Path values are separated by semicolons
 
-    Since version 0.6.2, *objpool* replaces *snf-common*. The objpool package is also available at synnefo repository and is automatically installed as a dependency. The *snf-common* dependency is removed.
+.. warning:: C:\\Python should be replaced with the actual python path in the system, e.g. C:\\Python27
 
-.. code-block:: console
+Install setuptools
+""""""""""""""""""
 
-    $ sudo apt-get install kamaki
+According to the corresponding `python org page <http://pypi.python.org/pypi/setuptools>`_, the setuptools installer doesn't currently work on 64bit machines.
 
-4. Install ansicolors and/or progress (Optional)
-""""""""""""""""""""""""""""""""""""""""""""""""
+* Users with 32-bit operating systems should download and run the graphic installer
 
-.. code-block:: console
+* Users with 64-bit machines should download the `ez_setup.py <http://peak.telecommunity.com/dist/ez_setup.py>`_ script and install it from a command shell. In the following example, the script was downloaded at C:\\Downloads::
 
-    $ sudo apt-get install python-ansicolors
-    $ sudo apt-get install python-progress
+    C:\> cd Downloads
+    C:\Downloads\> python ez_setup.py
+    ...
+    Installation finished
+    C:\Downloads\>
 
-Mac OS X
---------
+Install GIT
+"""""""""""
 
-Kamaki can be installed on Mac OS X systems from source, by following the steps at :ref:`installing-from-source-ref`.
+`Download GIT <http://git-scm.com/download/win>`_ and run the graphic installer. During the installation, users will be able to modify some installation options. The present guide is tested with the default selections.
 
-Windows
--------
+After the installation is completed, a GIT standalone shell will be installed (a desktop shortcut is created, by default). Users are advised to run kamaki through this shell.
+
+Install kamaki
+""""""""""""""
+
+* Run the GIT standalone shell
+
+* Enter the location where kamaki will be installed, e.g. **C:\\**
+
+    .. code-block:: console
+
+        $ cd /c/
+
+* Download source from GRNET repository
+
+    .. code-block:: console
+
+        $ git clone http://code.grnet.gr/git/kamaki
+        Cloning into 'kamaki'...
+        Receiving objects: ...
+        Resolving Deltas: ...
+
+* Enter source and install kamaki
+
+    .. code-block:: console
+
+        $ cd kamaki
+        $ python setup.py install
+        running install
+        ...
+        Finished processing dependencies for kamaki==0.6.2
+
+.. warning:: kamaki version should be 0.6.2 or better, otherwise it will not function. Users can test that by running::
 
-Although it is proven not too tricky to install kamaki on Windows console using `git for windows <http://git-scm.com/downloads>`_, Windows environments are not supported at the time being.
+    $ kamaki --version

docs/man/kamaki.rst

@@ -85,9 +85,9 @@ history commands
 
 Command user history, as stored in ~/.kamaki.history
 
-* all       show user history
+* show      show user history
 * clean     clean up history
-* load      Run previously executed command(s)
+* run       run previously executed command(s)
 
 
 server commands

docs/setup.rst

@@ -3,6 +3,30 @@ Setup
 
 Kamaki is easy to install from source or as a package. Some ui features are optional and can be install separately. Kamaki behavior can be configured in the kamaki config file.
 
+Quick Setup
+-----------
+
+Kamaki interfaces rely on a list of configuration options. Be default, they are configured to communicate with the `Okeanos IaaS <http://okeanos.grnet.gr>`_.
+
+.. note:: It is essential for users to get a configuration token (okeanos.grnet.gr users go `here <https://accounts.okeanos.grnet.gr/im/>`_) and provide it to kamaki:
+
+
+.. code-block:: console
+    :emphasize-lines: 1
+
+    Example 1.1: Set user token to myt0k3n==
+
+    $ kamaki set token myt0k3n==
+
+To use the storage service, a user should also provide the corresponding user-name:
+
+.. code-block:: console
+    :emphasize-lines: 1
+
+    Example 1.2: Set user name to user@domain.com
+
+    $ kamaki set store.account user@domain.com
+
 Optional features
 -----------------
 

docs/usage.rst

@@ -26,7 +26,7 @@ To use the storage service, a user should also provide the corresponding user-na
 
     Example 1.2: Set user name to user@domain.com
 
-    $ kamaki set account user@domain.com
+    $ kamaki set account store.user@domain.com
 
 Shell vs one-command
 --------------------
@@ -656,7 +656,7 @@ The following kamaki sequence copies and downloads a file from mycontainer1, upl
     6.  history show
 
     *repeat the process *
-    [history]: load 2-4
+    [history]: run 2-4
     store copy mycontainer1:somefile mycontainer1:myfile
     store download mycontainer1:myfile mylocalfile
     Download completed

kamaki/cli/__init__.py

@@ -45,6 +45,11 @@ _help = False
 _debug = False
 _verbose = False
 _colors = False
+kloger = None
+
+#  command auxiliary methods
+
+_best_match = []
 
 
 def _construct_command_syntax(cls):
@@ -66,16 +71,6 @@ def _construct_command_syntax(cls):
             cls.syntax += ' <%s ...>' % spec.varargs
 
 
-def _get_cmd_tree_from_spec(spec, cmd_tree_list):
-    for tree in cmd_tree_list:
-        if tree.name == spec:
-            return tree
-    return None
-
-
-_best_match = []
-
-
 def _num_of_matching_terms(basic_list, attack_list):
     if not attack_list:
         return len(basic_list)
@@ -111,27 +106,31 @@ def _update_best_match(name_terms, prefix=[]):
 
 def command(cmd_tree, prefix='', descedants_depth=1):
     """Load a class as a command
-        spec_cmd0_cmd1 will be command spec cmd0
-        @cmd_tree is initialized in cmd_spec file and is the structure
+        e.g. spec_cmd0_cmd1 will be command spec cmd0
+
+        :param cmd_tree: is initialized in cmd_spec file and is the structure
             where commands are loaded. Var name should be _commands
-        @param prefix if given, load only commands prefixed with prefix,
-        @param descedants_depth is the depth of the tree descedants of the
+        :param prefix: if given, load only commands prefixed with prefix,
+        :param descedants_depth: is the depth of the tree descedants of the
             prefix command. It is used ONLY if prefix and if prefix is not
             a terminal command
+
+        :returns: the specified class object
     """
 
     def wrap(cls):
+        global kloger
         cls_name = cls.__name__
 
         if not cmd_tree:
             if _debug:
-                print('Warning: command %s found but not loaded' % cls_name)
+                kloger.warning('command %s found but not loaded' % cls_name)
             return cls
 
         name_terms = cls_name.split('_')
         if not _update_best_match(name_terms, prefix):
             if _debug:
-                print('Warning: %s failed to update_best_match' % cls_name)
+                kloger.warning('%s failed to update_best_match' % cls_name)
             return None
 
         global _best_match
@@ -141,7 +140,7 @@ def command(cmd_tree, prefix='', descedants_depth=1):
             if not cmd_tree.has_command(partial):  # add partial path
                 cmd_tree.add_command(partial)
             if _debug:
-                print('Warning: %s failed max_len test' % cls_name)
+                kloger.warning('%s failed max_len test' % cls_name)
             return None
 
         cls.description, sep, cls.long_description\
@@ -153,11 +152,6 @@ def command(cmd_tree, prefix='', descedants_depth=1):
     return wrap
 
 
-def get_cmd_terms():
-    global command
-    return [term for term in command.func_defaults[0]\
-        if not term.startswith('-')]
-
 cmd_spec_locations = [
     'kamaki.cli.commands',
     'kamaki.commands',
@@ -166,6 +160,9 @@ cmd_spec_locations = [
     '']
 
 
+#  Generic init auxiliary functions
+
+
 def _setup_logging(silent=False, debug=False, verbose=False, include=False):
     """handle logging for clients package"""
 
@@ -179,18 +176,23 @@ def _setup_logging(silent=False, debug=False, verbose=False, include=False):
 
     if silent:
         add_handler('', logging.CRITICAL)
-    elif debug:
+        return
+
+    if debug:
         add_handler('requests', logging.INFO, prefix='* ')
         add_handler('clients.send', logging.DEBUG, prefix='> ')
         add_handler('clients.recv', logging.DEBUG, prefix='< ')
+        add_handler('kamaki', logging.DEBUG, prefix='(debug): ')
     elif verbose:
         add_handler('requests', logging.INFO, prefix='* ')
         add_handler('clients.send', logging.INFO, prefix='> ')
         add_handler('clients.recv', logging.INFO, prefix='< ')
+        add_handler('kamaki', logging.INFO, prefix='(i): ')
     elif include:
         add_handler('clients.recv', logging.INFO)
-    else:
-        add_handler('', logging.WARNING)
+    add_handler('kamaki', logging.WARNING, prefix='(warning): ')
+    global kloger
+    kloger = logging.getLogger('kamaki')
 
 
 def _init_session(arguments):
@@ -210,18 +212,6 @@ def _init_session(arguments):
     _setup_logging(_silent, _debug, _verbose, _include)
 
 
-def get_command_group(unparsed, arguments):
-    groups = arguments['config'].get_groups()
-    for term in unparsed:
-        if term.startswith('-'):
-            continue
-        if term in groups:
-            unparsed.remove(term)
-            return term
-        return None
-    return None
-
-
 def _load_spec_module(spec, arguments, module):
     spec_name = arguments['config'].get(spec, 'cli')
     if spec_name is None:
@@ -239,6 +229,7 @@ def _load_spec_module(spec, arguments, module):
 
 def _groups_help(arguments):
     global _debug
+    global kloger
     descriptions = {}
     for spec in arguments['config'].get_groups():
         pkg = _load_spec_module(spec, arguments, '_commands')
@@ -251,20 +242,41 @@ def _groups_help(arguments):
                 ]
             except AttributeError:
                 if _debug:
-                    print('Warning: No description for %s' % spec)
+                    kloger.warning('No description for %s' % spec)
             try:
                 for cmd in cmds:
                     descriptions[cmd.name] = cmd.description
             except TypeError:
                 if _debug:
-                    print('Warning: no cmd specs in module %s' % spec)
+                    kloger.warning('no cmd specs in module %s' % spec)
         elif _debug:
-            print('Warning: Loading of %s cmd spec failed' % spec)
+            kloger.warning('Loading of %s cmd spec failed' % spec)
     print('\nOptions:\n - - - -')
     print_dict(descriptions)
 
 
-def _print_subcommands_help(cmd):
+def _load_all_commands(cmd_tree, arguments):
+    _config = arguments['config']
+    for spec in [spec for spec in _config.get_groups()\
+            if _config.get(spec, 'cli')]:
+        try:
+            spec_module = _load_spec_module(spec, arguments, '_commands')
+            spec_commands = getattr(spec_module, '_commands')
+        except AttributeError:
+            if _debug:
+                global kloger
+                kloger.warning('No valid description for %s' % spec)
+            continue
+        for spec_tree in spec_commands:
+            if spec_tree.name == spec:
+                cmd_tree.add_tree(spec_tree)
+                break
+
+
+#  Methods to be used by CLI implementations
+
+
+def print_subcommands_help(cmd):
     printout = {}
     for subcmd in cmd.get_subcommands():
         spec, sep, print_path = subcmd.path.partition('_')
@@ -274,24 +286,28 @@ def _print_subcommands_help(cmd):
         print_dict(printout)
 
 
-def _update_parser_help(parser, cmd):
+def update_parser_help(parser, cmd):
     global _best_match
     parser.syntax = parser.syntax.split('<')[0]
     parser.syntax += ' '.join(_best_match)
 
+    description = ''
     if cmd.is_command:
         cls = cmd.get_class()
         parser.syntax += ' ' + cls.syntax
         parser.update_arguments(cls().arguments)
-        # arguments = cls().arguments
-        # update_arguments(parser, arguments)
+        description = getattr(cls, 'long_description', '')
+        description = description.strip()
     else:
         parser.syntax += ' <...>'
     if cmd.has_description:
-        parser.parser.description = cmd.help
+        parser.parser.description = cmd.help\
+        + (('\n%s' % description) if description else '')
+    else:
+        parser.parser.description = description
 
 
-def _print_error_message(cli_err):
+def print_error_message(cli_err):
     errmsg = '%s' % cli_err
     if cli_err.importance == 1:
         errmsg = magenta(errmsg)
@@ -300,20 +316,11 @@ def _print_error_message(cli_err):
     elif cli_err.importance > 2:
         errmsg = red(errmsg)
     stdout.write(errmsg)
-    print_list(cli_err.details)
-
-
-def _get_best_match_from_cmd_tree(cmd_tree, unparsed):
-    matched = [term for term in unparsed if not term.startswith('-')]
-    while matched:
-        try:
-            return cmd_tree.get_command('_'.join(matched))
-        except KeyError:
-            matched = matched[:-1]
-    return None
+    for errmsg in cli_err.details:
+        print('| %s' % errmsg)