Commit 131f4e86 authored by Stavros Sachtouris's avatar Stavros Sachtouris
Browse files

Merge branch 'master' into feature-quota

Conflicts:
	docs/installation.rst
parents a6417246 deefff94
CHANGELOG
1. Clients API relies on objpool instead of snf-common for connection pooling
2. MS Windows compatiblity:
* adjust pithos interface
* adjust pithos client local fs handling
* progress bar configuration not to rely on unicode characters
* adjust hidden Clients API unittests
3. Update documentation:
* rearange text for easier setup
* document undocumented methods in Clients API and cli auxiliary methods
* MS Windows setup guide
4. Improve interface output:
* detailed description mechanism
* uniformity between shell and one-command
* various command-specific improvements
* new print_list, print_dict and print_item methods have:
- better indentation
- enumeration flag
- redundancy of presented information flag
* mechanism for more descriptive syntax errors and arguments help
5. Improve error handling:
* concrete mechanism for client error propagation
* catch some unhandled errors
* mechanism for detailed error descriptions
* context-specific errors
* showcase error handling in store_list
6. CLI Code restructure:
* Introduce ArgumentParseManager class to manage argument parsing
* Introduce a seperate one_command package for handling one_command
* clean-up main CLI code (move interface-specific methods to their pckgs)
7. New history capabilities:
* allow command ranges
* allow backward counting
* in-shell sequensial script-like execution of previous commands
8. Minor new features:
* store_publish returns publication url
* store_list can list with prefixes instead of full path
* hidden optional testing of quoted text and cli.utils.print_* methods
* shell prompt easy to modify
9. Bugfixes:
* Shell does not repeat previous command in case of error
* handle ValueErrors and KeyErrors
* config set values are effective immidiately
* unwanted argument inheritance in shell, removed
......@@ -7,7 +7,7 @@
# dir.
#
# Note that not all possible configuration values are present in this
# autogenerated file.
# auto-generated file.
#
# All configuration values have a default; values that are commented out
# serve to show the default.
......@@ -19,7 +19,6 @@
from sys import path, stderr
import os
path.append('/home/saxtouri/src/objpool')
try:
from objpool import http
http
......@@ -27,7 +26,7 @@ except ImportError:
stderr.write("`objpool` package is required to build kamaki docs.\n")
#exit()
path.append(os.path.join(os.path.abspath(os.path.dirname(__file__)), '..'))
path.insert(0, os.path.join(os.path.abspath(os.path.dirname(__file__)), '..'))
# -- General configuration ---------------------------------------------------
......@@ -61,7 +60,7 @@ copyright = u'2012, GRNET'
# The short X.Y version.
version = '0.6'
# The full version, including alpha/beta/rc tags.
release = '0.6.1'
release = '0.6.2'
# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
......
......@@ -6,11 +6,74 @@ This guide describes the standard installation process for kamaki, with the aspi
* Kamaki repository: `http://code.grnet.gr/git/kamaki <http://code.grnet.gr/git/kamaki>`_
* Synnefo Linux packages: `http://apt.dev.grnet.gr <http://apt.dev.grnet.gr>`_
* Synnefo Linux packages: `http://apt.dev.grnet.gr <http://apt.dev.grnet.gr>`_, `http://apt2.dev.grnet.gr <http://apt2.dev.grnet.gr>`_
Linux and Unix-like enviroments
-------------------------------
Ubuntu and Debian packages
^^^^^^^^^^^^^^^^^^^^^^^^^^
The following steps describe a command-line approach, but any graphic package manager can be used instead.
Add the following to apt sources list
"""""""""""""""""""""""""""""""""""""
As root, append one of the following to */etc/apt/sources.list*:
* Debian Sid (kamaki 0.6.2)::
deb http://apt.dev.grnet.gr/ sid main
* Debian Stable (kamaki 0.6.1)::
deb http://apt.dev.grnet.gr/ squeeze main
deb http://apt2.dev.grnet.gr stable/
* Ubuntu (kamaki 0.6.1)::
deb http://apt.dev.grnet.gr/ precise main
Update
""""""
.. note:: make sure the GPG public key for the GRNET dev team is added:
.. code-block:: console
$ curl https://dev.grnet.gr/files/apt-grnetdev.pub|apt-key add -
otherwise *apt-get update* will produce GPG warnings.
.. code-block:: console
$ sudo apt-get update
Install kamaki
""""""""""""""
.. note:: **versions 0.6.0 - 0.6.1:**
The *snf-common* package (available at synnefo apt repository) will be automatically installed as a dependency.
.. note:: **versions 0.6.2 and on:**
Since version 0.6.2, *objpool* replaces *snf-common*. The objpool package is also available at synnefo repository and is automatically installed as a dependency. The *snf-common* dependency is removed.
.. code-block:: console
$ sudo apt-get install kamaki
Install ansicolors and/or progress (Optional)
"""""""""""""""""""""""""""""""""""""""""""""
.. code-block:: console
$ sudo apt-get install python-ansicolors
$ sudo apt-get install python-progress
.. _installing-from-source-ref:
Installing from source (git repos.)
......@@ -28,8 +91,8 @@ Optional:
* VirtualEnv (python-virtualenv) [http://www.virtualenv.org]
1. Setup a virtual enviroment (optional)
""""""""""""""""""""""""""""""""""""""""
Setup a virtual enviroment (optional)
"""""""""""""""""""""""""""""""""""""
With virtualenv users can setup kamaki and synnefo services in a sandbox environment.
......@@ -40,8 +103,8 @@ With virtualenv users can setup kamaki and synnefo services in a sandbox environ
A more detailed example of using virtual env can be found at the `snf-image-creator setup guide <http://docs.dev.grnet.gr/snf-image-creator/latest/install.html#python-virtual-environment>`_
2. Install objpool (was: snf-common)
""""""""""""""""""""""""""""""""""""""""""
Install objpool (was: snf-common)
"""""""""""""""""""""""""""""""""
.. note:: **versions 0.6.0 - 0.6.1**
......@@ -65,8 +128,8 @@ Since 0.6.2, kamaki is based on objpool (hence the snf-common dependency is now
$ ./setup build install
$ cd -
3. Install kamaki
"""""""""""""""""
Install kamaki
""""""""""""""
.. code-block:: console
......@@ -74,8 +137,8 @@ Since 0.6.2, kamaki is based on objpool (hence the snf-common dependency is now
$ cd kamaki
$ ./setup build install
4. Install progress and/or ansicolors (optional)
""""""""""""""""""""""""""""""""""""""""""""""""
Install progress and/or ansicolors (optional)
"""""""""""""""""""""""""""""""""""""""""""""
progress: command-line progress bars (in some commands)
......@@ -86,66 +149,102 @@ ansicolors: color kamaki output (can switched on and off in `setup <setup.html>`
$ pip install progress
$ pip install ansicolors
Ubuntu and Debian packages
^^^^^^^^^^^^^^^^^^^^^^^^^^
Mac OS X
--------
The following steps describe a command-line approach, but any graphic package manager can be used instead.
Kamaki can be installed on Mac OS X systems from source, by following the steps at :ref:`installing-from-source-ref`.
1. Add the following to apt sources list
""""""""""""""""""""""""""""""""""""""""
Windows
-------
* Debian::
Since version 0.6.2 kamaki can run on Windows, either on standard Windows console, or inside an improved command line shell. The present guide presents a tested method for using kamaki in windows
deb http://apt.dev.grnet.gr/ sid main
Requirements
^^^^^^^^^^^^
* Ubuntu::
* Python 2.7 or better (`Official versions <http://www.python.org/getit>`_)
deb http://apt.dev.grnet.gr/ precise main
* Git (download `windows version <http://git-scm.com/download/win>`_)
2. Update
"""""""""
* Setuptools (`Official versions and workarounds <http://pypi.python.org/pypi/setuptools>`_)
.. code-block:: console
Installation from source
^^^^^^^^^^^^^^^^^^^^^^^^
$ sudo apt-get update
Install python
""""""""""""""
.. note:: Don't forget to get the GPG public key for the GRNET dev team:
Download and run the Windows installer from `here <http://www.python.org/getit>`_
.. code-block:: console
Users should pick the installer that fits their windows version and architecture.
$ curl https://dev.grnet.gr/files/apt-grnetdev.pub|apt-key add -
Add python to windows path
""""""""""""""""""""""""""
otherwise *apt-get update* will produce GPG warnings.
The following will allow users to run python and python scripts from command line.
3. Install kamaki
"""""""""""""""""
* Select **System** from the Control Panel, select the **Advanced** tab, the **Environment Variables** button and then find the **PATH** (user or system) and **edit**
.. note:: **versions 0.6.0 - 0.6.1:**
* Without removing existing values, append the following to PATH::
The *snf-common* package (available at synnefo apt repository) will be automatically installed as a dependency.
C:\Python;C:\Python\Scripts
.. note:: **versions 0.6.2 and on:**
.. note:: Path values are separated by semicolons
Since version 0.6.2, *objpool* replaces *snf-common*. The objpool package is also available at synnefo repository and is automatically installed as a dependency. The *snf-common* dependency is removed.
.. warning:: C:\\Python should be replaced with the actual python path in the system, e.g. C:\\Python27
.. code-block:: console
Install setuptools
""""""""""""""""""
$ sudo apt-get install kamaki
According to the corresponding `python org page <http://pypi.python.org/pypi/setuptools>`_, the setuptools installer doesn't currently work on 64bit machines.
4. Install ansicolors and/or progress (Optional)
""""""""""""""""""""""""""""""""""""""""""""""""
* Users with 32-bit operating systems should download and run the graphic installer
.. code-block:: console
* Users with 64-bit machines should download the `ez_setup.py <http://peak.telecommunity.com/dist/ez_setup.py>`_ script and install it from a command shell. In the following example, the script was downloaded at C:\\Downloads::
$ sudo apt-get install python-ansicolors
$ sudo apt-get install python-progress
C:\> cd Downloads
C:\Downloads\> python ez_setup.py
...
Installation finished
C:\Downloads\>
Mac OS X
--------
Install GIT
"""""""""""
Kamaki can be installed on Mac OS X systems from source, by following the steps at :ref:`installing-from-source-ref`.
`Download GIT <http://git-scm.com/download/win>`_ and run the graphic installer. During the installation, users will be able to modify some installation options. The present guide is tested with the default selections.
Windows
-------
After the installation is completed, a GIT standalone shell will be installed (a desktop shortcut is created, by default). Users are advised to run kamaki through this shell.
Install kamaki
""""""""""""""
* Run the GIT standalone shell
* Enter the location where kamaki will be installed, e.g. **C:\\**
.. code-block:: console
$ cd /c/
* Download source from GRNET repository
.. code-block:: console
$ git clone http://code.grnet.gr/git/kamaki
Cloning into 'kamaki'...
Receiving objects: ...
Resolving Deltas: ...
* Enter source and install kamaki
.. code-block:: console
$ cd kamaki
$ python setup.py install
running install
...
Finished processing dependencies for kamaki==0.6.2
.. warning:: kamaki version should be 0.6.2 or better, otherwise it will not function. Users can test that by running::
Although it is proven not too tricky to install kamaki on Windows console using `git for windows <http://git-scm.com/downloads>`_, Windows environments are not supported at the time being.
$ kamaki --version
......@@ -85,9 +85,9 @@ history commands
Command user history, as stored in ~/.kamaki.history
* all show user history
* show show user history
* clean clean up history
* load Run previously executed command(s)
* run run previously executed command(s)
server commands
......
......@@ -3,6 +3,30 @@ Setup
Kamaki is easy to install from source or as a package. Some ui features are optional and can be install separately. Kamaki behavior can be configured in the kamaki config file.
Quick Setup
-----------
Kamaki interfaces rely on a list of configuration options. Be default, they are configured to communicate with the `Okeanos IaaS <http://okeanos.grnet.gr>`_.
.. note:: It is essential for users to get a configuration token (okeanos.grnet.gr users go `here <https://accounts.okeanos.grnet.gr/im/>`_) and provide it to kamaki:
.. code-block:: console
:emphasize-lines: 1
Example 1.1: Set user token to myt0k3n==
$ kamaki set token myt0k3n==
To use the storage service, a user should also provide the corresponding user-name:
.. code-block:: console
:emphasize-lines: 1
Example 1.2: Set user name to user@domain.com
$ kamaki set store.account user@domain.com
Optional features
-----------------
......
......@@ -26,7 +26,7 @@ To use the storage service, a user should also provide the corresponding user-na
Example 1.2: Set user name to user@domain.com
$ kamaki set account user@domain.com
$ kamaki set account store.user@domain.com
Shell vs one-command
--------------------
......@@ -656,7 +656,7 @@ The following kamaki sequence copies and downloads a file from mycontainer1, upl
6. history show
*repeat the process *
[history]: load 2-4
[history]: run 2-4
store copy mycontainer1:somefile mycontainer1:myfile
store download mycontainer1:myfile mylocalfile
Download completed
......
......@@ -45,6 +45,11 @@ _help = False
_debug = False
_verbose = False
_colors = False
kloger = None
# command auxiliary methods
_best_match = []
def _construct_command_syntax(cls):
......@@ -66,16 +71,6 @@ def _construct_command_syntax(cls):
cls.syntax += ' <%s ...>' % spec.varargs
def _get_cmd_tree_from_spec(spec, cmd_tree_list):
for tree in cmd_tree_list:
if tree.name == spec:
return tree
return None
_best_match = []
def _num_of_matching_terms(basic_list, attack_list):
if not attack_list:
return len(basic_list)
......@@ -111,27 +106,31 @@ def _update_best_match(name_terms, prefix=[]):
def command(cmd_tree, prefix='', descedants_depth=1):
"""Load a class as a command
spec_cmd0_cmd1 will be command spec cmd0
@cmd_tree is initialized in cmd_spec file and is the structure
e.g. spec_cmd0_cmd1 will be command spec cmd0
:param cmd_tree: is initialized in cmd_spec file and is the structure
where commands are loaded. Var name should be _commands
@param prefix if given, load only commands prefixed with prefix,
@param descedants_depth is the depth of the tree descedants of the
:param prefix: if given, load only commands prefixed with prefix,
:param descedants_depth: is the depth of the tree descedants of the
prefix command. It is used ONLY if prefix and if prefix is not
a terminal command
:returns: the specified class object
"""
def wrap(cls):
global kloger
cls_name = cls.__name__
if not cmd_tree:
if _debug:
print('Warning: command %s found but not loaded' % cls_name)
kloger.warning('command %s found but not loaded' % cls_name)
return cls
name_terms = cls_name.split('_')
if not _update_best_match(name_terms, prefix):
if _debug:
print('Warning: %s failed to update_best_match' % cls_name)
kloger.warning('%s failed to update_best_match' % cls_name)
return None
global _best_match
......@@ -141,7 +140,7 @@ def command(cmd_tree, prefix='', descedants_depth=1):
if not cmd_tree.has_command(partial): # add partial path
cmd_tree.add_command(partial)
if _debug:
print('Warning: %s failed max_len test' % cls_name)
kloger.warning('%s failed max_len test' % cls_name)
return None
cls.description, sep, cls.long_description\
......@@ -153,11 +152,6 @@ def command(cmd_tree, prefix='', descedants_depth=1):
return wrap
def get_cmd_terms():
global command
return [term for term in command.func_defaults[0]\
if not term.startswith('-')]
cmd_spec_locations = [
'kamaki.cli.commands',
'kamaki.commands',
......@@ -166,6 +160,9 @@ cmd_spec_locations = [
'']
# Generic init auxiliary functions
def _setup_logging(silent=False, debug=False, verbose=False, include=False):
"""handle logging for clients package"""
......@@ -179,18 +176,23 @@ def _setup_logging(silent=False, debug=False, verbose=False, include=False):
if silent:
add_handler('', logging.CRITICAL)
elif debug:
return
if debug:
add_handler('requests', logging.INFO, prefix='* ')
add_handler('clients.send', logging.DEBUG, prefix='> ')
add_handler('clients.recv', logging.DEBUG, prefix='< ')
add_handler('kamaki', logging.DEBUG, prefix='(debug): ')
elif verbose:
add_handler('requests', logging.INFO, prefix='* ')
add_handler('clients.send', logging.INFO, prefix='> ')
add_handler('clients.recv', logging.INFO, prefix='< ')
add_handler('kamaki', logging.INFO, prefix='(i): ')
elif include:
add_handler('clients.recv', logging.INFO)
else:
add_handler('', logging.WARNING)
add_handler('kamaki', logging.WARNING, prefix='(warning): ')
global kloger
kloger = logging.getLogger('kamaki')
def _init_session(arguments):
......@@ -210,18 +212,6 @@ def _init_session(arguments):
_setup_logging(_silent, _debug, _verbose, _include)
def get_command_group(unparsed, arguments):
groups = arguments['config'].get_groups()
for term in unparsed:
if term.startswith('-'):
continue
if term in groups:
unparsed.remove(term)
return term
return None
return None
def _load_spec_module(spec, arguments, module):
spec_name = arguments['config'].get(spec, 'cli')
if spec_name is None:
......@@ -239,6 +229,7 @@ def _load_spec_module(spec, arguments, module):
def _groups_help(arguments):
global _debug
global kloger
descriptions = {}
for spec in arguments['config'].get_groups():
pkg = _load_spec_module(spec, arguments, '_commands')
......@@ -251,20 +242,41 @@ def _groups_help(arguments):
]
except AttributeError:
if _debug:
print('Warning: No description for %s' % spec)
kloger.warning('No description for %s' % spec)
try:
for cmd in cmds:
descriptions[cmd.name] = cmd.description
except TypeError:
if _debug:
print('Warning: no cmd specs in module %s' % spec)
kloger.warning('no cmd specs in module %s' % spec)
elif _debug:
print('Warning: Loading of %s cmd spec failed' % spec)
kloger.warning('Loading of %s cmd spec failed' % spec)
print('\nOptions:\n - - - -')
print_dict(descriptions)
def _print_subcommands_help(cmd):
def _load_all_commands(cmd_tree, arguments):
_config = arguments['config']
for spec in [spec for spec in _config.get_groups()\
if _config.get(spec, 'cli')]:
try:
spec_module = _load_spec_module(spec, arguments, '_commands')
spec_commands = getattr(spec_module, '_commands')
except AttributeError:
if _debug:
global kloger
kloger.warning('No valid description for %s' % spec)
continue
for spec_tree in spec_commands:
if spec_tree.name == spec:
cmd_tree.add_tree(spec_tree)
break
# Methods to be used by CLI implementations
def print_subcommands_help(cmd):
printout = {}
for subcmd in cmd.get_subcommands():
spec, sep, print_path = subcmd.path.partition('_')
......@@ -274,24 +286,28 @@ def _print_subcommands_help(cmd):
print_dict(printout)
def _update_parser_help(parser, cmd):
def update_parser_help(parser, cmd):
global _best_match
parser.syntax = parser.syntax.split('<')[0]
parser.syntax += ' '.join(_best_match)
description = ''
if cmd.is_command:
cls = cmd.get_class()
parser.syntax += ' ' + cls.syntax
parser.update_arguments(cls().arguments)
# arguments = cls().arguments
# update_arguments(parser, arguments)
description = getattr(cls, 'long_description', '')
description = description.strip()
else:
parser.syntax += ' <...>'
if cmd.has_description:
parser.parser.description = cmd.help
parser.parser.description = cmd.help\
+ (('\n%s' % description) if description else '')
else:
parser.parser.description = description
def _print_error_message(cli_err):
def print_error_message(cli_err):
errmsg = '%s' % cli_err
if cli_err.importance == 1:
errmsg = magenta(errmsg)
......@@ -300,20 +316,11 @@ def _print_error_message(cli_err):
elif cli_err.importance > 2:
errmsg = red(errmsg)
stdout.write(errmsg)
print_list(cli_err.details)
def _get_best_match_from_cmd_tree(cmd_tree, unparsed):
matched = [term for term in unparsed if not term.startswith('-')]
while matched:
try:
return cmd_tree.get_command('_'.join(matched))
except KeyError:
matched = matched[:-1]
return None
for errmsg in cli_err.details:
print('| %s' % errmsg)