Commit 8c8a95b6 authored by Stavros Sachtouris's avatar Stavros Sachtouris
Browse files

Merge branch 'develop' into feature-network-api

Conflicts:
	kamaki/clients/network/__init__.py
parents 1ad949a7 e3f54dc0
......@@ -9,6 +9,9 @@ Changes:
2. Make post_user_catalogs obsolete, but keep for one more version [#4337]
3. Rename user commands for cached account requests as /user session [#4340]
4. Remove max_theads from config, move control to threaded commands [#4617]
5. Modify all commands [#4583]
New scheme for ALL <object> <verb> [object id] [--arguments]
e.g., file modidy --metadata-add=revier='Mr. Reviewer' /pithos/myfile.txt
Features:
......
......@@ -70,7 +70,7 @@ def _construct_command_syntax(cls):
n = len(args) - len(spec.defaults or ())
required = ' '.join(['<%s>' % _arg2syntax(x) for x in args[:n]])
optional = ' '.join(['[%s]' % _arg2syntax(x) for x in args[n:]])
cls.syntax = ' '.join(x for x in [required, optional] if x)
cls.syntax = ' '.join([required, optional])
if spec.varargs:
cls.syntax += ' <%s ...>' % spec.varargs
......@@ -548,7 +548,8 @@ def main():
if parser.unparsed:
run_one_cmd(exe, parser, cloud)
elif _help:
parser.parser.print_help()
#parser.parser.print_help()
parser.print_help()
_groups_help(parser.arguments)
else:
run_shell(exe, parser, cloud)
......
......@@ -33,10 +33,11 @@
from kamaki.cli.config import Config
from kamaki.cli.errors import CLISyntaxError, raiseCLIError
from kamaki.cli.utils import split_input
from kamaki.cli.utils import split_input, to_bytes
from datetime import datetime as dtm
from time import mktime
from sys import stderr
from logging import getLogger
from argparse import ArgumentParser, ArgumentError
......@@ -67,7 +68,7 @@ class Argument(object):
self, name)
assert name.startswith('-'), msg
self.default = default if (default or self.arity) else False
self.default = default or None
@property
def value(self):
......@@ -175,7 +176,7 @@ class FlagArgument(Argument):
:value: true if set, false otherwise
"""
def __init__(self, help='', parsed_name=None, default=False):
def __init__(self, help='', parsed_name=None, default=None):
super(FlagArgument, self).__init__(0, help, parsed_name, default)
......@@ -227,6 +228,45 @@ class IntArgument(ValueArgument):
details=['Value %s not an int' % newvalue]))
class DataSizeArgument(ValueArgument):
"""Input: a string of the form <number><unit>
Output: the number of bytes
Units: B, KiB, KB, MiB, MB, GiB, GB, TiB, TB
"""
@property
def value(self):
return getattr(self, '_value', self.default)
def _calculate_limit(self, user_input):
limit = 0
try:
limit = int(user_input)
except ValueError:
index = 0
digits = [str(num) for num in range(0, 10)] + ['.']
while user_input[index] in digits:
index += 1
limit = user_input[:index]
format = user_input[index:]
try:
return to_bytes(limit, format)
except Exception as qe:
msg = 'Failed to convert %s to bytes' % user_input,
raiseCLIError(qe, msg, details=[
'Syntax: containerlimit set <limit>[format] [container]',
'e.g.,: containerlimit set 2.3GB mycontainer',
'Valid formats:',
'(*1024): B, KiB, MiB, GiB, TiB',
'(*1000): B, KB, MB, GB, TB'])
return limit
@value.setter
def value(self, new_value):
if new_value:
self._value = self._calculate_limit(new_value)
class DateArgument(ValueArgument):
DATE_FORMAT = '%a %b %d %H:%M:%S %Y'
......@@ -282,10 +322,18 @@ class VersionArgument(FlagArgument):
class RepeatableArgument(Argument):
"""A value argument that can be repeated"""
def __init__(self, help='', parsed_name=None, default=[]):
def __init__(self, help='', parsed_name=None, default=None):
super(RepeatableArgument, self).__init__(
-1, help, parsed_name, default)
@property
def value(self):
return getattr(self, '_value', [])
@value.setter
def value(self, newvalue):
self._value = newvalue
class KeyValueArgument(Argument):
"""A Key=Value Argument that can be repeated
......@@ -293,7 +341,7 @@ class KeyValueArgument(Argument):
:syntax: --<arg> key1=value1 --<arg> key2=value2 ...
"""
def __init__(self, help='', parsed_name=None, default=[]):
def __init__(self, help='', parsed_name=None, default=None):
super(KeyValueArgument, self).__init__(-1, help, parsed_name, default)
@property
......@@ -301,21 +349,23 @@ class KeyValueArgument(Argument):
"""
:returns: (dict) {key1: val1, key2: val2, ...}
"""
return super(KeyValueArgument, self).value
return getattr(self, '_value', {})
@value.setter
def value(self, keyvalue_pairs):
"""
:param keyvalue_pairs: (str) ['key1=val1', 'key2=val2', ...]
"""
self._value = getattr(self, '_value', self.value) or {}
try:
for pair in keyvalue_pairs:
key, sep, val = pair.partition('=')
assert sep, ' %s misses a "=" (usage: key1=val1 )\n' % (pair)
self._value[key] = val
except Exception as e:
raiseCLIError(e, 'KeyValueArgument Syntax Error')
if keyvalue_pairs:
self._value = self.value
try:
for pair in keyvalue_pairs:
key, sep, val = pair.partition('=')
assert sep, ' %s misses a "=" (usage: key1=val1 )\n' % (
pair)
self._value[key] = val
except Exception as e:
raiseCLIError(e, 'KeyValueArgument Syntax Error')
class ProgressBarArgument(FlagArgument):
......@@ -393,16 +443,37 @@ _arguments = dict(
class ArgumentParseManager(object):
"""Manage (initialize and update) an ArgumentParser object"""
def __init__(self, exe, arguments=None):
def __init__(
self, exe,
arguments=None, required=None, syntax=None, description=None):
"""
:param exe: (str) the basic command (e.g. 'kamaki')
:param arguments: (dict) if given, overrides the global _argument as
the parsers arguments specification
:param required: (list or tuple) an iterable of argument keys, denoting
which arguments are required. A tuple denoted an AND relation,
while a list denotes an OR relation e.g., ['a', 'b'] means that
either 'a' or 'b' is required, while ('a', 'b') means that both 'a'
and 'b' ar required.
Nesting is allowed e.g., ['a', ('b', 'c'), ['d', 'e']] means that
this command required either 'a', or both 'b' and 'c', or one of
'd', 'e'.
Repeated arguments are also allowed e.g., [('a', 'b'), ('a', 'c'),
['b', 'c']] means that the command required either 'a' and 'b' or
'a' and 'c' or at least one of 'b', 'c' and could be written as
[('a', ['b', 'c']), ['b', 'c']]
:param syntax: (str) The basic syntax of the arguments. Default:
exe <cmd_group> [<cmd_subbroup> ...] <cmd>
:param description: (str) The description of the commands or ''
"""
self.parser = ArgumentParser(
add_help=False, formatter_class=RawDescriptionHelpFormatter)
self.syntax = '%s <cmd_group> [<cmd_subbroup> ...] <cmd>' % exe
self._exe = exe
self.syntax = syntax or (
'%s <cmd_group> [<cmd_subbroup> ...] <cmd>' % exe)
self.required = required
self.parser.description = description or ''
if arguments:
self.arguments = arguments
else:
......@@ -411,6 +482,71 @@ class ArgumentParseManager(object):
self._parser_modified, self._parsed, self._unparsed = False, None, None
self.parse()
@staticmethod
def required2list(required):
if isinstance(required, list) or isinstance(required, tuple):
terms = []
for r in required:
terms.append(ArgumentParseManager.required2list(r))
return list(set(terms).union())
return required
@staticmethod
def required2str(required, arguments, tab=''):
if isinstance(required, list):
return ' %sat least one of the following:\n%s' % (tab, ''.join(
[ArgumentParseManager.required2str(
r, arguments, tab + ' ') for r in required]))
elif isinstance(required, tuple):
return ' %sall of the following:\n%s' % (tab, ''.join(
[ArgumentParseManager.required2str(
r, arguments, tab + ' ') for r in required]))
else:
lt_pn, lt_all, arg = 23, 80, arguments[required]
tab2 = ' ' * lt_pn
ret = '%s%s' % (tab, ', '.join(arg.parsed_name))
if arg.arity != 0:
ret += ' %s' % required.upper()
ret = ('{:<%s}' % lt_pn).format(ret)
prefix = ('\n%s' % tab2) if len(ret) > lt_pn else ' '
cur = 0
while arg.help[cur:]:
next = cur + lt_all - lt_pn
ret += prefix
ret += ('{:<%s}' % (lt_all - lt_pn)).format(arg.help[cur:next])
cur, finish = next, '\n%s' % tab2
return ret + '\n'
@staticmethod
def _patch_with_required_args(arguments, required):
if isinstance(required, tuple):
return ' '.join([ArgumentParseManager._patch_with_required_args(
arguments, k) for k in required])
elif isinstance(required, list):
return '< %s >' % ' | '.join([
ArgumentParseManager._patch_with_required_args(
arguments, k) for k in required])
arg = arguments[required]
return '/'.join(arg.parsed_name) + (
' %s [...]' % required.upper() if arg.arity < 0 else (
' %s' % required.upper() if arg.arity else ''))
def print_help(self, out=stderr):
if self.required:
tmp_args = dict(self.arguments)
for term in self.required2list(self.required):
tmp_args.pop(term)
tmp_parser = ArgumentParseManager(self._exe, tmp_args)
tmp_parser.syntax = self.syntax + self._patch_with_required_args(
self.arguments, self.required)
tmp_parser.parser.description = '%s\n\nrequired arguments:\n%s' % (
self.parser.description,
self.required2str(self.required, self.arguments))
tmp_parser.update_parser()
tmp_parser.parser.print_help()
else:
self.parser.print_help()
@property
def syntax(self):
"""The command syntax (useful for help messages, descriptions, etc)"""
......@@ -465,15 +601,35 @@ class ArgumentParseManager(object):
:param new_arguments: (dict)
"""
if new_arguments:
assert isinstance(new_arguments, dict)
assert isinstance(new_arguments, dict), 'Arguments not in dict !!!'
self._arguments.update(new_arguments)
self.update_parser()
def _parse_required_arguments(self, required, parsed_args):
if not required:
return True
if isinstance(required, tuple):
for item in required:
if not self._parse_required_arguments(item, parsed_args):
return False
return True
if isinstance(required, list):
for item in required:
if self._parse_required_arguments(item, parsed_args):
return True
return False
return required in parsed_args
def parse(self, new_args=None):
"""Parse user input"""
try:
pkargs = (new_args,) if new_args else ()
self._parsed, unparsed = self.parser.parse_known_args(*pkargs)
parsed_args = [
k for k, v in vars(self._parsed).items() if v not in (None, )]
if not self._parse_required_arguments(self.required, parsed_args):
self.print_help()
raise CLISyntaxError('Missing required arguments')
except SystemExit:
raiseCLIError(CLISyntaxError('Argument Syntax Error'))
for name, arg in self.arguments.items():
......
......@@ -33,7 +33,7 @@
from cmd import Cmd
from os import popen
from sys import stdout
from sys import stdout, stderr
from kamaki.cli import exec_cmd, print_error_message, print_subcommands_help
from kamaki.cli.argument import ArgumentParseManager
......@@ -167,7 +167,7 @@ class Shell(Cmd):
self.__dict__ = oldcontext
@staticmethod
def _create_help_method(cmd_name, args, descr, syntax):
def _create_help_method(cmd_name, args, required, descr, syntax):
tmp_args = dict(args)
#tmp_args.pop('options', None)
tmp_args.pop('cloud', None)
......@@ -175,10 +175,9 @@ class Shell(Cmd):
tmp_args.pop('verbose', None)
tmp_args.pop('silent', None)
tmp_args.pop('config', None)
help_parser = ArgumentParseManager(cmd_name, tmp_args)
help_parser.parser.description = descr
help_parser.syntax = syntax
return help_parser.parser.print_help
help_parser = ArgumentParseManager(
cmd_name, tmp_args, required, syntax=syntax, description=descr)
return help_parser.print_help
def _register_command(self, cmd_path):
cmd = self.cmd_tree.get_command(cmd_path)
......@@ -200,6 +199,7 @@ class Shell(Cmd):
if subcmd.is_command: # exec command
try:
cls = subcmd.cmd_class
cmd_parser.required = getattr(cls, 'required', None)
ldescr = getattr(cls, 'long_description', '')
if subcmd.path == 'history_run':
instance = cls(
......@@ -211,11 +211,13 @@ class Shell(Cmd):
self.auth_base, self.cloud)
cmd_parser.update_arguments(instance.arguments)
cmd_parser.arguments = instance.arguments
subpath = subcmd.path.split('_')[
(len(cmd.path.split('_')) - 1):]
cmd_parser.syntax = '%s %s' % (
subcmd.path.replace('_', ' '), cls.syntax)
' '.join(subpath), instance.syntax)
help_method = self._create_help_method(
cmd.name, cmd_parser.arguments,
subcmd.help, cmd_parser.syntax)
cmd_parser.required, subcmd.help, cmd_parser.syntax)
if '-h' in cmd_args or '--help' in cmd_args:
help_method()
if ldescr.strip():
......
......@@ -62,12 +62,19 @@ def addLogSettings(foo):
class _command_init(object):
# self.arguments (dict) contains all non-positional arguments
# self.required (list or tuple) contains required argument keys
# if it is a list, at least one of these arguments is required
# if it is a tuple, all arguments are required
# Lists and tuples can nest other lists and/or tuples
def __init__(
self,
arguments={}, auth_base=None, cloud=None,
_in=None, _out=None, _err=None):
self._in, self._out, self._err = (
_in or stdin, _out or stdout, _err or stderr)
self.required = getattr(self, 'required', None)
if hasattr(self, 'arguments'):
arguments.update(self.arguments)
if isinstance(self, _optional_output_cmd):
......
......@@ -44,10 +44,27 @@ from kamaki.cli.argument import (
FlagArgument, ValueArgument, IntArgument, CommaSeparatedListArgument)
from kamaki.cli.utils import format_size
# Mandatory
user_commands = CommandTree('user', 'Astakos/Identity API commands')
admin_commands = CommandTree('admin', 'Astakos/Account API commands')
quota_commands = CommandTree(
'quota', 'Astakos/Account API commands for quotas')
resource_commands = CommandTree(
'resource', 'Astakos/Account API commands for resources')
project_commands = CommandTree('project', 'Astakos project API commands')
_commands = [user_commands, admin_commands, project_commands]
# Optional
endpoint_commands = CommandTree(
'endpoint', 'Astakos/Account API commands for endpoints')
service_commands = CommandTree('service', 'Astakos API commands for services')
commission_commands = CommandTree(
'commission', 'Astakos API commands for commissions')
_commands = [
user_commands, quota_commands, resource_commands, project_commands,
service_commands, commission_commands, endpoint_commands]
def with_temp_token(foo):
......@@ -146,8 +163,8 @@ class user_name2uuid(_init_synnefo_astakosclient, _optional_json):
self._run(usernames=((username, ) + more_usernames))
@command(user_commands)
class user_quotas(_init_synnefo_astakosclient, _optional_json):
@command(quota_commands)
class quota_list(_init_synnefo_astakosclient, _optional_json):
"""Get user quotas"""
_to_format = set(['cyclades.disk', 'pithos.diskspace', 'cyclades.ram'])
......@@ -178,12 +195,7 @@ class user_quotas(_init_synnefo_astakosclient, _optional_json):
@command(user_commands)
class user_session(_init_synnefo_astakosclient):
"""User session commands (cached identity calls for kamaki sessions)"""
@command(user_commands)
class user_session_info(_init_synnefo_astakosclient, _optional_json):
class user_info(_init_synnefo_astakosclient, _optional_json):
"""Get info for (current) session user"""
arguments = dict(
......@@ -208,15 +220,15 @@ class user_session_info(_init_synnefo_astakosclient, _optional_json):
raise CLIError(
'No user with %s in the cached session list' % msg, details=[
'To see all cached session users',
' /user session list',
' /user list',
'To authenticate and add a new user in the session list',
' /user session authenticate <new token>'])
' /user add <new token>'])
self._print(self.auth_base.user_info(token), self.print_dict)
@command(user_commands)
class user_session_authenticate(_init_synnefo_astakosclient, _optional_json):
"""Authenticate a user by token and update cache"""
class user_add(_init_synnefo_astakosclient, _optional_json):
"""Authenticate a user by token and add to kamaki session (cache)"""
@errors.generic.all
@errors.user.astakosclient
......@@ -239,8 +251,8 @@ class user_session_authenticate(_init_synnefo_astakosclient, _optional_json):
@command(user_commands)
class user_session_list(_init_synnefo_astakosclient, _optional_json):
"""List cached users"""
class user_list(_init_synnefo_astakosclient, _optional_json):
"""List (cached) session users"""
arguments = dict(
detail=FlagArgument('Detailed listing', ('-l', '--detail'))
......@@ -258,8 +270,8 @@ class user_session_list(_init_synnefo_astakosclient, _optional_json):
@command(user_commands)
class user_session_select(_init_synnefo_astakosclient):
"""Pick a user from the cached list to be the current session user"""
class user_select(_init_synnefo_astakosclient):
"""Select a user from the (cached) list as the current session user"""
@errors.generic.all
@errors.user.astakosclient
......@@ -271,9 +283,9 @@ class user_session_select(_init_synnefo_astakosclient):
'No user with uuid %s in the cached session list' % uuid,
details=[
'To see all cached session users',
' /user session list',
' /user list',
'To authenticate and add a new user in the session list',
' /user session authenticate <new token>'])
' /user add <new token>'])
if self.auth_base.token != first_token:
self.auth_base.token = first_token
msg = 'User with id %s is now the current session user.\n' % uuid
......@@ -297,8 +309,8 @@ class user_session_select(_init_synnefo_astakosclient):
@command(user_commands)
class user_session_remove(_init_synnefo_astakosclient):
"""Delete a user (token) from the cached list of session users"""
class user_delete(_init_synnefo_astakosclient):
"""Delete a user (token) from the (cached) list of session users"""
@errors.generic.all
@errors.user.astakosclient
......@@ -306,20 +318,20 @@ class user_session_remove(_init_synnefo_astakosclient):
if uuid == self.auth_base.user_term('id'):
raise CLIError('Cannot remove current session user', details=[
'To see all cached session users',
' /user session list',
' /user list',
'To see current session user',
' /user session info',
' /user info',
'To select a different session user',
' /user session select <user uuid>'])
' /user select <user uuid>'])
try:
self.auth_base.remove_user(uuid)
except KeyError:
raise CLIError('No user with uuid %s in session list' % uuid,
details=[
'To see all cached session users',
' /user session list',
' /user list',
'To authenticate and add a new user in the session list',
' /user session authenticate <new token>'])
' /user add <new token>'])
if self.ask_user(
'User is removed from current session, but will be restored in'
' the next session. Remove the user from future sessions?'):
......@@ -334,13 +346,8 @@ class user_session_remove(_init_synnefo_astakosclient):
# command admin
@command(admin_commands)
class admin_service(_init_synnefo_astakosclient):
"""Manage commissions (special privileges required)"""
@command(admin_commands)
class admin_service_list(_init_synnefo_astakosclient, _optional_json):
@command(service_commands)
class service_list(_init_synnefo_astakosclient, _optional_json):
"""List available services"""
@errors.generic.all
......@@ -353,8 +360,8 @@ class admin_service_list(_init_synnefo_astakosclient, _optional_json):
self._run()
@command(admin_commands)
class admin_service_uuid2username(_init_synnefo_astakosclient, _optional_json):
@command(service_commands)
class service_uuid2username(_init_synnefo_astakosclient, _optional_json):
"""Get service username(s) from uuid(s)"""
@errors.generic.all
......@@ -373,8 +380,8 @@ class admin_service_uuid2username(_init_synnefo_astakosclient, _optional_json):
self._run([uuid] + list(more_uuids), token=service_token)
@command(admin_commands)
class admin_service_username2uuid(_init_synnefo_astakosclient, _optional_json):
@command(service_commands)
class service_username2uuid(_init_synnefo_astakosclient, _optional_json):
"""Get service uuid(s) from username(s)"""
@errors.generic.all
......@@ -393,8 +400,8 @@ class admin_service_username2uuid(_init_synnefo_astakosclient, _optional_json):
self._run([usernames] + list(more_usernames), token=service_token)
@command(admin_commands)
class admin_service_quotas(_init_synnefo_astakosclient, _optional_json):
@command(service_commands)
class service_quotas(_init_synnefo_astakosclient, _optional_json):
"""Get service quotas"""
arguments = dict(
......@@ -412,13 +419,8 @@ class admin_service_quotas(_init_synnefo_astakosclient, _optional_json):
self._run(token=service_token)
@command(admin_commands)
class admin_commission(_init_synnefo_astakosclient):
"""Manage commissions (special privileges required)"""
@command(admin_commands)
class admin_commission_pending(_init_synnefo_astakosclient, _optional_json):
@command(commission_commands)
class commission_pending(_init_synnefo_astakosclient, _optional_json):
"""List pending commissions (special privileges required)"""
@errors.generic.all
......@@ -431,8 +433,8 @@ class admin_commission_pending(_init_synnefo_astakosclient, _optional_json):
self._run()
@command(admin_commands)
class admin_commission_info(_init_synnefo_astakosclient, _optional_json):
@command(commission_commands)
class commission_info(_init_synnefo_astakosclient, _optional_json):
"""Get commission info (special privileges required)"""
@errors.generic.all
......@@ -447,8 +449,8 @@ class admin_commission_info(_init_synnefo_astakosclient, _optional_json):
self._run(commission_id)
@command(admin_commands)