Merge branch 'develop' into feature-network-api

Conflicts:
	kamaki/clients/network/__init__.py

Changelog

@@ -9,6 +9,9 @@ Changes:
 2. Make post_user_catalogs obsolete, but keep for one more version [#4337]
 3. Rename user commands for cached account requests as /user session [#4340]
 4. Remove max_theads from config, move control to threaded commands [#4617]
+5. Modify all commands [#4583]
+    New scheme for ALL <object> <verb> [object id] [--arguments]
+    e.g., file modidy --metadata-add=revier='Mr. Reviewer' /pithos/myfile.txt
 
 Features:
 

kamaki/cli/__init__.py

@@ -70,7 +70,7 @@ def _construct_command_syntax(cls):
         n = len(args) - len(spec.defaults or ())
         required = ' '.join(['<%s>' % _arg2syntax(x) for x in args[:n]])
         optional = ' '.join(['[%s]' % _arg2syntax(x) for x in args[n:]])
-        cls.syntax = ' '.join(x for x in [required, optional] if x)
+        cls.syntax = ' '.join([required, optional])
         if spec.varargs:
             cls.syntax += ' <%s ...>' % spec.varargs
 
@@ -548,7 +548,8 @@ def main():
         if parser.unparsed:
             run_one_cmd(exe, parser, cloud)
         elif _help:
-            parser.parser.print_help()
+            #parser.parser.print_help()
+            parser.print_help()
             _groups_help(parser.arguments)
         else:
             run_shell(exe, parser, cloud)

kamaki/cli/argument/__init__.py

@@ -33,10 +33,11 @@
 
 from kamaki.cli.config import Config
 from kamaki.cli.errors import CLISyntaxError, raiseCLIError
-from kamaki.cli.utils import split_input
+from kamaki.cli.utils import split_input, to_bytes
 
 from datetime import datetime as dtm
 from time import mktime
+from sys import stderr
 
 from logging import getLogger
 from argparse import ArgumentParser, ArgumentError
@@ -67,7 +68,7 @@ class Argument(object):
                     self, name)
             assert name.startswith('-'), msg
 
-        self.default = default if (default or self.arity) else False
+        self.default = default or None
 
     @property
     def value(self):
@@ -175,7 +176,7 @@ class FlagArgument(Argument):
     :value: true if set, false otherwise
     """
 
-    def __init__(self, help='', parsed_name=None, default=False):
+    def __init__(self, help='', parsed_name=None, default=None):
         super(FlagArgument, self).__init__(0, help, parsed_name, default)
 
 
@@ -227,6 +228,45 @@ class IntArgument(ValueArgument):
                 details=['Value %s not an int' % newvalue]))
 
 
+class DataSizeArgument(ValueArgument):
+    """Input: a string of the form <number><unit>
+    Output: the number of bytes
+    Units: B, KiB, KB, MiB, MB, GiB, GB, TiB, TB
+    """
+
+    @property
+    def value(self):
+        return getattr(self, '_value', self.default)
+
+    def _calculate_limit(self, user_input):
+        limit = 0
+        try:
+            limit = int(user_input)
+        except ValueError:
+            index = 0
+            digits = [str(num) for num in range(0, 10)] + ['.']
+            while user_input[index] in digits:
+                index += 1
+            limit = user_input[:index]
+            format = user_input[index:]
+            try:
+                return to_bytes(limit, format)
+            except Exception as qe:
+                msg = 'Failed to convert %s to bytes' % user_input,
+                raiseCLIError(qe, msg, details=[
+                    'Syntax: containerlimit set <limit>[format] [container]',
+                    'e.g.,: containerlimit set 2.3GB mycontainer',
+                    'Valid formats:',
+                    '(*1024): B, KiB, MiB, GiB, TiB',
+                    '(*1000): B, KB, MB, GB, TB'])
+        return limit
+
+    @value.setter
+    def value(self, new_value):
+        if new_value:
+            self._value = self._calculate_limit(new_value)
+
+
 class DateArgument(ValueArgument):
 
     DATE_FORMAT = '%a %b %d %H:%M:%S %Y'
@@ -282,10 +322,18 @@ class VersionArgument(FlagArgument):
 class RepeatableArgument(Argument):
     """A value argument that can be repeated"""
 
-    def __init__(self, help='', parsed_name=None, default=[]):
+    def __init__(self, help='', parsed_name=None, default=None):
         super(RepeatableArgument, self).__init__(
             -1, help, parsed_name, default)
 
+    @property
+    def value(self):
+        return getattr(self, '_value', [])
+
+    @value.setter
+    def value(self, newvalue):
+        self._value = newvalue
+
 
 class KeyValueArgument(Argument):
     """A Key=Value Argument that can be repeated
@@ -293,7 +341,7 @@ class KeyValueArgument(Argument):
     :syntax: --<arg> key1=value1 --<arg> key2=value2 ...
     """
 
-    def __init__(self, help='', parsed_name=None, default=[]):
+    def __init__(self, help='', parsed_name=None, default=None):
         super(KeyValueArgument, self).__init__(-1, help, parsed_name, default)
 
     @property
@@ -301,21 +349,23 @@ class KeyValueArgument(Argument):
         """
         :returns: (dict) {key1: val1, key2: val2, ...}
         """
-        return super(KeyValueArgument, self).value
+        return getattr(self, '_value', {})
 
     @value.setter
     def value(self, keyvalue_pairs):
         """
         :param keyvalue_pairs: (str) ['key1=val1', 'key2=val2', ...]
         """
-        self._value = getattr(self, '_value', self.value) or {}
-        try:
-            for pair in keyvalue_pairs:
-                key, sep, val = pair.partition('=')
-                assert sep, ' %s misses a "=" (usage: key1=val1 )\n' % (pair)
-                self._value[key] = val
-        except Exception as e:
-            raiseCLIError(e, 'KeyValueArgument Syntax Error')
+        if keyvalue_pairs:
+            self._value = self.value
+            try:
+                for pair in keyvalue_pairs:
+                    key, sep, val = pair.partition('=')
+                    assert sep, ' %s misses a "=" (usage: key1=val1 )\n' % (
+                        pair)
+                    self._value[key] = val
+            except Exception as e:
+                raiseCLIError(e, 'KeyValueArgument Syntax Error')
 
 
 class ProgressBarArgument(FlagArgument):
@@ -393,16 +443,37 @@ _arguments = dict(
 class ArgumentParseManager(object):
     """Manage (initialize and update) an ArgumentParser object"""
 
-    def __init__(self, exe, arguments=None):
+    def __init__(
+            self, exe,
+            arguments=None, required=None, syntax=None, description=None):
         """
         :param exe: (str) the basic command (e.g. 'kamaki')
 
         :param arguments: (dict) if given, overrides the global _argument as
             the parsers arguments specification
+        :param required: (list or tuple) an iterable of argument keys, denoting
+            which arguments are required. A tuple denoted an AND relation,
+            while a list denotes an OR relation e.g., ['a', 'b'] means that
+            either 'a' or 'b' is required, while ('a', 'b') means that both 'a'
+            and 'b' ar required.
+            Nesting is allowed e.g., ['a', ('b', 'c'), ['d', 'e']] means that
+            this command required either 'a', or both 'b' and 'c', or one of
+            'd', 'e'.
+            Repeated arguments are also allowed e.g., [('a', 'b'), ('a', 'c'),
+            ['b', 'c']] means that the command required either 'a' and 'b' or
+            'a' and 'c' or at least one of 'b', 'c' and could be written as
+            [('a', ['b', 'c']), ['b', 'c']]
+        :param syntax: (str) The basic syntax of the arguments. Default:
+            exe <cmd_group> [<cmd_subbroup> ...] <cmd>
+        :param description: (str) The description of the commands or ''
         """
         self.parser = ArgumentParser(
             add_help=False, formatter_class=RawDescriptionHelpFormatter)
-        self.syntax = '%s <cmd_group> [<cmd_subbroup> ...] <cmd>' % exe
+        self._exe = exe
+        self.syntax = syntax or (
+            '%s <cmd_group> [<cmd_subbroup> ...] <cmd>' % exe)
+        self.required = required
+        self.parser.description = description or ''
         if arguments:
             self.arguments = arguments
         else:
@@ -411,6 +482,71 @@ class ArgumentParseManager(object):
         self._parser_modified, self._parsed, self._unparsed = False, None, None
         self.parse()
 
+    @staticmethod
+    def required2list(required):
+        if isinstance(required, list) or isinstance(required, tuple):
+            terms = []
+            for r in required:
+                terms.append(ArgumentParseManager.required2list(r))
+            return list(set(terms).union())
+        return required
+
+    @staticmethod
+    def required2str(required, arguments, tab=''):
+        if isinstance(required, list):
+            return ' %sat least one of the following:\n%s' % (tab, ''.join(
+                [ArgumentParseManager.required2str(
+                    r, arguments, tab + '  ') for r in required]))
+        elif isinstance(required, tuple):
+            return ' %sall of the following:\n%s' % (tab, ''.join(
+                [ArgumentParseManager.required2str(
+                    r, arguments, tab + '  ') for r in required]))
+        else:
+            lt_pn, lt_all, arg = 23, 80, arguments[required]
+            tab2 = ' ' * lt_pn
+            ret = '%s%s' % (tab, ', '.join(arg.parsed_name))
+            if arg.arity != 0:
+                ret += ' %s' % required.upper()
+            ret = ('{:<%s}' % lt_pn).format(ret)
+            prefix = ('\n%s' % tab2) if len(ret) > lt_pn else ' '
+            cur = 0
+            while arg.help[cur:]:
+                next = cur + lt_all - lt_pn
+                ret += prefix
+                ret += ('{:<%s}' % (lt_all - lt_pn)).format(arg.help[cur:next])
+                cur, finish = next, '\n%s' % tab2
+            return ret + '\n'
+
+    @staticmethod
+    def _patch_with_required_args(arguments, required):
+        if isinstance(required, tuple):
+            return ' '.join([ArgumentParseManager._patch_with_required_args(
+                arguments, k) for k in required])
+        elif isinstance(required, list):
+            return '< %s >' % ' | '.join([
+                ArgumentParseManager._patch_with_required_args(
+                    arguments, k) for k in required])
+        arg = arguments[required]
+        return '/'.join(arg.parsed_name) + (
+            ' %s [...]' % required.upper() if arg.arity < 0 else (
+                ' %s' % required.upper() if arg.arity else ''))
+
+    def print_help(self, out=stderr):
+        if self.required:
+            tmp_args = dict(self.arguments)
+            for term in self.required2list(self.required):
+                tmp_args.pop(term)
+            tmp_parser = ArgumentParseManager(self._exe, tmp_args)
+            tmp_parser.syntax = self.syntax + self._patch_with_required_args(
+                self.arguments, self.required)
+            tmp_parser.parser.description = '%s\n\nrequired arguments:\n%s' % (
+                self.parser.description,
+                self.required2str(self.required, self.arguments))
+            tmp_parser.update_parser()
+            tmp_parser.parser.print_help()
+        else:
+            self.parser.print_help()
+
     @property
     def syntax(self):
         """The command syntax (useful for help messages, descriptions, etc)"""
@@ -465,15 +601,35 @@ class ArgumentParseManager(object):
         :param new_arguments: (dict)
         """
         if new_arguments:
-            assert isinstance(new_arguments, dict)
+            assert isinstance(new_arguments, dict), 'Arguments not in dict !!!'
             self._arguments.update(new_arguments)
             self.update_parser()
 
+    def _parse_required_arguments(self, required, parsed_args):
+        if not required:
+            return True
+        if isinstance(required, tuple):
+            for item in required:
+                if not self._parse_required_arguments(item, parsed_args):
+                    return False
+            return True
+        if isinstance(required, list):
+            for item in required:
+                if self._parse_required_arguments(item, parsed_args):
+                    return True
+            return False
+        return required in parsed_args
+
     def parse(self, new_args=None):
         """Parse user input"""
         try:
             pkargs = (new_args,) if new_args else ()
             self._parsed, unparsed = self.parser.parse_known_args(*pkargs)
+            parsed_args = [
+                k for k, v in vars(self._parsed).items() if v not in (None, )]
+            if not self._parse_required_arguments(self.required, parsed_args):
+                self.print_help()
+                raise CLISyntaxError('Missing required arguments')
         except SystemExit:
             raiseCLIError(CLISyntaxError('Argument Syntax Error'))
         for name, arg in self.arguments.items():

kamaki/cli/command_shell.py

@@ -33,7 +33,7 @@
 
 from cmd import Cmd
 from os import popen
-from sys import stdout
+from sys import stdout, stderr
 
 from kamaki.cli import exec_cmd, print_error_message, print_subcommands_help
 from kamaki.cli.argument import ArgumentParseManager
@@ -167,7 +167,7 @@ class Shell(Cmd):
         self.__dict__ = oldcontext
 
     @staticmethod
-    def _create_help_method(cmd_name, args, descr, syntax):
+    def _create_help_method(cmd_name, args, required, descr, syntax):
         tmp_args = dict(args)
         #tmp_args.pop('options', None)
         tmp_args.pop('cloud', None)
@@ -175,10 +175,9 @@ class Shell(Cmd):
         tmp_args.pop('verbose', None)
         tmp_args.pop('silent', None)
         tmp_args.pop('config', None)
-        help_parser = ArgumentParseManager(cmd_name, tmp_args)
-        help_parser.parser.description = descr
-        help_parser.syntax = syntax
-        return help_parser.parser.print_help
+        help_parser = ArgumentParseManager(
+            cmd_name, tmp_args, required, syntax=syntax, description=descr)
+        return help_parser.print_help
 
     def _register_command(self, cmd_path):
         cmd = self.cmd_tree.get_command(cmd_path)
@@ -200,6 +199,7 @@ class Shell(Cmd):
             if subcmd.is_command:  # exec command
                 try:
                     cls = subcmd.cmd_class
+                    cmd_parser.required = getattr(cls, 'required', None)
                     ldescr = getattr(cls, 'long_description', '')
                     if subcmd.path == 'history_run':
                         instance = cls(
@@ -211,11 +211,13 @@ class Shell(Cmd):
                             self.auth_base, self.cloud)
                     cmd_parser.update_arguments(instance.arguments)
                     cmd_parser.arguments = instance.arguments
+                    subpath = subcmd.path.split('_')[
+                        (len(cmd.path.split('_')) - 1):]
                     cmd_parser.syntax = '%s %s' % (
-                        subcmd.path.replace('_', ' '), cls.syntax)
+                        ' '.join(subpath), instance.syntax)
                     help_method = self._create_help_method(
                         cmd.name, cmd_parser.arguments,
-                        subcmd.help, cmd_parser.syntax)
+                        cmd_parser.required, subcmd.help, cmd_parser.syntax)
                     if '-h' in cmd_args or '--help' in cmd_args:
                         help_method()
                         if ldescr.strip():

kamaki/cli/commands/__init__.py

@@ -62,12 +62,19 @@ def addLogSettings(foo):
 
 class _command_init(object):
 
+    # self.arguments (dict) contains all non-positional arguments
+    # self.required (list or tuple) contains required argument keys
+    #     if it is a list, at least one of these arguments is required
+    #     if it is a tuple, all arguments are required
+    #     Lists and tuples can nest other lists and/or tuples
+
     def __init__(
             self,
             arguments={}, auth_base=None, cloud=None,
             _in=None, _out=None, _err=None):
         self._in, self._out, self._err = (
             _in or stdin, _out or stdout, _err or stderr)
+        self.required = getattr(self, 'required', None)
         if hasattr(self, 'arguments'):
             arguments.update(self.arguments)
         if isinstance(self, _optional_output_cmd):

kamaki/cli/commands/astakos.py

@@ -44,10 +44,27 @@ from kamaki.cli.argument import (
     FlagArgument, ValueArgument, IntArgument, CommaSeparatedListArgument)
 from kamaki.cli.utils import format_size
 
+#  Mandatory
+
 user_commands = CommandTree('user', 'Astakos/Identity API commands')
-admin_commands = CommandTree('admin', 'Astakos/Account API commands')
+quota_commands = CommandTree(
+    'quota', 'Astakos/Account API commands for quotas')
+resource_commands = CommandTree(
+    'resource', 'Astakos/Account API commands for resources')
 project_commands = CommandTree('project', 'Astakos project API commands')
-_commands = [user_commands, admin_commands, project_commands]
+
+
+#  Optional
+
+endpoint_commands = CommandTree(
+    'endpoint', 'Astakos/Account API commands for endpoints')
+service_commands = CommandTree('service', 'Astakos API commands for services')
+commission_commands = CommandTree(
+    'commission', 'Astakos API commands for commissions')
+
+_commands = [
+    user_commands, quota_commands, resource_commands, project_commands,
+    service_commands, commission_commands, endpoint_commands]
 
 
 def with_temp_token(foo):
@@ -146,8 +163,8 @@ class user_name2uuid(_init_synnefo_astakosclient, _optional_json):
         self._run(usernames=((username, ) + more_usernames))
 
 
-@command(user_commands)
-class user_quotas(_init_synnefo_astakosclient, _optional_json):
+@command(quota_commands)
+class quota_list(_init_synnefo_astakosclient, _optional_json):
     """Get user quotas"""
 
     _to_format = set(['cyclades.disk', 'pithos.diskspace', 'cyclades.ram'])
@@ -178,12 +195,7 @@ class user_quotas(_init_synnefo_astakosclient, _optional_json):
 
 
 @command(user_commands)
-class user_session(_init_synnefo_astakosclient):
-    """User session commands (cached identity calls for kamaki sessions)"""
-
-
-@command(user_commands)
-class user_session_info(_init_synnefo_astakosclient, _optional_json):
+class user_info(_init_synnefo_astakosclient, _optional_json):
     """Get info for (current) session user"""
 
     arguments = dict(
@@ -208,15 +220,15 @@ class user_session_info(_init_synnefo_astakosclient, _optional_json):
             raise CLIError(
                 'No user with %s in the cached session list' % msg, details=[
                     'To see all cached session users',
-                    '  /user session list',
+                    '  /user list',
                     'To authenticate and add a new user in the session list',
-                    '  /user session authenticate <new token>'])
+                    '  /user add <new token>'])
         self._print(self.auth_base.user_info(token), self.print_dict)
 
 
 @command(user_commands)
-class user_session_authenticate(_init_synnefo_astakosclient, _optional_json):
-    """Authenticate a user by token and update cache"""
+class user_add(_init_synnefo_astakosclient, _optional_json):
+    """Authenticate a user by token and add to kamaki session (cache)"""
 
     @errors.generic.all
     @errors.user.astakosclient
@@ -239,8 +251,8 @@ class user_session_authenticate(_init_synnefo_astakosclient, _optional_json):
 
 
 @command(user_commands)
-class user_session_list(_init_synnefo_astakosclient, _optional_json):
-    """List cached users"""
+class user_list(_init_synnefo_astakosclient, _optional_json):
+    """List (cached) session users"""
 
     arguments = dict(
         detail=FlagArgument('Detailed listing', ('-l', '--detail'))
@@ -258,8 +270,8 @@ class user_session_list(_init_synnefo_astakosclient, _optional_json):
 
 
 @command(user_commands)
-class user_session_select(_init_synnefo_astakosclient):
-    """Pick a user from the cached list to be the current session user"""
+class user_select(_init_synnefo_astakosclient):
+    """Select a user from the (cached) list as the current session user"""
 
     @errors.generic.all
     @errors.user.astakosclient
@@ -271,9 +283,9 @@ class user_session_select(_init_synnefo_astakosclient):
                 'No user with uuid %s in the cached session list' % uuid,
                 details=[
                     'To see all cached session users',
-                    '  /user session list',
+                    '  /user list',
                     'To authenticate and add a new user in the session list',
-                    '  /user session authenticate <new token>'])
+                    '  /user add <new token>'])
         if self.auth_base.token != first_token:
             self.auth_base.token = first_token
             msg = 'User with id %s is now the current session user.\n' % uuid
@@ -297,8 +309,8 @@ class user_session_select(_init_synnefo_astakosclient):
 
 
 @command(user_commands)
-class user_session_remove(_init_synnefo_astakosclient):
-    """Delete a user (token) from the cached list of session users"""
+class user_delete(_init_synnefo_astakosclient):
+    """Delete a user (token) from the (cached) list of session users"""
 
     @errors.generic.all
     @errors.user.astakosclient
@@ -306,20 +318,20 @@ class user_session_remove(_init_synnefo_astakosclient):
         if uuid == self.auth_base.user_term('id'):
             raise CLIError('Cannot remove current session user', details=[
                 'To see all cached session users',
-                '  /user session list',
+                '  /user list',
                 'To see current session user',
-                '  /user session info',
+                '  /user info',
                 'To select a different session user',
-                '  /user session select <user uuid>'])
+                '  /user select <user uuid>'])
         try:
             self.auth_base.remove_user(uuid)
         except KeyError:
             raise CLIError('No user with uuid %s in session list' % uuid,
                 details=[
                     'To see all cached session users',
-                    '  /user session list',
+                    '  /user list',
                     'To authenticate and add a new user in the session list',
-                    '  /user session authenticate <new token>'])
+                    '  /user add <new token>'])
         if self.ask_user(
                 'User is removed from current session, but will be restored in'
                 ' the next session. Remove the user from future sessions?'):
@@ -334,13 +346,8 @@ class user_session_remove(_init_synnefo_astakosclient):
 
 #  command admin
 
-@command(admin_commands)
-class admin_service(_init_synnefo_astakosclient):
-    """Manage commissions (special privileges required)"""
-
-
-@command(admin_commands)
-class admin_service_list(_init_synnefo_astakosclient, _optional_json):
+@command(service_commands)
+class service_list(_init_synnefo_astakosclient, _optional_json):
     """List available services"""
 
     @errors.generic.all
@@ -353,8 +360,8 @@ class admin_service_list(_init_synnefo_astakosclient, _optional_json):
         self._run()
 
 
-@command(admin_commands)
-class admin_service_uuid2username(_init_synnefo_astakosclient, _optional_json):
+@command(service_commands)
+class service_uuid2username(_init_synnefo_astakosclient, _optional_json):
     """Get service username(s) from uuid(s)"""
 
     @errors.generic.all
@@ -373,8 +380,8 @@ class admin_service_uuid2username(_init_synnefo_astakosclient, _optional_json):
         self._run([uuid] + list(more_uuids), token=service_token)
 
 
-@command(admin_commands)
-class admin_service_username2uuid(_init_synnefo_astakosclient, _optional_json):
+@command(service_commands)
+class service_username2uuid(_init_synnefo_astakosclient, _optional_json):
     """Get service uuid(s) from username(s)"""
 
     @errors.generic.all
@@ -393,8 +400,8 @@ class admin_service_username2uuid(_init_synnefo_astakosclient, _optional_json):
         self._run([usernames] + list(more_usernames), token=service_token)
 
 
-@command(admin_commands)
-class admin_service_quotas(_init_synnefo_astakosclient, _optional_json):
+@command(service_commands)
+class service_quotas(_init_synnefo_astakosclient, _optional_json):
     """Get service quotas"""
 
     arguments = dict(
@@ -412,13 +419,8 @@ class admin_service_quotas(_init_synnefo_astakosclient, _optional_json):
         self._run(token=service_token)
 
 
-@command(admin_commands)
-class admin_commission(_init_synnefo_astakosclient):
-    """Manage commissions (special privileges required)"""
-
-
-@command(admin_commands)
-class admin_commission_pending(_init_synnefo_astakosclient, _optional_json):
+@command(commission_commands)
+class commission_pending(_init_synnefo_astakosclient, _optional_json):
     """List pending commissions (special privileges required)"""
 
     @errors.generic.all
@@ -431,8 +433,8 @@ class admin_commission_pending(_init_synnefo_astakosclient, _optional_json):
         self._run()
 
 
-@command(admin_commands)
-class admin_commission_info(_init_synnefo_astakosclient, _optional_json):
+@command(commission_commands)
+class commission_info(_init_synnefo_astakosclient, _optional_json):
     """Get commission info (special privileges required)"""
 
     @errors.generic.all
@@ -447,8 +449,8 @@ class admin_commission_info(_init_synnefo_astakosclient, _optional_json):
         self._run(commission_id)
 
 
-@command(admin_commands)
-