Merge branch 'feature-customizable-output' into develop

Changelog

@@ -6,6 +6,9 @@ Changes:
 
 - Logs do not contain kamaki.clients pids by default [#4242]
 - Replace page_hold method with pydoc.pager [#4279]
+- Remove commands.pithos.DelimiterArgument, replace with ValueArgument
+    wherever it is used
+- Replace print methods with member print methods in _commands.* [#4292]
 
 Features:
 
@@ -13,4 +16,5 @@ Features:
 - Expand kamaki.cli unitests for the following packages ( [#4058] ):
   errors
 - Modify print methods in cli utils to use arbitary stream objects [#4288]
+- Implement wrapers for cli.utils print methods, in _commands [#4292]
 

kamaki/cli/__init__.py

@@ -32,7 +32,7 @@
 # or implied, of GRNET S.A.command
 
 import logging
-from sys import argv, exit, stdout
+from sys import argv, exit, stdout, stderr
 from os.path import basename, exists
 from inspect import getargspec
 
@@ -420,7 +420,7 @@ def update_parser_help(parser, cmd):
         cmd.help + ('\n' if description else '')) if cmd.help else description
 
 
-def print_error_message(cli_err):
+def print_error_message(cli_err, out=stderr):
     errmsg = '%s' % cli_err
     if cli_err.importance == 1:
         errmsg = magenta(errmsg)
@@ -428,9 +428,10 @@ def print_error_message(cli_err):
         errmsg = yellow(errmsg)
     elif cli_err.importance > 2:
         errmsg = red(errmsg)
-    stdout.write(errmsg)
+    out.write(errmsg)
     for errmsg in cli_err.details:
-        print('|  %s' % errmsg)
+        out.write('|  %s\n' % errmsg)
+        out.flush()
 
 
 def exec_cmd(instance, cmd_args, help_method):

kamaki/cli/commands/__init__.py

@@ -33,8 +33,10 @@
 
 from kamaki.cli.logger import get_logger
 from kamaki.cli.utils import (
-    print_json, print_items, filter_dicts_by_dict, stdout)
+    print_list, print_dict, print_json, print_items, ask_user,
+    filter_dicts_by_dict)
 from kamaki.cli.argument import FlagArgument, ValueArgument
+from sys import stdin, stdout, stderr
 
 log = get_logger(__name__)
 
@@ -60,7 +62,12 @@ def addLogSettings(foo):
 
 class _command_init(object):
 
-    def __init__(self, arguments={}, auth_base=None, cloud=None):
+    def __init__(
+            self,
+            arguments={}, auth_base=None, cloud=None,
+            _in=None, _out=None, _err=None):
+        self._in, self._out, self._err = (
+            _in or stdin, _out or stdout, _err or stderr)
         if hasattr(self, 'arguments'):
             arguments.update(self.arguments)
         if isinstance(self, _optional_output_cmd):
@@ -83,6 +90,38 @@ class _command_init(object):
         self.auth_base = auth_base or getattr(self, 'auth_base', None)
         self.cloud = cloud or getattr(self, 'cloud', None)
 
+    def write(self, s):
+        self._out.write(u'%s' % s)
+        self._out.flush()
+
+    def writeln(self, s=''):
+        self.write(u'%s\n' % s)
+
+    def error(self, s=''):
+        self._err.write(u'%s\n' % s)
+        self._err.flush()
+
+    def print_list(self, *args, **kwargs):
+        kwargs.setdefault('out', self._out)
+        return print_list(*args, **kwargs)
+
+    def print_dict(self, *args, **kwargs):
+        kwargs.setdefault('out', self._out)
+        return print_dict(*args, **kwargs)
+
+    def print_json(self, *args, **kwargs):
+        kwargs.setdefault('out', self._out)
+        return print_json(*args, **kwargs)
+
+    def print_items(self, *args, **kwargs):
+        kwargs.setdefault('out', self._out)
+        return print_items(*args, **kwargs)
+
+    def ask_user(self, *args, **kwargs):
+        kwargs.setdefault('user_in', self._in)
+        kwargs.setdefault('out', self._out)
+        return ask_user(*args, **kwargs)
+
     @DontRaiseKeyError
     def _custom_url(self, service):
         return self.config.get_cloud(self.cloud, '%s_url' % service)
@@ -142,6 +181,7 @@ class _command_init(object):
         """Try to get a progress bar, but do not raise errors"""
         try:
             progress_bar = self.arguments[arg]
+            progress_bar.file = self._err
             gen = progress_bar.get_generator(msg)
         except Exception:
             return (None, None)
@@ -209,9 +249,9 @@ class _optional_output_cmd(object):
 
     def _optional_output(self, r):
         if self['json_output']:
-            print_json(r)
+            print_json(r, out=self._out)
         elif self['with_output']:
-            print_items([r] if isinstance(r, dict) else r)
+            print_items([r] if isinstance(r, dict) else r, out=self._out)
 
 
 class _optional_json(object):
@@ -220,14 +260,11 @@ class _optional_json(object):
         json_output=FlagArgument('show headers in json', ('-j', '--json'))
     )
 
-    def _print(
-            self, output,
-            print_method=print_items, out=stdout,
-            **print_method_kwargs):
+    def _print(self, output, print_method=print_items, **print_method_kwargs):
         if self['json_output']:
-            print_json(output, out=out)
+            print_json(output, out=self._out)
         else:
-            print_method(output, out=out, **print_method_kwargs)
+            print_method(output, out=self._out, **print_method_kwargs)
 
 
 class _name_filter(object):

kamaki/cli/commands/astakos.py

@@ -37,7 +37,6 @@ from kamaki.cli.commands import (
     _command_init, errors, _optional_json, addLogSettings)
 from kamaki.cli.command_tree import CommandTree
 from kamaki.cli.errors import CLIBaseUrlError, CLIError
-from kamaki.cli.utils import print_dict, ask_user, stdout
 
 user_cmds = CommandTree('user', 'Astakos API commands')
 _commands = [user_cmds]
@@ -87,25 +86,25 @@ class user_authenticate(_user_init, _optional_json):
     (In case of another named cloud, use its name instead of default)
     """
 
-    @staticmethod
-    def _print_access(r, out=stdout):
-        print_dict(r['access'], out=out)
-
     @errors.generic.all
     @errors.user.authenticate
     def _run(self, custom_token=None):
         token_bu = self.client.token
         try:
             r = self.client.authenticate(custom_token)
-            if (token_bu != self.client.token and
-                    ask_user('Permanently save token as cloud.%s.token ?' % (
+            if (token_bu != self.client.token and self.ask_user(
+                    'Permanently save token as cloud.%s.token ?' % (
                         self.cloud))):
                 self._write_main_token(self.client.token)
         except Exception:
             #recover old token
             self.client.token = token_bu
             raise
-        self._print(r, self._print_access)
+
+        def _print_access(r, out):
+            self.print_dict(r['access'], out=out)
+
+        self._print(r, _print_access)
 
     def main(self, custom_token=None):
         super(self.__class__, self)._run()
@@ -131,7 +130,7 @@ class user_whoami(_user_init, _optional_json):
 
     @errors.generic.all
     def _run(self):
-        self._print(self.client.user_info(), print_dict)
+        self._print(self.client.user_info(), self.print_dict)
 
     def main(self):
         super(self.__class__, self)._run()
@@ -155,16 +154,17 @@ class user_set(_user_init, _optional_json):
             if user.get('id', None) in (uuid,):
                 ntoken = user['auth_token']
                 if ntoken == self.client.token:
-                    print('%s (%s) is already the session user' % (
+                    self.error('%s (%s) is already the session user' % (
                         self.client.user_term('name'),
                         self.client.user_term('id')))
                     return
                 self.client.token = user['auth_token']
-                print('Session user set to %s (%s)' % (
+                self.error('Session user set to %s (%s)' % (
                         self.client.user_term('name'),
                         self.client.user_term('id')))
-                if ask_user('Permanently make %s the main user?' % (
-                        self.client.user_term('name'))):
+                if self.ask_user(
+                        'Permanently make %s the main user?' % (
+                            self.client.user_term('name'))):
                     self._write_main_token(self.client.token)
                 return
         raise CLIError(

kamaki/cli/commands/config.py

@@ -31,8 +31,6 @@
 # interpreted as representing official policies, either expressed
 # or implied, of GRNET S.A.
 
-from sys import stdout
-
 from kamaki.cli import command
 from kamaki.cli.argument import FlagArgument
 from kamaki.cli.commands import _command_init, errors
@@ -50,12 +48,13 @@ about_options = '\nAbout options:\
     \n.   [group]\
     \n.   option=value\
     \n.   (more options can be set per group)\
+    \n.\
     \n. special case: named clouds.\
+    \n. example: cloud.demo.url\
     \n. E.g. for a cloud "demo":\
     \n.   [cloud "demo"]\
     \n.   url = <http://single/authentication/url/for/demo/site>\
-    \n.   token = <auth_token_from_demo_site>\
-    \n. which are referenced as cloud.demo.url , cloud.demo.token'
+    \n.   token = <auth_token_from_demo_site>'
 
 
 @command(config_cmds)
@@ -76,9 +75,9 @@ class config_list(_command_init):
                 if section in ('cloud',):
                     prefix = '%s.%s' % (section, key)
                     for k, v in val.items():
-                        print('%s.%s = %s' % (prefix, k, v))
+                        self.writeln('%s.%s = %s' % (prefix, k, v))
                 else:
-                    print('%s.%s = %s' % (section, key, val))
+                    self.writeln('%s.%s = %s' % (section, key, val))
 
     def main(self):
         self._run()
@@ -98,7 +97,7 @@ class config_get(_command_init):
             for k in self.config.keys(key):
                 match = True
                 if option != 'cloud':
-                    stdout.write('%s.%s =' % (option, k))
+                    self.write('%s.%s =' % (option, k))
                 self._run('%s.%s' % (option, k))
             if match:
                 return
@@ -110,9 +109,9 @@ class config_get(_command_init):
         value = get(section, key)
         if isinstance(value, dict):
             for k, v in value.items():
-                print('%s.%s.%s = %s' % (section, key, k, v))
+                self.writeln('%s.%s.%s = %s' % (section, key, k, v))
         elif value:
-            print(value)
+            self.writeln(value)
 
     def main(self, option):
         self._run(option)

kamaki/cli/commands/cyclades.py

@@ -31,10 +31,14 @@
 # interpreted as representing official policies, either expressed
 # or implied, of GRNET S.A.
 
+from base64 import b64encode
+from os.path import exists
+from io import StringIO
+from pydoc import pager
+
 from kamaki.cli import command
 from kamaki.cli.command_tree import CommandTree
-from kamaki.cli.utils import (
-    print_dict, remove_from_items, filter_dicts_by_dict, pager)
+from kamaki.cli.utils import remove_from_items, filter_dicts_by_dict
 from kamaki.cli.errors import raiseCLIError, CLISyntaxError, CLIBaseUrlError
 from kamaki.clients.cyclades import CycladesClient, ClientError
 from kamaki.cli.argument import FlagArgument, ValueArgument, KeyValueArgument
@@ -43,10 +47,6 @@ from kamaki.cli.commands import _command_init, errors, addLogSettings
 from kamaki.cli.commands import (
     _optional_output_cmd, _optional_json, _name_filter, _id_filter)
 
-from base64 import b64encode
-from os.path import exists
-from io import StringIO
-
 
 server_cmds = CommandTree('server', 'Cyclades/Compute API server commands')
 flavor_cmds = CommandTree('flavor', 'Cyclades/Compute API flavor commands')
@@ -59,71 +59,50 @@ about_authentication = '\nUser Authentication:\
     \n* to set authentication token: /config set cloud.<cloud>.token <token>'
 
 howto_personality = [
-    'Defines a file to be injected to VMs personality.',
-    'Personality value syntax: PATH,[SERVER_PATH,[OWNER,[GROUP,[MODE]]]]',
-    '  PATH: of local file to be injected',
+    'Defines a file to be injected to VMs file system.',
+    'syntax:  PATH,[SERVER_PATH,[OWNER,[GROUP,[MODE]]]]',
+    '  PATH: local file to be injected (relative or absolute)',
     '  SERVER_PATH: destination location inside server Image',
-    '  OWNER: user id of destination file owner',
-    '  GROUP: group id or name to own destination file',
+    '  OWNER: VMs user id of the remote destination file',
+    '  GROUP: VMs group id or name of the destination file',
     '  MODEL: permition in octal (e.g. 0777 or o+rwx)']
 
 
-class _server_wait(object):
+class _service_wait(object):
 
     wait_arguments = dict(
         progress_bar=ProgressBarArgument(
-            'do not show progress bar',
-            ('-N', '--no-progress-bar'),
-            False
-        )
+            'do not show progress bar', ('-N', '--no-progress-bar'), False)
     )
 
-    def _wait(self, server_id, currect_status):
+    def _wait(self, service, service_id, status_method, currect_status):
         (progress_bar, wait_cb) = self._safe_progress_bar(
-            'Server %s still in %s mode' % (server_id, currect_status))
+            '%s %s still in %s mode' % (service, service_id, currect_status))
 
         try:
-            new_mode = self.client.wait_server(
-                server_id,
-                currect_status,
-                wait_cb=wait_cb)
-        except Exception:
-            raise
+            new_mode = status_method(
+                service_id, currect_status, wait_cb=wait_cb)
         finally:
             self._safe_progress_bar_finish(progress_bar)
         if new_mode:
-            print('Server %s is now in %s mode' % (server_id, new_mode))
+            self.error('%s %s is now in %s mode' % (
+                service, service_id, new_mode))
         else:
             raiseCLIError(None, 'Time out')
 
 
-class _network_wait(object):
+class _server_wait(_service_wait):
 
-    wait_arguments = dict(
-        progress_bar=ProgressBarArgument(
-            'do not show progress bar',
-            ('-N', '--no-progress-bar'),
-            False
-        )
-    )
+    def _wait(self, server_id, currect_status):
+        super(_server_wait, self)._wait(
+            'Server', server_id, self.client.wait_server, currect_status)
 
-    def _wait(self, net_id, currect_status):
-        (progress_bar, wait_cb) = self._safe_progress_bar(
-            'Network %s still in %s mode' % (net_id, currect_status))
 
-        try:
-            new_mode = self.client.wait_network(
-                net_id,
-                currect_status,
-                wait_cb=wait_cb)
-        except Exception:
-            raise
-        finally:
-            self._safe_progress_bar_finish(progress_bar)
-        if new_mode:
-            print('Network %s is now in %s mode' % (net_id, new_mode))
-        else:
-            raiseCLIError(None, 'Time out')
+class _network_wait(_service_wait):
+
+    def _wait(self, net_id, currect_status):
+        super(_network_wait, self)._wait(
+            'Network', net_id, self.client.wait_network, currect_status)
 
 
 class _init_cyclades(_command_init):
@@ -131,14 +110,12 @@ class _init_cyclades(_command_init):
     @addLogSettings
     def _run(self, service='compute'):
         if getattr(self, 'cloud', None):
-            base_url = self._custom_url(service)\
-                or self._custom_url('cyclades')
+            base_url = self._custom_url(service) or self._custom_url(
+                'cyclades')
             if base_url:
-                token = self._custom_token(service)\
-                    or self._custom_token('cyclades')\
-                    or self.config.get_cloud('token')
-                self.client = CycladesClient(
-                    base_url=base_url, token=token)
+                token = self._custom_token(service) or self._custom_token(
+                    'cyclades') or self.config.get_cloud('token')
+                self.client = CycladesClient(base_url=base_url, token=token)
                 return
         else:
             self.cloud = 'default'
@@ -207,19 +184,12 @@ class server_list(_init_cyclades, _optional_json, _name_filter, _id_filter):
 
     def _filter_by_image(self, servers):
         iid = self['image_id']
-        new_servers = []
-        for srv in servers:
-            if srv['image']['id'] == iid:
-                new_servers.append(srv)
-        return new_servers
+        return [srv for srv in servers if srv['image']['id'] == iid]
 
     def _filter_by_flavor(self, servers):
         fid = self['flavor_id']
-        new_servers = []
-        for srv in servers:
-            if '%s' % srv['flavor']['id'] == '%s' % fid:
-                new_servers.append(srv)
-        return new_servers
+        return [srv for srv in servers if (
+            '%s' % srv['image']['id'] == '%s' % fid)]
 
     def _filter_by_metadata(self, servers):
         new_servers = []
@@ -300,7 +270,7 @@ class server_info(_init_cyclades, _optional_json):
         uuids = self._uuids2usernames([vm['user_id'], vm['tenant_id']])
         vm['user_id'] += ' (%s)' % uuids[vm['user_id']]
         vm['tenant_id'] += ' (%s)' % uuids[vm['tenant_id']]
-        self._print(vm, print_dict)
+        self._print(vm, self.print_dict)
 
     def main(self, server_id):
         super(self.__class__, self)._run()
@@ -327,8 +297,7 @@ class PersonalityArgument(KeyValueArgument):
                 raiseCLIError(
                     None,
                     '--personality: File %s does not exist' % path,
-                    importance=1,
-                    details=howto_personality)
+                    importance=1, details=howto_personality)
             self._value.append(dict(path=path))
             with open(path) as f:
                 self._value[i]['contents'] = b64encode(f.read())
@@ -366,7 +335,7 @@ class server_create(_init_cyclades, _optional_json, _server_wait):
         usernames = self._uuids2usernames([r['user_id'], r['tenant_id']])
         r['user_id'] += ' (%s)' % usernames[r['user_id']]
         r['tenant_id'] += ' (%s)' % usernames[r['tenant_id']]
-        self._print(r, print_dict)
+        self._print(r, self.print_dict)
         if self['wait']:
             self._wait(r['id'], r['status'])
 
@@ -519,7 +488,7 @@ class server_console(_init_cyclades, _optional_json):
     @errors.cyclades.server_id
     def _run(self, server_id):
         self._print(
-            self.client.get_server_console(int(server_id)), print_dict)
+            self.client.get_server_console(int(server_id)), self.print_dict)
 
     def main(self, server_id):
         super(self.__class__, self)._run()
@@ -581,7 +550,7 @@ class server_firewall_get(_init_cyclades):
     @errors.cyclades.connection
     @errors.cyclades.server_id
     def _run(self, server_id):
-        print(self.client.get_firewall_profile(server_id))
+        self.writeln(self.client.get_firewall_profile(server_id))
 
     def main(self, server_id):
         super(self.__class__, self)._run()
@@ -601,8 +570,7 @@ class server_addr(_init_cyclades, _optional_json):
     @errors.cyclades.server_id
     def _run(self, server_id):
         reply = self.client.list_server_nics(int(server_id))
-        self._print(
-            reply, with_enumeration=self['enum'] and len(reply) > 1)
+        self._print(reply, with_enumeration=self['enum'] and (reply) > 1)
 
     def main(self, server_id):
         super(self.__class__, self)._run()
@@ -624,7 +592,8 @@ class server_metadata_list(_init_cyclades, _optional_json):
     @errors.cyclades.metadata
     def _run(self, server_id, key=''):
         self._print(
-            self.client.get_server_metadata(int(server_id), key), print_dict)
+            self.client.get_server_metadata(int(server_id), key),
+            self.print_dict)
 
     def main(self, server_id, key=''):
         super(self.__class__, self)._run()
@@ -659,7 +628,7 @@ class server_metadata_set(_init_cyclades, _optional_json):
                         'key1=value1 key2=value2'])
         self._print(
             self.client.update_server_metadata(int(server_id), **metadata),
-            print_dict)
+            self.print_dict)
 
     def main(self, server_id, *key_equals_val):
         super(self.__class__, self)._run()
@@ -691,7 +660,8 @@ class server_stats(_init_cyclades, _optional_json):
     @errors.cyclades.connection
     @errors.cyclades.server_id
     def _run(self, server_id):
-        self._print(self.client.get_server_stats(int(server_id)), print_dict)
+        self._print(
+            self.client.get_server_stats(int(server_id)), self.print_dict)
 
     def main(self, server_id):
         super(self.__class__, self)._run()
@@ -786,7 +756,7 @@ class flavor_info(_init_cyclades, _optional_json):
     @errors.cyclades.flavor_id
     def _run(self, flavor_id):
         self._print(
-            self.client.get_flavor_details(int(flavor_id)), print_dict)
+            self.client.get_flavor_details(int(flavor_id)), self.print_dict)
 
     def main(self, flavor_id):
         super(self.__class__, self)._run()
@@ -819,7 +789,7 @@ class network_info(_init_cyclades, _optional_json):
     def _run(self, network_id):
         network = self.client.get_network_details(int(network_id))
         _add_name(self, network)
-        self._print(network, print_dict, exclude=('id'))
+        self._print(network, self.print_dict, exclude=('id'))
 
     def main(self, network_id):
         super(self.__class__, self)._run()
@@ -881,7 +851,7 @@ class network_list(_init_cyclades, _optional_json, _name_filter, _id_filter):
         for net in networks:
             v = net.get(key, None)
             if v:
-                net[key] += ' (%s)' % uuids[net[key]]
+                net[key] += ' (%s)' % uuids[v]
         return networks
 
     @errors.generic.all
@@ -951,7 +921,7 @@ class network_create(_init_cyclades, _optional_json, _network_wait):
             dhcp=self['dhcp'],
             type=self['type'])
         _add_name(self, r)
-        self._print(r, print_dict)
+        self._print(r, self.print_dict)
         if self['wait']:
             self._wait(r['id'], 'PENDING')
 
@@ -1044,8 +1014,7 @@ class network_disconnect(_init_cyclades):
         if not num_of_disconnected:
             raise ClientError(
                 'Network Interface %s not found on server %s' % (
-                    nic_id,
-                    server_id),
+                    nic_id, server_id),
                 status=404)
         print('Disconnected %s connections' % num_of_disconnected)
 
@@ -1112,7 +1081,7 @@ class server_ip_info(_init_cyclades, _optional_json):
     @errors.generic.all
     @errors.cyclades.connection
     def _run(self, ip):
-        self._print(self.client.get_floating_ip(ip), print_dict)
+        self._print(self.client.get_floating_ip(ip), self.print_dict)
 
     def main(self, ip):
         super(self.__class__, self)._run()
@@ -1123,9 +1092,7 @@ class server_ip_info(_init_cyclades, _optional_json):
 class server_ip_create(_init_cyclades, _optional_json):
     """Create a new floating IP"""
 
-    arguments = dict(
-        pool=ValueArgument('Source IP pool', ('--pool'), None)
-    )
+    arguments = dict(pool=ValueArgument('Source IP pool', ('--pool'), None))
 
     @errors.generic.all
     @errors.cyclades.connection

kamaki/cli/commands/history.py

@@ -111,7 +111,7 @@ class history_show(_init_history):
         ret = self.history.get(match_terms=self['match'], limit=self['limit'])
 
         if not cmd_ids:
-            print(''.join(ret))
+            self.print_list(ret)
             return
 
         num_list = []
@@ -122,7 +122,7 @@ class history_show(_init_history):
             try:
                 cur_id = int(cmd_id)
                 if cur_id:
-                    print(ret[cur_id - (1 if cur_id > 0 else 0)][:-1])
+                    self.writeln(ret[cur_id - (1 if cur_id > 0 else 0)][:-1])
             except IndexError as e2:
                 raiseCLIError(e2, 'Command id out of 1-%s range' % len(ret))
 
@@ -168,26 +168,21 @@ class history_run(_init_history):
     def _run_from_line(self, line):
         terms = split_input(line)
         cmd, args = self._cmd_tree.find_best_match(terms)
-        if not cmd.is_command:
-            return
-        try:
-            instance = cmd.cmd_class(