Commit 59eaf3b9 authored by Stavros Sachtouris's avatar Stavros Sachtouris
Browse files

Merge branch 'feature-global-pager' into develop

parents f5ff79d9 97d3537c
......@@ -5,10 +5,12 @@ Bug Fixes:
Changes:
- Logs do not contain kamaki.clients pids by default [#4242]
- Replace page_hold method with pydoc.pager [#4279]
Features:
- New config option log_pid (default: off) can allow pid-logging [#4242]
- Expand kamaki.cli unitests for the following packages ( [#4058] ):
errors
- Modify print methods in cli utils to use arbitary stream objects [#4288]
......@@ -34,12 +34,13 @@
from sys import stdout
from time import localtime, strftime
from os import path, makedirs, walk
from io import StringIO
from kamaki.cli import command
from kamaki.cli.command_tree import CommandTree
from kamaki.cli.errors import raiseCLIError, CLISyntaxError, CLIBaseUrlError
from kamaki.cli.utils import (
format_size, to_bytes, print_dict, print_items, page_hold, bold, ask_user,
format_size, to_bytes, print_dict, print_items, pager, bold, ask_user,
get_path_size, print_json, guess_mime_type)
from kamaki.cli.argument import FlagArgument, ValueArgument, IntArgument
from kamaki.cli.argument import KeyValueArgument, DateArgument
......@@ -354,9 +355,7 @@ class file_list(_file_container_command, _optional_json, _name_filter):
format=ValueArgument(
'format to parse until data (default: d/m/Y H:M:S )', '--format'),
shared=FlagArgument('show only shared', '--shared'),
more=FlagArgument(
'output results in pages (-n to set items per page, default 10)',
'--more'),
more=FlagArgument('read long results', '--more'),
exact_match=FlagArgument(
'Show only objects that match exactly with path',
'--exact-match'),
......@@ -367,7 +366,7 @@ class file_list(_file_container_command, _optional_json, _name_filter):
if self['json_output']:
print_json(object_list)
return
limit = int(self['limit']) if self['limit'] > 0 else len(object_list)
out = StringIO() if self['more'] else stdout
for index, obj in enumerate(object_list):
if self['exact_match'] and self.path and not (
obj['name'] == self.path or 'content_type' in obj):
......@@ -384,47 +383,46 @@ class file_list(_file_container_command, _optional_json, _name_filter):
isDir = False
size = format_size(obj['bytes'])
pretty_obj['bytes'] = '%s (%s)' % (obj['bytes'], size)
oname = bold(obj['name'])
oname = obj['name'] if self['more'] else bold(obj['name'])
prfx = ('%s%s. ' % (empty_space, index)) if self['enum'] else ''
if self['detail']:
print('%s%s' % (prfx, oname))
print_dict(pretty_obj, exclude=('name'))
print
out.writelines(u'%s%s\n' % (prfx, oname))
print_dict(pretty_obj, exclude=('name'), out=out)
out.writelines(u'\n')
else:
oname = '%s%9s %s' % (prfx, size, oname)
oname += '/' if isDir else ''
print(oname)
if self['more']:
page_hold(index, limit, len(object_list))
oname = u'%s%9s %s' % (prfx, size, oname)
oname += u'/' if isDir else u''
out.writelines(oname + u'\n')
if self['more']:
pager(out.getvalue())
def print_containers(self, container_list):
if self['json_output']:
print_json(container_list)
return
limit = int(self['limit']) if self['limit'] > 0\
else len(container_list)
out = StringIO() if self['more'] else stdout
for index, container in enumerate(container_list):
if 'bytes' in container:
size = format_size(container['bytes'])
prfx = ('%s. ' % (index + 1)) if self['enum'] else ''
cname = '%s%s' % (prfx, bold(container['name']))
_cname = container['name'] if (
self['more']) else bold(container['name'])
cname = u'%s%s' % (prfx, _cname)
if self['detail']:
print(cname)
out.writelines(cname + u'\n')
pretty_c = container.copy()
if 'bytes' in container:
pretty_c['bytes'] = '%s (%s)' % (container['bytes'], size)
print_dict(pretty_c, exclude=('name'))
print
print_dict(pretty_c, exclude=('name'), out=out)
out.writelines(u'\n')
else:
if 'count' in container and 'bytes' in container:
print('%s (%s, %s objects)' % (
cname,
size,
container['count']))
out.writelines(u'%s (%s, %s objects)\n' % (
cname, size, container['count']))
else:
print(cname)
if self['more']:
page_hold(index + 1, limit, len(container_list))
out.writelines(cname + '\n')
if self['more']:
pager(out.getvalue())
@errors.generic.all
@errors.pithos.connection
......@@ -1239,8 +1237,8 @@ class file_download(_file_container_command):
* download <container>:<path> <local dir> -R
will download all files on <container> prefixed as <path>,
to <local dir>/<full path> (or <local dir>\<full path> in windows)
* download <container>:<path> <local dir> --exact-match
will download only one file, exactly matching <path>
* download <container>:<path> <local dir>
will download only one file<path>
ATTENTION: to download cont:dir1/dir2/file there must exist objects
cont:dir1 and cont:dir1/dir2 of type application/directory
To create directory objects, use /file mkdir
......
......@@ -31,11 +31,11 @@
# interpreted as representing official policies, either expressed
# or implied, of GRNET S.A.
from sys import stdout
from sys import stdout, stdin
from re import compile as regex_compile
from time import sleep
from os import walk, path
from json import dumps
from pydoc import pager
from kamaki.cli.errors import raiseCLIError
......@@ -57,26 +57,6 @@ except ImportError:
suggest['ansicolors']['active'] = True
def _print(w):
"""Print wrapper is used to help unittests check what is printed"""
print w
def _write(w):
"""stdout.write wrapper is used to help unittests check what is printed"""
stdout.write(w)
def _flush():
"""stdout.flush wrapper is used to help unittests check what is called"""
stdout.flush()
def _readline():
"""raw_input wrapper is used to help unittests"""
return raw_input()
def suggest_missing(miss=None, exclude=[]):
global suggest
sgs = dict(suggest)
......@@ -131,22 +111,20 @@ def pretty_keys(d, delim='_', recursive=False):
return new_d
def print_json(data):
def print_json(data, out=stdout):
"""Print a list or dict as json in console
:param data: json-dumpable data
"""
_print(dumps(data, indent=INDENT_TAB))
def pretty_dict(d, *args, **kwargs):
print_dict(pretty_keys(d, *args, **kwargs))
:param out: Input/Output stream to dump values into
"""
out.writelines(unicode(dumps(data, indent=INDENT_TAB) + '\n'))
def print_dict(
d,
exclude=(), indent=0,
with_enumeration=False, recursive_enumeration=False):
with_enumeration=False, recursive_enumeration=False, out=stdout):
"""Pretty-print a dictionary object
<indent>key: <non iterable item>
<indent>key:
......@@ -163,6 +141,8 @@ def print_dict(
:param recursive_enumeration: (bool) recursively enumerate iterables (does
not enumerate 1st level keys)
:param out: Input/Output stream to dump values into
:raises CLIError: if preconditions fail
"""
assert isinstance(d, dict), 'print_dict input must be a dict'
......@@ -172,27 +152,27 @@ def print_dict(
k = ('%s' % k).strip()
if k in exclude:
continue
print_str = ' ' * indent
print_str += '%s.' % (i + 1) if with_enumeration else ''
print_str += '%s:' % k
print_str = u' ' * indent
print_str += u'%s.' % (i + 1) if with_enumeration else u''
print_str += u'%s:' % k
if isinstance(v, dict):
_print(print_str)
out.writelines(print_str + '\n')
print_dict(
v, exclude, indent + INDENT_TAB,
recursive_enumeration, recursive_enumeration)
recursive_enumeration, recursive_enumeration, out)
elif isinstance(v, list) or isinstance(v, tuple):
_print(print_str)
out.writelines(print_str + '\n')
print_list(
v, exclude, indent + INDENT_TAB,
recursive_enumeration, recursive_enumeration)
recursive_enumeration, recursive_enumeration, out)
else:
_print('%s %s' % (print_str, v))
out.writelines(u'%s %s\n' % (print_str, v))
def print_list(
l,
exclude=(), indent=0,
with_enumeration=False, recursive_enumeration=False):
with_enumeration=False, recursive_enumeration=False, out=stdout):
"""Pretty-print a list of items
<indent>key: <non iterable item>
<indent>key:
......@@ -209,6 +189,8 @@ def print_list(
:param recursive_enumeration: (bool) recursively enumerate iterables (does
not enumerate 1st level keys)
:param out: Input/Output stream to dump values into
:raises CLIError: if preconditions fail
"""
assert isinstance(l, list) or isinstance(l, tuple), (
......@@ -216,52 +198,35 @@ def print_list(
assert indent >= 0, 'print_list indent must be >= 0'
for i, item in enumerate(l):
print_str = ' ' * indent
print_str += '%s.' % (i + 1) if with_enumeration else ''
print_str = u' ' * indent
print_str += u'%s.' % (i + 1) if with_enumeration else u''
if isinstance(item, dict):
if with_enumeration:
_print(print_str)
out.writelines(print_str + '\n')
elif i and i < len(l):
_print('')
out.writelines(u'\n')
print_dict(
item, exclude,
indent + (INDENT_TAB if with_enumeration else 0),
recursive_enumeration, recursive_enumeration)
recursive_enumeration, recursive_enumeration, out)
elif isinstance(item, list) or isinstance(item, tuple):
if with_enumeration:
_print(print_str)
out.writelines(print_str + '\n')
elif i and i < len(l):
_print()
out.writelines(u'\n')
print_list(
item, exclude, indent + INDENT_TAB,
recursive_enumeration, recursive_enumeration)
recursive_enumeration, recursive_enumeration, out)
else:
item = ('%s' % item).strip()
if item in exclude:
continue
_print('%s%s' % (print_str, item))
def page_hold(index, limit, maxlen):
"""Check if there are results to show, and hold the page when needed
:param index: (int) > 0, index of current element
:param limit: (int) 0 < limit <= max, page hold if limit mod index == 0
:param maxlen: (int) Don't hold if index reaches maxlen
:returns: True if there are more to show, False if all results are shown
"""
if index >= maxlen:
return False
if index and index % limit == 0:
raw_input('(%s listed - %s more - "enter" to continue)' % (
index, maxlen - index))
return True
out.writelines(u'%s%s\n' % (print_str, item))
def print_items(
items, title=('id', 'name'),
with_enumeration=False, with_redundancy=False,
page_size=0):
with_enumeration=False, with_redundancy=False, out=stdout):
"""print dict or list items in a list, using some values as title
Objects of next level don't inherit enumeration (default: off) or titles
......@@ -273,38 +238,29 @@ def print_items(
:param with_redundancy: (boolean) values in title also appear on body
:param page_size: (int) show results in pages of page_size items, enter to
continue
:param out: Input/Output stream to dump values into
"""
if not items:
return
if not (isinstance(items, dict) or isinstance(items, list) or isinstance(
items, tuple)):
_print('%s' % items)
out.writelines(u'%s\n' % items)
return
page_size = int(page_size or 0)
try:
page_size = page_size if page_size > 0 else len(items)
except:
page_size = len(items)
num_of_pages = len(items) // page_size
num_of_pages += 1 if len(items) % page_size else 0
for i, item in enumerate(items):
if with_enumeration:
_write('%s. ' % (i + 1))
out.write(u'%s. ' % (i + 1))
if isinstance(item, dict):
item = dict(item)
title = sorted(set(title).intersection(item))
pick = item.get if with_redundancy else item.pop
header = ' '.join('%s' % pick(key) for key in title)
_print(bold(header))
print_dict(item, indent=INDENT_TAB)
header = u' '.join(u'%s' % pick(key) for key in title)
out.writelines(unicode(bold(header) + '\n'))
print_dict(item, indent=INDENT_TAB, out=out)
elif isinstance(item, list) or isinstance(item, tuple):
print_list(item, indent=INDENT_TAB)
print_list(item, indent=INDENT_TAB, out=out)
else:
_print(' %s' % item)
page_hold(i + 1, page_size, len(items))
out.writelines(u' %s\n' % item)
def format_size(size, decimal_factors=False):
......@@ -418,31 +374,19 @@ def split_input(line):
return terms
def ask_user(msg, true_resp=('y', )):
def ask_user(msg, true_resp=('y', ), out=stdout, user_in=stdin):
"""Print msg and read user response
:param true_resp: (tuple of chars)
:returns: (bool) True if reponse in true responses, False otherwise
"""
_write('%s [%s/N]: ' % (msg, ', '.join(true_resp)))
_flush()
user_response = _readline()
return user_response[0].lower() in true_resp
def spiner(size=None):
spins = ('/', '-', '\\', '|')
_write(' ')
size = size or -1
i = 0
while size - i:
_write('\b%s' % spins[i % len(spins)])
_flush()
i += 1
sleep(0.1)
yield
yield
yep = ', '.join(true_resp)
nope = '<not %s>' % yep if 'n' in true_resp or 'N' in true_resp else 'N'
out.write(u'%s [%s/%s]: ' % (msg, yep, nope))
out.flush()
user_response = user_in.readline()
return user_response[0].lower() in [s.lower() for s in true_resp]
def get_path_size(testpath):
......
......@@ -35,6 +35,7 @@ from unittest import TestCase
from tempfile import NamedTemporaryFile
from mock import patch, call
from itertools import product
from io import StringIO
class UtilsMethods(TestCase):
......@@ -64,17 +65,16 @@ class UtilsMethods(TestCase):
self.assertRaises(AssertionError, guess_mime_type, *args)
@patch('kamaki.cli.utils.dumps', return_value='(dumps output)')
@patch('kamaki.cli.utils._print')
def test_print_json(self, PR, JD):
def test_print_json(self, JD):
from kamaki.cli.utils import print_json, INDENT_TAB
print_json('some data')
out = StringIO()
print_json('some data', out)
JD.assert_called_once_with('some data', indent=INDENT_TAB)
PR.assert_called_once_with('(dumps output)')
self.assertEqual(out.getvalue(), u'(dumps output)\n')
@patch('kamaki.cli.utils._print')
def test_print_dict(self, PR):
def test_print_dict(self):
from kamaki.cli.utils import print_dict, INDENT_TAB
call_counter = 0
out = StringIO()
self.assertRaises(AssertionError, print_dict, 'non-dict think')
self.assertRaises(AssertionError, print_dict, {}, indent=-10)
for args in product(
......@@ -104,14 +104,14 @@ class UtilsMethods(TestCase):
with patch('kamaki.cli.utils.print_dict') as PD:
with patch('kamaki.cli.utils.print_list') as PL:
pd_calls, pl_calls = 0, 0
print_dict(*args)
exp_calls = []
print_dict(*args, out=out)
exp_calls = u''
for i, (k, v) in enumerate(d.items()):
if k in exclude:
continue
str_k = ' ' * indent
str_k += '%s.' % (i + 1) if with_enumeration else ''
str_k += '%s:' % k
str_k = u' ' * indent
str_k += u'%s.' % (i + 1) if with_enumeration else u''
str_k += u'%s:' % k
if isinstance(v, dict):
self.assertEqual(
PD.mock_calls[pd_calls],
......@@ -120,9 +120,10 @@ class UtilsMethods(TestCase):
exclude,
indent + INDENT_TAB,
recursive_enumeration,
recursive_enumeration))
recursive_enumeration,
out))
pd_calls += 1
exp_calls.append(call(str_k))
exp_calls += str_k + '\n'
elif isinstance(v, list) or isinstance(v, tuple):
self.assertEqual(
PL.mock_calls[pl_calls],
......@@ -131,19 +132,18 @@ class UtilsMethods(TestCase):
exclude,
indent + INDENT_TAB,
recursive_enumeration,
recursive_enumeration))
recursive_enumeration,
out))
pl_calls += 1
exp_calls.append(call(str_k))
exp_calls += str_k + '\n'
else:
exp_calls.append(call('%s %s' % (str_k, v)))
real_calls = PR.mock_calls[call_counter:]
call_counter = len(PR.mock_calls)
self.assertEqual(sorted(real_calls), sorted(exp_calls))
exp_calls += u'%s %s\n' % (str_k, v)
self.assertEqual(exp_calls, out.getvalue())
out = StringIO()
@patch('kamaki.cli.utils._print')
def test_print_list(self, PR):
def test_print_list(self):
from kamaki.cli.utils import print_list, INDENT_TAB
call_counter = 0
out = StringIO()
self.assertRaises(AssertionError, print_list, 'non-list non-tuple')
self.assertRaises(AssertionError, print_list, {}, indent=-10)
for args in product(
......@@ -159,16 +159,16 @@ class UtilsMethods(TestCase):
with patch('kamaki.cli.utils.print_dict') as PD:
with patch('kamaki.cli.utils.print_list') as PL:
pd_calls, pl_calls = 0, 0
print_list(*args)
exp_calls = []
print_list(*args, out=out)
exp_calls = ''
for i, v in enumerate(l):
str_v = ' ' * indent
str_v += '%s.' % (i + 1) if with_enumeration else ''
str_v = u' ' * indent
str_v += u'%s.' % (i + 1) if with_enumeration else u''
if isinstance(v, dict):
if with_enumeration:
exp_calls.append(call(str_v))
exp_calls += str_v + '\n'
elif i and i < len(l):
exp_calls.append(call())
exp_calls += u'\n'
self.assertEqual(
PD.mock_calls[pd_calls],
call(
......@@ -177,13 +177,14 @@ class UtilsMethods(TestCase):
indent + (
INDENT_TAB if with_enumeration else 0),
recursive_enumeration,
recursive_enumeration))
recursive_enumeration,
out))
pd_calls += 1
elif isinstance(v, list) or isinstance(v, tuple):
if with_enumeration:
exp_calls.append(call(str_v))
exp_calls += str_v + '\n'
elif i and i < len(l):
exp_calls.append(call())
exp_calls += u'\n'
self.assertEqual(
PL.mock_calls[pl_calls],
call(
......@@ -191,46 +192,20 @@ class UtilsMethods(TestCase):
exclude,
indent + INDENT_TAB,
recursive_enumeration,
recursive_enumeration))
recursive_enumeration,
out))
pl_calls += 1
elif ('%s' % v) in exclude:
continue
else:
exp_calls.append(call('%s%s' % (str_v, v)))
real_calls = PR.mock_calls[call_counter:]
call_counter = len(PR.mock_calls)
self.assertEqual(sorted(real_calls), sorted(exp_calls))
@patch('__builtin__.raw_input')
def test_page_hold(self, RI):
from kamaki.cli.utils import page_hold
ri_counter = 0
for args, expected in (
((0, 0, 0), False),
((1, 3, 10), True),
((3, 3, 10), True),
((5, 3, 10), True),
((6, 3, 10), True),
((10, 3, 10), False),
((11, 3, 10), False)):
self.assertEqual(page_hold(*args), expected)
index, limit, maxlen = args
if index and index < maxlen and index % limit == 0:
self.assertEqual(ri_counter + 1, len(RI.mock_calls))
self.assertEqual(RI.mock_calls[-1], call(
'(%s listed - %s more - "enter" to continue)' % (
index, maxlen - index)))
else:
self.assertEqual(ri_counter, len(RI.mock_calls))
ri_counter = len(RI.mock_calls)
exp_calls += u'%s%s\n' % (str_v, v)
self.assertEqual(out.getvalue(), exp_calls)
out = StringIO()
@patch('kamaki.cli.utils._print')
@patch('kamaki.cli.utils._write')
@patch('kamaki.cli.utils.print_dict')
@patch('kamaki.cli.utils.print_list')
@patch('kamaki.cli.utils.page_hold')
@patch('kamaki.cli.utils.bold', return_value='bold')
def test_print_items(self, bold, PH, PL, PD, WR, PR):
def test_print_items(self, bold, PL, PD):
from kamaki.cli.utils import print_items, INDENT_TAB
for args in product(
(
......@@ -239,52 +214,42 @@ class UtilsMethods(TestCase):
({'k': 1, 'id': 2}, [5, 6, 7], (8, 9), '10')),
(('id', 'name'), ('something', 2), ('lala', )),
(False, True),
(False, True),
(0, 1, 2, 10)):
items, title, with_enumeration, with_redundancy, page_size = args
wr_counter, pr_counter = len(WR.mock_calls), len(PR.mock_calls)
(False, True)):
items, title, with_enumeration, with_redundancy = args
pl_counter, pd_counter = len(PL.mock_calls), len(PD.mock_calls)
bold_counter, ph_counter = len(bold.mock_calls), len(PH.mock_calls)
print_items(*args)
bold_counter, out_counter = len(bold.mock_calls), 0
out = StringIO()
print_items(*args, out=out)
out.seek(0)
if not (isinstance(items, dict) or isinstance(