Commit 6764f588 authored by Stavros Sachtouris's avatar Stavros Sachtouris

Apply devflow ways of handling versions

Devflow is a GRNET system for handling versions and packaging the verious
GRNET.gr Synnefo software piecies
parent 8046291b
CHANGELOG for version 0.6.4
CHANGELOG for version 0.7
1. Clean up CLI error handling code
2. Unify and improve data size units presentation
3. Ask for user permission at store-delete
4. Intuitive semantics for store-move/copy/upload
4. Intuitive semantics for store-move/copy/download/upload (modified syntax and clients)
5. Dynamically limit max number of threads
6. Various CLI and dependency fixes
\ No newline at end of file
......@@ -31,4 +31,4 @@
# interpreted as representing official policies, either expressed
# or implied, of GRNET S.A.
__version__ = '0.6.4'
from kamaki.version import __version__
......@@ -36,7 +36,7 @@ from kamaki.clients.commissioning.specificator import CanonifyException
from kamaki.clients.commissioning.exception import CorruptedError
from kamaki.clients.commissioning.exception import InvalidDataError
from kamaki.clients.commissioning.exception import ReturnButFail
from kamaki.clients.commissioning.importing import imp_module
from kamaki.clients.commissioning.importing import imp_module
from re import compile as re_compile, sub as re_sub
......@@ -76,17 +76,18 @@ class Callpoint(object):
#raise ValueError(m)
call_func = getattr(self, call_name)
if not callable(call_func):
m = ( "api spec '%s', method '%s' is not a "
"callable attribute in callpoint '%s'" %
( type(canonifier).__name__,
call_name,
type(self).__name ) )
m = ("api spec '%s', method '%s' is not a "\
"callable attribute in callpoint '%s'" %\
(type(canonifier).__name__,
call_name,
type(self).__name))
raise ValueError(m)
original_calls[call_name] = call_func
def mk_call_func():
local_call_name = call_name
def call_func(**data):
return self.make_call(local_call_name, data)
......@@ -114,7 +115,7 @@ class Callpoint(object):
def make_call_from_json_description(self, json_description):
try:
description = self.json_loads(json_description)
except ValueError, e:
except ValueError:
m = "Cannot load json description"
raise self.InvalidDataError(m)
......@@ -191,6 +192,7 @@ def mkcallargs(**kw):
versiontag_pattern = re_compile('[^a-zA-Z0-9_-]')
def mk_versiontag(version):
if not version or version == 'v':
return ''
......@@ -211,8 +213,7 @@ def get_callpoint(pointname, version=None, automake=None, **kw):
if not category or category not in ['clients', 'servers']:
raise ValueError("invalid pointname '%s'" % (pointname,))
modname = ('%s.callpoint.API_Callpoint%s'
% (pointname, versiontag))
modname = ('%s.callpoint.API_Callpoint%s' % (pointname, versiontag))
try:
API_Callpoint = imp_module(modname)
......@@ -222,7 +223,8 @@ def get_callpoint(pointname, version=None, automake=None, **kw):
raise
if category != 'clients':
m = ("Can only auto-make callpoint in 'clients' not '%s'" % (category,))
m = (
"Can only auto-make callpoint in 'clients' not '%s'" % (category))
raise ValueError(m)
components = components[1:]
......@@ -250,4 +252,3 @@ def get_callpoint(pointname, version=None, automake=None, **kw):
api_spec = API_Spec()
return AutoCallpoint
......@@ -68,7 +68,7 @@ class CallError(Exception):
args = None
try:
args = tuple(exc.args)
except (TypeError, AttributeError), e:
except (TypeError, AttributeError):
pass
if args is None:
......@@ -87,7 +87,7 @@ class CallError(Exception):
if 'error_args' in dictobj and 'call_error' in dictobj:
args = dictobj['error_args']
call_error = dictobj['call_error']
except TypeError, e:
except TypeError:
pass
if args is None:
......@@ -100,25 +100,30 @@ class CallError(Exception):
self = cls(*args, call_error=call_error, **kw)
return self
def register_exceptions(*exceptions):
for exception in exceptions:
if not issubclass(exception, CallError):
m = "Registering '%s': is not a CallError subclass" % (exception,)
m = "Registering '%s': is not a CallError subclass" % (exception,)
raise ValueError(m)
CallError.exceptions[exception.__name__] = exception
def register_exception(exc):
register_exceptions(exc)
return exc
@register_exception
class CorruptedError(CallError):
pass
@register_exception
class InvalidDataError(CallError):
pass
class ReturnButFail(Exception):
def __init__(self, retval=None):
self.data = retval
......@@ -35,6 +35,7 @@ from imp import find_module, load_module
_modules = {}
def imp_module(fullname):
if fullname in _modules:
return _modules[fullname]
......@@ -77,5 +78,3 @@ def imp_module(fullname):
def list_modules():
return sorted(_modules.keys())
......@@ -46,6 +46,7 @@ try:
except ImportError:
from kamaki.clients.commissioning.utils.ordereddict import OrderedDict
def shorts(s):
if not isinstance(s, unicode):
s = str(s)
......@@ -59,6 +60,7 @@ def shorts(s):
class CanonifyException(Exception):
pass
class SpecifyException(Exception):
pass
......@@ -159,15 +161,15 @@ class Canonical(object):
joinchar = ',\n'
padchar = '\n'
args = [a.tostring( depth=depth,
showopts=showopts,
multiline=multiline) for a in self.args]
args += [("%s=%s" %
(k, v.tostring( depth=depth,
showopts=showopts,
multiline=multiline)))
for k, v in self.kw.items()]
args = [a.tostring(
depth=depth,
showopts=showopts,
multiline=multiline) for a in self.args]
args += [("%s=%s" % (k, v.tostring(
depth=depth,
showopts=showopts,
multiline=multiline)))
for k, v in self.kw.items()]
if showopts:
args += [("%s=%s" % (k, str(v))) for k, v in self.opts.items()]
......@@ -194,6 +196,7 @@ class Canonical(object):
def _show(self):
return self.name
class Null(Canonical):
def _check(self, item):
......@@ -207,13 +210,13 @@ class Integer(Canonical):
def _check(self, item):
try:
num = long(item)
except ValueError, e:
except ValueError:
try:
num = long(item, 16)
except Exception:
m = "%s: cannot convert '%s' to long" % (self, shorts(item))
raise CanonifyException(m)
except TypeError, e:
except TypeError:
m = "%s: cannot convert '%s' to long" % (self, shorts(item))
raise CanonifyException(m)
......@@ -246,11 +249,7 @@ class Integer(Canonical):
return long(minimum + r * (maximum - minimum))
Serial = Integer(
classname = 'Serial',
null = True,
)
Serial = Integer(classname='Serial', null=True)
class Text(Canonical):
......@@ -316,8 +315,7 @@ class Text(Canonical):
matcher = self.matcher
if matcher is not None:
match = matcher.match(item)
if ( match is None
or (match.start(), match.end()) != (0, itemlen) ):
if ((not match) or (match.start(), match.end()) != (0, itemlen)):
m = ("%s: '%s' does not match '%s'"
% (self, shorts(item), self.pat))
......@@ -344,7 +342,7 @@ class Text(Canonical):
g = log(z, 2)
r = random() * g
z = minlen + int(2**r)
z = minlen + int(2 ** r)
s = u''
for _ in xrange(z):
......@@ -410,9 +408,7 @@ class Bytes(Canonical):
matcher = self.matcher
if matcher is not None:
match = matcher.match(item)
if ( match is None
or (match.start(), match.end()) != (0, itemlen) ):
if ((not match) or (match.start(), match.end()) != (0, itemlen)):
m = ("%s: '%s' does not match '%s'"
% (self, shorts(item), self.pat))
raise CanonifyException(m)
......@@ -438,7 +434,7 @@ class Bytes(Canonical):
g = log(z, 2)
r = random() * g
z = minlen + int(2**r)
z = minlen + int(2 ** r)
s = u''
for _ in xrange(z):
......@@ -475,7 +471,7 @@ class ListOf(Canonical):
try:
items = iter(item)
except TypeError, e:
except TypeError:
m = "%s: %s is not iterable" % (self, shorts(item))
raise CanonifyException(m)
......@@ -502,6 +498,7 @@ class ListOf(Canonical):
def _show(self):
return '[ ' + self.canonical.show() + ' ... ]'
class Args(Canonical):
def _unpack(self, item):
......@@ -524,8 +521,8 @@ class Args(Canonical):
for i in range(position, arglen):
key = keys[i]
if not key in named_args.keys():
position = i + 1
break
position = i + 1
break
else:
m = "Formal arguments exhausted"
raise AssertionError(m)
......@@ -535,8 +532,8 @@ class Args(Canonical):
def _check(self, item):
try:
arglist = OrderedDict(item).items()
except (TypeError, ValueError), e:
OrderedDict(item).items()
except (TypeError, ValueError):
m = "%s: %s is not dict-able" % (self, shorts(item))
raise CanonifyException(m)
......@@ -569,7 +566,7 @@ class Tuple(Canonical):
def _check(self, item):
try:
items = list(item)
except TypeError, e:
except TypeError:
m = "%s: %s is not iterable" % (self, shorts(item))
raise CanonifyException(m)
......@@ -578,7 +575,11 @@ class Tuple(Canonical):
zc = len(canonicals)
if zi != zc:
m = "%s: expecting %d elements, not %d (%s)" % (self, zc, zi, str(items))
m = "%s: expecting %d elements, not %d (%s)" % (
self,
zc,
zi,
str(items))
raise CanonifyException(m)
g = (canonical(element) for canonical, element in zip(self.args, item))
......@@ -598,6 +599,7 @@ class Tuple(Canonical):
strings = [x for x in [c.show() for c in canonicals] if x]
return '[ ' + ' '.join(strings) + ' ]'
class Dict(Canonical):
def _check(self, item):
......@@ -731,7 +733,7 @@ class Specificator(object):
deflen = len(defaults)
if arglen != deflen:
a = (f.__name__, args[:arglen-deflen])
a = (f.__name__, args[:arglen - deflen])
m = "Unspecified arguments in '%s': %s" % a
raise SpecifyException(m)
......@@ -758,4 +760,3 @@ class Specificator(object):
def __call__(self):
return self
# Copyright 2012-2013 GRNET S.A. All rights reserved.
#
# Redistribution and use in source and binary forms, with or
# without modification, are permitted provided that the following
# conditions are met:
#
# 1. Redistributions of source code must retain the above
# copyright notice, self.list of conditions and the following
# disclaimer.
#
# 2. Redistributions in binary form must reproduce the above
# copyright notice, self.list of conditions and the following
# disclaimer in the documentation and/or other materials
# provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
#
# The views and conclusions contained in the software and
# documentation are those of the authors and should not be
# interpreted as representing official policies, either expressed
# or implied, of GRNET S.A.
......@@ -160,6 +160,7 @@ def argmap_encode(obj, output):
output(']')
m = "Unsupported type '%s'" % (type(obj))
m += ''
def argmap_decode(inputf, s=None):
......@@ -216,7 +217,7 @@ def argmap_decode_args(inputf):
if s == ']':
if key is not None:
append((None, key))
args.append(ARGMAP_MAGIC)
args.append(ARGMAP_MAGIC)
return args, None
if s == '=':
......@@ -247,9 +248,10 @@ def argmap_check(obj):
return ARGMAP_MAGIC in obj
if hasattr(obj, '__len__'):
length = len(obj)
return length and obj[length-1] == ARGMAP_MAGIC
return length and obj[length - 1] == ARGMAP_MAGIC
return False
def argmap_unzip_dict(argmap):
if not hasattr(argmap, 'keys'):
m = "argmap unzip dict: not a dict"
......@@ -262,6 +264,7 @@ def argmap_unzip_dict(argmap):
del kw[ARGMAP_MAGIC]
return args, kw
def argmap_unzip_list(argmap):
if not argmap or argmap.pop() != ARGMAP_MAGIC:
m = "argmap unzip list: magic not found"
......@@ -277,6 +280,7 @@ def argmap_unzip_list(argmap):
kw[k] = v
return args, kw
def argmap_unzip(argmap):
if hasattr(argmap, 'keys'):
return argmap_unzip_dict(argmap)
......@@ -286,9 +290,11 @@ def argmap_unzip(argmap):
m = "argmap: cannot unzip type %s" % (type(argmap),)
raise ValueError(m)
def argmap_zip_list(args, kw):
return [(None, a) for a in args] + kw.items() + [ARGMAP_MAGIC]
def argmap_zip_dict(args, kw):
argmap = OrderedDict()
argmap.update(kw)
......@@ -298,22 +304,26 @@ def argmap_zip_dict(args, kw):
argmap_zip = argmap_zip_list
def argmap_list_to_dict(argmap):
args, kw = argmap_unzip_list(argmap)
kw[ARGMAP_MAGIC] = ARGMAP_MAGIC
kw[None] = args
return kw
def argmap_dict_to_list(argmap):
args, kw = argmap_unzip_dict(argmap)
return args + kw.items() + [ARGMAP_MAGIC]
def argmap_unpack_list(argmap):
kw = argmap_list_to_dict(argmap)
if len(kw) == 2:
return kw[None]
return kw
def argmap_unpack_dict(argmap):
if hasattr(argmap, 'keys') and callable(argmap.keys):
return argmap_dict_to_list(argmap)[:-1]
......
......@@ -39,11 +39,13 @@ keywords = set(['true', 'false', 'null'])
unquoted = set('{}[]"\'0123456789')
name_matcher = re.compile('^[\w @_.+-]+$', re.UNICODE)
def is_name(token):
if name_matcher.match(token):
return 1
return 0
def quote(token, is_dict):
if not token:
return '""'
......@@ -85,6 +87,8 @@ def clijson(argv):
dictionary = 0
token_join = None
dictionary += 0
for t in tokens:
t = t.strip()
......@@ -118,4 +122,3 @@ if __name__ == '__main__':
from sys import argv
print clijson(argv[1:])
......@@ -38,6 +38,7 @@ from os import environ
_logger = None
def init_logger_file(name, level='DEBUG'):
logger = logging.getLogger(name)
handler = logging.FileHandler(name + '.log')
......@@ -50,6 +51,7 @@ def init_logger_file(name, level='DEBUG'):
_logger = logger
return logger
def init_logger_stderr(name, level='DEBUG'):
logger = logging.getLogger(name)
from sys import stderr
......@@ -63,12 +65,14 @@ def init_logger_stderr(name, level='DEBUG'):
_logger = logger
return logger
def debug(fmt, *args):
global _logger
if _logger is None:
init_logger_stderr('logger', get_level())
_logger.debug(fmt % args)
def get_level(default='INFO'):
try:
return environ['DEBUG_LEVEL']
......
......@@ -34,10 +34,10 @@ from time import time
_counter = 0
def newname(prefix):
global _counter;
global _counter
_counter += 1
ident = id(locals())
nonce = int(time() * 1000) + _counter
return "%s%x%x" % (prefix, ident, nonce)
......@@ -48,18 +48,18 @@ except ImportError:
class OrderedDict(dict):
'Dictionary that remembers insertion order'
# An inherited dict maps keys to values.
# The inherited dict provides __getitem__, __len__, __contains__, and get.
# The remaining methods are order-aware.
# Big-O running times for all methods are the same as for regular
# dictionaries.
# The internal self.__map dictionary maps keys to links in a doubly linked
# list.
# The circular doubly linked list starts and ends with a sentinel element.
# The sentinel element never gets deleted (this simplifies the algorithm).
# Each link is stored as a list of length three: [PREV, NEXT, KEY].
"""Dictionary that remembers insertion order
An inherited dict maps keys to values.
The inherited dict provides __getitem__, __len__, __contains__, and get.
The remaining methods are order-aware.
Big-O running times for all methods are the same as for regular
dictionaries.
The internal self.__map dictionary maps keys to links in a doubly linked
list.
The circular doubly linked list starts and ends with a sentinel element.
The sentinel element never gets deleted (this simplifies the algorithm).
Each link is stored as a list of length three: [PREV, NEXT, KEY].
"""
def __init__(self, *args, **kwds):
'''Initialize an ordered dictionary. Signature is the same as for
......@@ -128,10 +128,11 @@ class OrderedDict(dict):
dict.clear(self)
def popitem(self, last=True):
'''od.popitem() -> (k, v), return and remove a (key, value) pair.
Pairs are returned in LIFO order if last is true or FIFO order if false.
"""od.popitem() -> (k, v), return and remove a (key, value) pair.
Pairs are returned in LIFO order if last is true or FIFO order if
false.
"""
'''
if not self:
raise KeyError('dictionary is empty')
root = self.__root
......@@ -209,15 +210,16 @@ class OrderedDict(dict):
for key, value in kwds.items():
self[key] = value
__update = update # let subclasses override update without breaking __init__
# let subclasses override update without breaking __init__
__update = update
__marker = object()
def pop(self, key, default=__marker):
'''od.pop(k[,d]) -> v, remove specified key and return the corresponding value.
If key is not found, d is returned if given, otherwise KeyError is raised.
'''
"""od.pop(k[,d]) -> v, remove specified key and return the
corresponding value. If key is not found, d is returned if given,
otherwise KeyError is raised.
"""
if key in self:
result = self[key]
del self[key]
......@@ -227,14 +229,16 @@ class OrderedDict(dict):
return default
def setdefault(self, key, default=None):
'od.setdefault(k[,d]) -> od.get(k,d), also set od[k]=d if k not in od'
"""od.setdefault(k[,d]) -> od.get(k,d), also set od[k]=d if k not in
od
"""
if key in self:
return self[key]
self[key] = default
return default
def __repr__(self, _repr_running={}):
'od.__repr__() <==> repr(od)'
"""od.__repr__() <==> repr(od)"""
call_key = id(self), _get_ident()
if call_key in _repr_running:
return '...'
......@@ -247,7 +251,7 @@ class OrderedDict(dict):
del _repr_running[call_key]
def __reduce__(self):
'Return state information for pickling'
"""Return state information for pickling"""
items = [[k, self[k]] for k in self]
inst_dict = vars(self).copy()
for k in vars(OrderedDict()):
......@@ -257,27 +261,26 @@ class OrderedDict(dict):
return self.__class__, (items,)
def copy(self):
'od.copy() -> a shallow copy of od'
"""od.copy() -> a shallow copy of od"""
return self.__class__(self)
@classmethod
def fromkeys(cls, iterable, value=None):
'''OD.fromkeys(S[, v]) -> New ordered dictionary with keys from S
"""OD.fromkeys(S[, v]) -> New ordered dictionary with keys from S
and values equal to v (which defaults to None).
'''
"""
d = cls()
for key in iterable:
d[key] = value
return d
def __eq__(self, other):
'''od.__eq__(y) <==> od==y. Comparison to another OD is order-sensitive
while comparison to a regular mapping is order-insensitive.
'''
"""od.__eq__(y) <==> od==y. Comparison to another OD is
order-sensitive while comparison to a regular mapping is
order-insensitive.
"""
if isinstance(other, OrderedDict):
return len(self)==len(other) and self.items() == other.items()
return len(self) == len(other) and self.items() == other.items()
return dict.__eq__(self, other)
def __ne__(self, other):
......
......@@ -71,7 +71,7 @@ class Positive(Integer):
self.opts.update({'minimum': 1})
QH_PRACTICALLY_INFINITE = 10**32
QH_PRACTICALLY_INFINITE = 10 ** 32
Serial = Positive(classname='Serial')
......@@ -169,8 +169,15 @@ class QuotaholderAPI(Specificator):
context=Context,
get_holding=ListOf(Entity, Resource, Key)
):
holdings = ListOf( Entity, Resource, Policy,
Imported, Exported, Returned, Released, Flags )
holdings = ListOf(
Entity,
Resource,
Policy,