Apply devflow ways of handling versions

Devflow is a GRNET system for handling versions and packaging the verious
GRNET.gr Synnefo software piecies

Changelog

-CHANGELOG for version 0.6.4
+CHANGELOG for version 0.7
 
 1. Clean up CLI error handling code
 2. Unify and improve data size units presentation
 3. Ask for user permission at store-delete
-4. Intuitive semantics for store-move/copy/upload
+4. Intuitive semantics for store-move/copy/download/upload (modified syntax and clients)
 5. Dynamically limit max number of threads
 6. Various CLI and dependency fixes
\ No newline at end of file

kamaki/__init__.py

@@ -31,4 +31,4 @@
 # interpreted as representing official policies, either expressed
 # or implied, of GRNET S.A.
 
-__version__ = '0.6.4'
+from kamaki.version import __version__

kamaki/clients/commissioning/callpoint.py

@@ -36,7 +36,7 @@ from kamaki.clients.commissioning.specificator import CanonifyException
 from kamaki.clients.commissioning.exception import CorruptedError
 from kamaki.clients.commissioning.exception import InvalidDataError
 from kamaki.clients.commissioning.exception import ReturnButFail
-from kamaki.clients.commissioning.importing import  imp_module
+from kamaki.clients.commissioning.importing import imp_module
 
 from re import compile as re_compile, sub as re_sub
 
@@ -76,17 +76,18 @@ class Callpoint(object):
                 #raise ValueError(m)
                 call_func = getattr(self, call_name)
                 if not callable(call_func):
-                    m = (   "api spec '%s', method '%s' is not a "
-                            "callable attribute in callpoint '%s'" % 
-                            (   type(canonifier).__name__,
-                                call_name,
-                                type(self).__name       )           )
+                    m = ("api spec '%s', method '%s' is not a "\
+                            "callable attribute in callpoint '%s'" %\
+                            (type(canonifier).__name__,
+                            call_name,
+                            type(self).__name))
                     raise ValueError(m)
 
                 original_calls[call_name] = call_func
 
             def mk_call_func():
                 local_call_name = call_name
+
                 def call_func(**data):
                     return self.make_call(local_call_name, data)
 
@@ -114,7 +115,7 @@ class Callpoint(object):
     def make_call_from_json_description(self, json_description):
         try:
             description = self.json_loads(json_description)
-        except ValueError, e:
+        except ValueError:
             m = "Cannot load json description"
             raise self.InvalidDataError(m)
 
@@ -191,6 +192,7 @@ def mkcallargs(**kw):
 
 versiontag_pattern = re_compile('[^a-zA-Z0-9_-]')
 
+
 def mk_versiontag(version):
     if not version or version == 'v':
         return ''
@@ -211,8 +213,7 @@ def get_callpoint(pointname, version=None, automake=None, **kw):
     if not category or category not in ['clients', 'servers']:
         raise ValueError("invalid pointname '%s'" % (pointname,))
 
-    modname = ('%s.callpoint.API_Callpoint%s' 
-                                            % (pointname, versiontag))
+    modname = ('%s.callpoint.API_Callpoint%s' % (pointname, versiontag))
 
     try:
         API_Callpoint = imp_module(modname)
@@ -222,7 +223,8 @@ def get_callpoint(pointname, version=None, automake=None, **kw):
             raise
 
     if category != 'clients':
-        m = ("Can only auto-make callpoint in 'clients' not '%s'" % (category,))
+        m = (
+            "Can only auto-make callpoint in 'clients' not '%s'" % (category))
         raise ValueError(m)
 
     components = components[1:]
@@ -250,4 +252,3 @@ def get_callpoint(pointname, version=None, automake=None, **kw):
         api_spec = API_Spec()
 
     return AutoCallpoint
-

kamaki/clients/commissioning/exception.py

@@ -68,7 +68,7 @@ class CallError(Exception):
         args = None
         try:
             args = tuple(exc.args)
-        except (TypeError, AttributeError), e:
+        except (TypeError, AttributeError):
             pass
 
         if args is None:
@@ -87,7 +87,7 @@ class CallError(Exception):
             if 'error_args' in dictobj and 'call_error' in dictobj:
                 args = dictobj['error_args']
                 call_error = dictobj['call_error']
-        except TypeError, e:
+        except TypeError:
             pass
 
         if args is None:
@@ -100,25 +100,30 @@ class CallError(Exception):
         self = cls(*args, call_error=call_error, **kw)
         return self
 
+
 def register_exceptions(*exceptions):
     for exception in exceptions:
         if not issubclass(exception, CallError):
-            m = "Registering '%s': is not a CallError subclass" % (exception,) 
+            m = "Registering '%s': is not a CallError subclass" % (exception,)
             raise ValueError(m)
         CallError.exceptions[exception.__name__] = exception
 
+
 def register_exception(exc):
     register_exceptions(exc)
     return exc
 
+
 @register_exception
 class CorruptedError(CallError):
     pass
 
+
 @register_exception
 class InvalidDataError(CallError):
     pass
 
+
 class ReturnButFail(Exception):
     def __init__(self, retval=None):
         self.data = retval

kamaki/clients/commissioning/importing.py

@@ -35,6 +35,7 @@ from imp import find_module, load_module
 
 _modules = {}
 
+
 def imp_module(fullname):
     if fullname in _modules:
         return _modules[fullname]
@@ -77,5 +78,3 @@ def imp_module(fullname):
 
 def list_modules():
     return sorted(_modules.keys())
-
-

kamaki/clients/commissioning/specificator.py

@@ -46,6 +46,7 @@ try:
 except ImportError:
     from kamaki.clients.commissioning.utils.ordereddict import OrderedDict
 
+
 def shorts(s):
     if not isinstance(s, unicode):
         s = str(s)
@@ -59,6 +60,7 @@ def shorts(s):
 class CanonifyException(Exception):
     pass
 
+
 class SpecifyException(Exception):
     pass
 
@@ -159,15 +161,15 @@ class Canonical(object):
             joinchar = ',\n'
             padchar = '\n'
 
-        args = [a.tostring( depth=depth,
-                            showopts=showopts,
-                            multiline=multiline) for a in self.args]
-        args += [("%s=%s" %
-                    (k, v.tostring( depth=depth,
-                                    showopts=showopts,
-                                    multiline=multiline)))
-
-                                    for k, v in self.kw.items()]
+        args = [a.tostring(
+                    depth=depth,
+                    showopts=showopts,
+                    multiline=multiline) for a in self.args]
+        args += [("%s=%s" % (k, v.tostring(
+                                depth=depth,
+                                showopts=showopts,
+                                multiline=multiline)))
+                            for k, v in self.kw.items()]
         if showopts:
             args += [("%s=%s" % (k, str(v))) for k, v in self.opts.items()]
 
@@ -194,6 +196,7 @@ class Canonical(object):
     def _show(self):
         return self.name
 
+
 class Null(Canonical):
 
     def _check(self, item):
@@ -207,13 +210,13 @@ class Integer(Canonical):
     def _check(self, item):
         try:
             num = long(item)
-        except ValueError, e:
+        except ValueError:
             try:
                 num = long(item, 16)
             except Exception:
                 m = "%s: cannot convert '%s' to long" % (self, shorts(item))
                 raise CanonifyException(m)
-        except TypeError, e:
+        except TypeError:
             m = "%s: cannot convert '%s' to long" % (self, shorts(item))
             raise CanonifyException(m)
 
@@ -246,11 +249,7 @@ class Integer(Canonical):
         return long(minimum + r * (maximum - minimum))
 
 
-
-Serial = Integer(
-            classname   =   'Serial',
-            null        =   True,
-)
+Serial = Integer(classname='Serial', null=True)
 
 
 class Text(Canonical):
@@ -316,8 +315,7 @@ class Text(Canonical):
         matcher = self.matcher
         if matcher is not None:
             match = matcher.match(item)
-            if  (       match is None
-                    or  (match.start(), match.end()) != (0, itemlen)    ):
+            if ((not match) or (match.start(), match.end()) != (0, itemlen)):
 
                     m = ("%s: '%s' does not match '%s'"
                             % (self, shorts(item), self.pat))
@@ -344,7 +342,7 @@ class Text(Canonical):
 
         g = log(z, 2)
         r = random() * g
-        z = minlen + int(2**r)
+        z = minlen + int(2 ** r)
 
         s = u''
         for _ in xrange(z):
@@ -410,9 +408,7 @@ class Bytes(Canonical):
         matcher = self.matcher
         if matcher is not None:
             match = matcher.match(item)
-            if  (       match is None
-                    or  (match.start(), match.end()) != (0, itemlen)    ):
-
+            if ((not match) or (match.start(), match.end()) != (0, itemlen)):
                     m = ("%s: '%s' does not match '%s'"
                             % (self, shorts(item), self.pat))
                     raise CanonifyException(m)
@@ -438,7 +434,7 @@ class Bytes(Canonical):
 
         g = log(z, 2)
         r = random() * g
-        z = minlen + int(2**r)
+        z = minlen + int(2 ** r)
 
         s = u''
         for _ in xrange(z):
@@ -475,7 +471,7 @@ class ListOf(Canonical):
 
         try:
             items = iter(item)
-        except TypeError, e:
+        except TypeError:
             m = "%s: %s is not iterable" % (self, shorts(item))
             raise CanonifyException(m)
 
@@ -502,6 +498,7 @@ class ListOf(Canonical):
     def _show(self):
         return '[ ' + self.canonical.show() + ' ... ]'
 
+
 class Args(Canonical):
 
     def _unpack(self, item):
@@ -524,8 +521,8 @@ class Args(Canonical):
                 for i in range(position, arglen):
                     key = keys[i]
                     if not key in named_args.keys():
-                       position = i + 1
-                       break
+                        position = i + 1
+                        break
                 else:
                     m = "Formal arguments exhausted"
                     raise AssertionError(m)
@@ -535,8 +532,8 @@ class Args(Canonical):
 
     def _check(self, item):
         try:
-            arglist = OrderedDict(item).items()
-        except (TypeError, ValueError), e:
+            OrderedDict(item).items()
+        except (TypeError, ValueError):
             m = "%s: %s is not dict-able" % (self, shorts(item))
             raise CanonifyException(m)
 
@@ -569,7 +566,7 @@ class Tuple(Canonical):
     def _check(self, item):
         try:
             items = list(item)
-        except TypeError, e:
+        except TypeError:
             m = "%s: %s is not iterable" % (self, shorts(item))
             raise CanonifyException(m)
 
@@ -578,7 +575,11 @@ class Tuple(Canonical):
         zc = len(canonicals)
 
         if zi != zc:
-            m = "%s: expecting %d elements, not %d (%s)" % (self, zc, zi, str(items))
+            m = "%s: expecting %d elements, not %d (%s)" % (
+                self,
+                zc,
+                zi,
+                str(items))
             raise CanonifyException(m)
 
         g = (canonical(element) for canonical, element in zip(self.args, item))
@@ -598,6 +599,7 @@ class Tuple(Canonical):
         strings = [x for x in [c.show() for c in canonicals] if x]
         return '[ ' + ' '.join(strings) + ' ]'
 
+
 class Dict(Canonical):
 
     def _check(self, item):
@@ -731,7 +733,7 @@ class Specificator(object):
             deflen = len(defaults)
 
             if arglen != deflen:
-                a = (f.__name__, args[:arglen-deflen])
+                a = (f.__name__, args[:arglen - deflen])
                 m = "Unspecified arguments in '%s': %s" % a
                 raise SpecifyException(m)
 
@@ -758,4 +760,3 @@ class Specificator(object):
 
     def __call__(self):
         return self
-

kamaki/clients/commissioning/utils/__init__.py

-
+# Copyright 2012-2013 GRNET S.A. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or
+# without modification, are permitted provided that the following
+# conditions are met:
+#
+#   1. Redistributions of source code must retain the above
+#      copyright notice, self.list of conditions and the following
+#      disclaimer.
+#
+#   2. Redistributions in binary form must reproduce the above
+#      copyright notice, self.list of conditions and the following
+#      disclaimer in the documentation and/or other materials
+#      provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
+# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
+# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+#
+# The views and conclusions contained in the software and
+# documentation are those of the authors and should not be
+# interpreted as representing official policies, either expressed
+# or implied, of GRNET S.A.

kamaki/clients/commissioning/utils/argmap.py

@@ -160,6 +160,7 @@ def argmap_encode(obj, output):
         output(']')
 
     m = "Unsupported type '%s'" % (type(obj))
+    m += ''
 
 
 def argmap_decode(inputf, s=None):
@@ -216,7 +217,7 @@ def argmap_decode_args(inputf):
         if s == ']':
             if key is not None:
                 append((None, key))
-	    args.append(ARGMAP_MAGIC)
+            args.append(ARGMAP_MAGIC)
             return args, None
 
         if s == '=':
@@ -247,9 +248,10 @@ def argmap_check(obj):
         return ARGMAP_MAGIC in obj
     if hasattr(obj, '__len__'):
         length = len(obj)
-        return length and obj[length-1] == ARGMAP_MAGIC
+        return length and obj[length - 1] == ARGMAP_MAGIC
     return False
 
+
 def argmap_unzip_dict(argmap):
     if not hasattr(argmap, 'keys'):
         m = "argmap unzip dict: not a dict"
@@ -262,6 +264,7 @@ def argmap_unzip_dict(argmap):
     del kw[ARGMAP_MAGIC]
     return args, kw
 
+
 def argmap_unzip_list(argmap):
     if not argmap or argmap.pop() != ARGMAP_MAGIC:
         m = "argmap unzip list: magic not found"
@@ -277,6 +280,7 @@ def argmap_unzip_list(argmap):
             kw[k] = v
     return args, kw
 
+
 def argmap_unzip(argmap):
     if hasattr(argmap, 'keys'):
         return argmap_unzip_dict(argmap)
@@ -286,9 +290,11 @@ def argmap_unzip(argmap):
         m = "argmap: cannot unzip type %s" % (type(argmap),)
         raise ValueError(m)
 
+
 def argmap_zip_list(args, kw):
     return [(None, a) for a in args] + kw.items() + [ARGMAP_MAGIC]
 
+
 def argmap_zip_dict(args, kw):
     argmap = OrderedDict()
     argmap.update(kw)
@@ -298,22 +304,26 @@ def argmap_zip_dict(args, kw):
 
 argmap_zip = argmap_zip_list
 
+
 def argmap_list_to_dict(argmap):
     args, kw = argmap_unzip_list(argmap)
     kw[ARGMAP_MAGIC] = ARGMAP_MAGIC
     kw[None] = args
     return kw
 
+
 def argmap_dict_to_list(argmap):
     args, kw = argmap_unzip_dict(argmap)
     return args + kw.items() + [ARGMAP_MAGIC]
 
+
 def argmap_unpack_list(argmap):
     kw = argmap_list_to_dict(argmap)
     if len(kw) == 2:
         return kw[None]
     return kw
 
+
 def argmap_unpack_dict(argmap):
     if hasattr(argmap, 'keys') and callable(argmap.keys):
         return argmap_dict_to_list(argmap)[:-1]

kamaki/clients/commissioning/utils/clijson.py

@@ -39,11 +39,13 @@ keywords = set(['true', 'false', 'null'])
 unquoted = set('{}[]"\'0123456789')
 name_matcher = re.compile('^[\w @_.+-]+$', re.UNICODE)
 
+
 def is_name(token):
     if name_matcher.match(token):
         return 1
     return 0
 
+
 def quote(token, is_dict):
     if not token:
         return '""'
@@ -85,6 +87,8 @@ def clijson(argv):
     dictionary = 0
     token_join = None
 
+    dictionary += 0
+
     for t in tokens:
         t = t.strip()
 
@@ -118,4 +122,3 @@ if __name__ == '__main__':
     from sys import argv
 
     print clijson(argv[1:])
-

kamaki/clients/commissioning/utils/debug.py

@@ -38,6 +38,7 @@ from os import environ
 
 _logger = None
 
+
 def init_logger_file(name, level='DEBUG'):
     logger = logging.getLogger(name)
     handler = logging.FileHandler(name + '.log')
@@ -50,6 +51,7 @@ def init_logger_file(name, level='DEBUG'):
     _logger = logger
     return logger
 
+
 def init_logger_stderr(name, level='DEBUG'):
     logger = logging.getLogger(name)
     from sys import stderr
@@ -63,12 +65,14 @@ def init_logger_stderr(name, level='DEBUG'):
     _logger = logger
     return logger
 
+
 def debug(fmt, *args):
     global _logger
     if _logger is None:
         init_logger_stderr('logger', get_level())
     _logger.debug(fmt % args)
 
+
 def get_level(default='INFO'):
     try:
         return environ['DEBUG_LEVEL']

kamaki/clients/commissioning/utils/newname.py

@@ -34,10 +34,10 @@ from time import time
 
 _counter = 0
 
+
 def newname(prefix):
-    global _counter;
+    global _counter
     _counter += 1
     ident = id(locals())
     nonce = int(time() * 1000) + _counter
     return "%s%x%x" % (prefix, ident, nonce)
-

kamaki/clients/commissioning/utils/ordereddict.py

@@ -48,18 +48,18 @@ except ImportError:
 
 
 class OrderedDict(dict):
-    'Dictionary that remembers insertion order'
-    # An inherited dict maps keys to values.
-    # The inherited dict provides __getitem__, __len__, __contains__, and get.
-    # The remaining methods are order-aware.
-    # Big-O running times for all methods are the same as for regular
-    # dictionaries.
-
-    # The internal self.__map dictionary maps keys to links in a doubly linked 
-    # list.
-    # The circular doubly linked list starts and ends with a sentinel element.
-    # The sentinel element never gets deleted (this simplifies the algorithm).
-    # Each link is stored as a list of length three:  [PREV, NEXT, KEY].
+    """Dictionary that remembers insertion order
+    An inherited dict maps keys to values.
+    The inherited dict provides __getitem__, __len__, __contains__, and get.
+    The remaining methods are order-aware.
+    Big-O running times for all methods are the same as for regular
+    dictionaries.
+    The internal self.__map dictionary maps keys to links in a doubly linked
+    list.
+    The circular doubly linked list starts and ends with a sentinel element.
+    The sentinel element never gets deleted (this simplifies the algorithm).
+    Each link is stored as a list of length three:  [PREV, NEXT, KEY].
+    """
 
     def __init__(self, *args, **kwds):
         '''Initialize an ordered dictionary.  Signature is the same as for
@@ -128,10 +128,11 @@ class OrderedDict(dict):
         dict.clear(self)
 
     def popitem(self, last=True):
-        '''od.popitem() -> (k, v), return and remove a (key, value) pair.
-        Pairs are returned in LIFO order if last is true or FIFO order if false.
+        """od.popitem() -> (k, v), return and remove a (key, value) pair.
+        Pairs are returned in LIFO order if last is true or FIFO order if
+        false.
+        """
 
-        '''
         if not self:
             raise KeyError('dictionary is empty')
         root = self.__root
@@ -209,15 +210,16 @@ class OrderedDict(dict):
         for key, value in kwds.items():
             self[key] = value
 
-    __update = update  # let subclasses override update without breaking __init__
+    #  let subclasses override update without breaking __init__
+    __update = update
 
     __marker = object()
 
     def pop(self, key, default=__marker):
-        '''od.pop(k[,d]) -> v, remove specified key and return the corresponding value.
-        If key is not found, d is returned if given, otherwise KeyError is raised.
-
-        '''
+        """od.pop(k[,d]) -> v, remove specified key and return the
+        corresponding value. If key is not found, d is returned if given,
+        otherwise KeyError is raised.
+        """
         if key in self:
             result = self[key]
             del self[key]
@@ -227,14 +229,16 @@ class OrderedDict(dict):
         return default
 
     def setdefault(self, key, default=None):
-        'od.setdefault(k[,d]) -> od.get(k,d), also set od[k]=d if k not in od'
+        """od.setdefault(k[,d]) -> od.get(k,d), also set od[k]=d if k not in
+        od
+        """
         if key in self:
             return self[key]
         self[key] = default
         return default
 
     def __repr__(self, _repr_running={}):
-        'od.__repr__() <==> repr(od)'
+        """od.__repr__() <==> repr(od)"""
         call_key = id(self), _get_ident()
         if call_key in _repr_running:
             return '...'
@@ -247,7 +251,7 @@ class OrderedDict(dict):
             del _repr_running[call_key]
 
     def __reduce__(self):
-        'Return state information for pickling'
+        """Return state information for pickling"""
         items = [[k, self[k]] for k in self]
         inst_dict = vars(self).copy()
         for k in vars(OrderedDict()):
@@ -257,27 +261,26 @@ class OrderedDict(dict):
         return self.__class__, (items,)
 
     def copy(self):
-        'od.copy() -> a shallow copy of od'
+        """od.copy() -> a shallow copy of od"""
         return self.__class__(self)
 
     @classmethod
     def fromkeys(cls, iterable, value=None):
-        '''OD.fromkeys(S[, v]) -> New ordered dictionary with keys from S
+        """OD.fromkeys(S[, v]) -> New ordered dictionary with keys from S
         and values equal to v (which defaults to None).
-
-        '''
+        """
         d = cls()
         for key in iterable:
             d[key] = value
         return d
 
     def __eq__(self, other):
-        '''od.__eq__(y) <==> od==y.  Comparison to another OD is order-sensitive
-        while comparison to a regular mapping is order-insensitive.
-
-        '''
+        """od.__eq__(y) <==> od==y.  Comparison to another OD is
+        order-sensitive while comparison to a regular mapping is
+        order-insensitive.
+        """
         if isinstance(other, OrderedDict):
-            return len(self)==len(other) and self.items() == other.items()
+            return len(self) == len(other) and self.items() == other.items()
         return dict.__eq__(self, other)
 
     def __ne__(self, other):

kamaki/clients/quotaholder/api/quotaholder.py

@@ -71,7 +71,7 @@ class Positive(Integer):
         self.opts.update({'minimum': 1})
 
 
-QH_PRACTICALLY_INFINITE = 10**32
+QH_PRACTICALLY_INFINITE = 10 ** 32
 
 Serial = Positive(classname='Serial')
 
@@ -169,8 +169,15 @@ class QuotaholderAPI(Specificator):
         context=Context,
         get_holding=ListOf(Entity, Resource, Key)
     ):
-        holdings = ListOf(  Entity, Resource, Policy,
-                            Imported, Exported, Returned, Released, Flags   )
+        holdings = ListOf(
+            Entity,
+            Resource,
+            Policy,