Merge branch 'master' into develop

Bump version to 0.13.2next
Conflicts:
	kamaki/version.py
	version

Changelog

@@ -23,6 +23,17 @@ Features
 
 .. _Changelog-0.13:
 
+v0.13.2
+=======
+
+Bug Fixes
+---------
+
+* Fix delimiter handling when uploading/downloading in windows systems
+* Introduce a new command "kamaki scripts verifyfs" to verify and
+  repair affected containers
+* Create subnets without gateway IPs
+
 v0.13.1
 =======
 

kamaki/cli/cmds/network.py

@@ -372,6 +372,7 @@ class subnet_create(_NetworkInit, OptionalOutput):
             ' e.g., --alloc-pool=123.45.67.1,123.45.67.8',
             '--alloc-pool'),
         gateway=ValueArgument('Gateway IP', '--gateway'),
+        no_gateway=FlagArgument('Do not assign a gateway IP', '--no-gateway'),
         subnet_id=ValueArgument('The id for the subnet', '--id'),
         ipv6=FlagArgument('If set, IP version is set to 6, else 4', '--ipv6'),
         enable_dhcp=FlagArgument('Enable dhcp (default: off)', '--with-dhcp'),
@@ -383,10 +384,11 @@ class subnet_create(_NetworkInit, OptionalOutput):
     @errors.Generic.all
     @errors.Cyclades.connection
     def _run(self):
+        gateway = '' if self['no_gateway'] else self['gateway']
         try:
             net = self.client.create_subnet(
                 self['network_id'], self['cidr'],
-                self['name'], self['allocation_pools'], self['gateway'],
+                self['name'], self['allocation_pools'], gateway,
                 self['subnet_id'], self['ipv6'], self['enable_dhcp'])
         except ClientError as ce:
             if ce.status in (404, 400):
@@ -396,6 +398,11 @@ class subnet_create(_NetworkInit, OptionalOutput):
 
     def main(self):
         super(self.__class__, self)._run()
+        if self['gateway'] and self['no_gateway']:
+            raise CLIInvalidArgument('Conflicting arguments', details=[
+                'Arguments %s and %s cannot be used together' % (
+                    self.arguments['gateway'].lvalue,
+                    self.arguments['no_gateway'].lvalue)])
         self._run()
 
 

kamaki/cli/config/__init__.py

@@ -168,6 +168,7 @@ DEFAULTS = {
         'config_cli': 'config',
         'history_cli': 'history',
         'ignore_ssl': 'off',
+        'scripts_cli': 'contrib.scripts',
         'ca_certs': CACERTS_DEFAULT_PATH,
         #  Optional command specs:
         #  'service_cli': 'astakos'

kamaki/cli/contrib/__init__.py

+# Copyright 2015 GRNET S.A. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or
+# without modification, are permitted provided that the following
+# conditions are met:
+#
+#   1. Redistributions of source code must retain the above
+#      copyright notice, this list of conditions and the following
+#      disclaimer.
+#
+#   2. Redistributions in binary form must reproduce the above
+#      copyright notice, this list of conditions and the following
+#      disclaimer in the documentation and/or other materials
+#      provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
+# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
+# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+#
+# The views and conclusions contained in the software and
+# documentation are those of the authors and should not be
+# interpreted as representing official policies, either expressed
+# or implied, of GRNET S.A.

kamaki/cli/contrib/scripts.py

+# Copyright 2015 GRNET S.A. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or
+# without modification, are permitted provided that the following
+# conditions are met:
+#
+#   1. Redistributions of source code must retain the above
+#      copyright notice, this list of conditions and the following
+#      disclaimer.
+#
+#   2. Redistributions in binary form must reproduce the above
+#      copyright notice, this list of conditions and the following
+#      disclaimer in the documentation and/or other materials
+#      provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
+# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
+# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+#
+# The views and conclusions contained in the software and
+# documentation are those of the authors and should not be
+# interpreted as representing official policies, either expressed
+# or implied, of GRNET S.A.
+
+from datetime import date
+
+from kamaki.cli import command
+from kamaki.cli.errors import CLIError
+from kamaki.cli.cmdtree import CommandTree
+from kamaki.cli.cmds import errors, OptionalOutput
+from kamaki.cli.cmds.pithos import _PithosAccount
+from kamaki.cli.argument import FlagArgument
+
+scripts_cmds = CommandTree('scripts', 'Useful scripts')
+namespaces = [scripts_cmds, ]
+
+
+@command(scripts_cmds)
+class scripts_verifyfs(_PithosAccount, OptionalOutput):
+    """Verify/Fix the structure of directory objects inside a container"""
+
+    arguments = dict(
+        fix_conflicts=FlagArgument(
+            'Fix conflicting names by renaming them '
+            '(prepare the structure of directory objects to be consistent)',
+            '--fix-conflicts'),
+        fix_names=FlagArgument(
+            'Rename directory objects containing \\',
+            '--fix-dir-names'),
+        fix_missing=FlagArgument(
+            'Create missing directories objects',
+            '--fix-missing-dirs'),
+        yes=FlagArgument('Do not prompt for permission', '--yes'),
+    )
+
+    @errors.Generic.all
+    @errors.Pithos.connection
+    @errors.Pithos.container
+    def _run(self):
+        dirs, files, empty_files = [], [], []
+        result = self.client.container_get()
+        for o in result.json:
+            name = o['name']
+            if self.object_is_dir(o):
+                dirs.append(name)
+            elif o['bytes'] == 0:
+                empty_files.append(name)
+            else:
+                files.append(name)
+
+        # Find all directories with backslashes
+        wrong = set()
+        for d in dirs:
+            if '\\' in d:
+                wrong.add(d)
+
+        # Find all intermediate directories and see if a missing directory
+        # exists or if an intermediate directory conflicts with an existing
+        # object name
+        missing = set()
+        conflicts = set()
+        for n in files + dirs:
+            inter = n.split('/')
+            inter.pop()
+            d = []
+            for i in inter:
+                d.append(i)
+                p = '/'.join(d)
+                if p not in dirs:
+                    missing.add(p)
+                if p in files + empty_files:
+                    conflicts.add(p)
+
+        # First try to resolve conflicts
+        if self['fix_conflicts']:
+            for c in conflicts:
+                if self['yes'] or self.ask_user('Rename %s?' % c):
+                    backup = '%s_orig_%s' % (c, date.today().isoformat())
+                    # TODO: check if backup name already exists
+                    self.error(' * Renaming %s to %s' % (c, backup))
+                    self.client.move_object(
+                        src_container=self.client.container,
+                        src_object=c,
+                        dst_container=self.client.container,
+                        dst_object=backup)
+
+        elif conflicts:
+            raise CLIError(
+                'Conflicting object names found: %s' % conflicts,
+                details=[
+                    'They should be directory objects instead',
+                    'Use --fix-conflicts to rename them and prepare'
+                    ' the directory structure for further fix actions'])
+
+        # renames should take place after fixing conflicts
+        elif self['fix_names']:
+            for w in wrong:
+                correct = w.replace('\\', '/')
+                if self['yes'] or self.ask_user('Rename %s?' % w):
+                    self.error(' * Renaming %s to %s' % (w, correct))
+                    self.client.move_object(
+                        src_container=self.client.container,
+                        src_object=w,
+                        dst_container=self.client.container,
+                        dst_object=correct)
+        elif wrong:
+            raise CLIError(
+                'Directory objects with backslashes found: %s' % wrong,
+                details=['Use --fix-dir-names to sanitize them'])
+
+        # missing dirs should be created after fixing names
+        elif self['fix_missing']:
+            for d in missing:
+                if self['yes'] or self.ask_user('Create %s?' % d):
+                    self.error(' * Creating directory object %s' % d)
+                    self.client.create_directory(d)
+
+        elif missing:
+            raise CLIError(
+                'Missing directory objects found: %s' % missing,
+                details=['Use --fix-missing-dirs to create them'])
+
+    def main(self, container):
+        super(self.__class__, self)._run()
+        self.container, self.client.container = container, container
+        self._run()

kamaki/cli/utils/__init__.py

@@ -387,8 +387,10 @@ def ask_user(msg, true_resp=('y', ), **kwargs):
     msg = escape_ctrl_chars(msg).encode(pref_enc, 'replace')
     yep = yep.encode(pref_enc, 'replace')
     nope = nope.encode(pref_enc, 'replace')
-    user_response = raw_input(
+    response = raw_input(
         '%s [%s/%s]: ' % (msg, yep, nope))
+    # Pressing just enter gives an empty response!
+    user_response = response if response else 'N'
     return user_response[0].lower() in [s.lower() for s in true_resp]
 
 

kamaki/clients/network/__init__.py

@@ -123,8 +123,9 @@ class NetworkClient(NetworkRestClient, Waiter):
         :param name: (str) The subnet name
         :param allocation_pools: (list of dicts) start/end addresses of
             allocation pools: [{'start': ..., 'end': ...}, ...]
-        :param gateway_ip: (str)
-        :param subnet_id: (str)
+        :param gateway_ip: (str) Special cases:
+            None: server applies the default policy
+            empty iterable: no gateway IP on this subnet
         :param ipv6: (bool) ip_version == 6 if true else 4 (default)
         :param enable_dhcp: (bool)
         """
@@ -134,8 +135,8 @@ class NetworkClient(NetworkRestClient, Waiter):
             subnet['name'] = name
         if allocation_pools:
             subnet['allocation_pools'] = allocation_pools
-        if gateway_ip:
-            subnet['gateway_ip'] = gateway_ip
+        if gateway_ip is not None:
+            subnet['gateway_ip'] = gateway_ip or None
         if subnet_id:
             subnet['id'] = subnet_id
         if enable_dhcp not in (None, ):

kamaki/version.py

 
-__version__ = "0.13.1next"
+__version__ = "0.13.2next"
 __version_vcs_info__ = {
-    'branch': 'feature-wait',
-    'revid': '68cc68c',
-    'revno': 2492}
-__version_user_email__ = "saxtouri@admin.grnet.gr"
-__version_user_name__ = "Stavros Sachtouris"
+    'branch': 'develop',
+    'revid': 'ce8d24b',
+    'revno': 2506}
+__version_user_email__ = "skalkoto@grnet.gr"
+__version_user_name__ = "Nikos Skalkotos"

setup.py

 #!/usr/bin/env python
 
-# Copyright 2011-2013 GRNET S.A. All rights reserved.
+# Copyright 2011-2015 GRNET S.A. All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or
 # without modification, are permitted provided that the following
@@ -67,6 +67,7 @@ setup(
         'kamaki.cli.argument',
         'kamaki.cli.cmds',
         'kamaki.cli.cmdtree',
+        'kamaki.cli.contrib',
         'kamaki.clients',
         'kamaki.clients.utils',
         'kamaki.clients.astakos',

version

-0.13.1next
+0.13.2next