Merge branch 'master' into develop

Bump version to 0.13.2next
Conflicts:
	kamaki/version.py
	version

Changelog

@@ -23,6 +23,17 @@ Features
 
 .. _Changelog-0.13:
 
+v0.13.2
+=======
+
+Bug Fixes
+---------
+
+* Fix delimiter handling when uploading/downloading in windows systems
+* Introduce a new command "kamaki scripts verifyfs" to verify and
+  repair affected containers
+* Create subnets without gateway IPs
+
 v0.13.1
 =======
 

kamaki/cli/cmds/network.py

@@ -372,6 +372,7 @@ class subnet_create(_NetworkInit, OptionalOutput):
             ' e.g., --alloc-pool=123.45.67.1,123.45.67.8',
             '--alloc-pool'),
         gateway=ValueArgument('Gateway IP', '--gateway'),
+        no_gateway=FlagArgument('Do not assign a gateway IP', '--no-gateway'),
         subnet_id=ValueArgument('The id for the subnet', '--id'),
         ipv6=FlagArgument('If set, IP version is set to 6, else 4', '--ipv6'),
         enable_dhcp=FlagArgument('Enable dhcp (default: off)', '--with-dhcp'),
@@ -383,10 +384,11 @@ class subnet_create(_NetworkInit, OptionalOutput):
     @errors.Generic.all
     @errors.Cyclades.connection
     def _run(self):
+        gateway = '' if self['no_gateway'] else self['gateway']
         try:
             net = self.client.create_subnet(
                 self['network_id'], self['cidr'],
-                self['name'], self['allocation_pools'], self['gateway'],
+                self['name'], self['allocation_pools'], gateway,
                 self['subnet_id'], self['ipv6'], self['enable_dhcp'])
         except ClientError as ce:
             if ce.status in (404, 400):
@@ -396,6 +398,11 @@ class subnet_create(_NetworkInit, OptionalOutput):
 
     def main(self):
         super(self.__class__, self)._run()
+        if self['gateway'] and self['no_gateway']:
+            raise CLIInvalidArgument('Conflicting arguments', details=[
+                'Arguments %s and %s cannot be used together' % (
+                    self.arguments['gateway'].lvalue,
+                    self.arguments['no_gateway'].lvalue)])
         self._run()
 
 

kamaki/cli/cmds/pithos.py

@@ -519,10 +519,31 @@ class file_delete(_PithosContainer):
         self.client.get_object_info(self.path)
         if self['yes'] or self.ask_user(
                 'Delete /%s/%s ?' % (self.container, self.path)):
-            self.client.del_object(
-                self.path,
-                until=self['until_date'],
-                delimiter='/' if self['recursive'] else self['delimiter'])
+            # See if any objects exist under prefix
+            # Add a trailing / to object's name
+            prefix = self.path.rstrip('/') + '/'
+            result = self.client.container_get(prefix=prefix)
+
+            if result.json:
+                count = len(result.json)
+                self.error(' * %d other object(s) with %s as prefix found' %
+                    (count, prefix))
+
+                if self['recursive']:
+                    msg = 'The above %d object(s) will be deleted, too' % \
+                        count
+                else:
+                    msg = 'The above %d object(s) will be preserved,' \
+                        ' but the directory structure' \
+                        ' will become inconsistent' % count
+
+                self.error(' * %s!' % msg)
+
+            if not result.json or self.ask_user("Continue?"):
+                self.client.del_object(
+                    self.path,
+                    until=self['until_date'],
+                    delimiter='/' if self['recursive'] else self['delimiter'])
         else:
             self.error('Aborted')
 
@@ -998,7 +1019,10 @@ class file_upload(_PithosContainer):
                         rel_path = rpath + top.split(lpath)[1]
                     except IndexError:
                         rel_path = rpath
-                    self.error('mkdir /%s/%s' % (
+                    # Use the '/' separator for directories that
+                    # are about to be created in Pithos
+                    rel_path = rel_path.replace(path.sep, '/')
+                    self.error('remote: mkdir /%s/%s' % (
                         self.client.container, rel_path))
                     self.client.create_directory(rel_path)
                 for f in files:
@@ -1221,11 +1245,16 @@ class file_download(_PithosContainer):
         and they are pretended to other objects in a very strict order (shorter
         to longer path)."""
         ret, obj = [], None
+        # The prefix is actually the relative remote path without
+        # the trailing separator.
+        prefix = self.path.rstrip('/')
         try:
-            if self.path:
+            # prefix here is the object's path we requested to download
+            if prefix:
                 obj = self.client.get_object_info(
-                    self.path, version=self['object_version'])
-                obj.setdefault('name', self.path.strip('/'))
+                    prefix, version=self['object_version'])
+                #FIXME: why is this needed????
+                obj.setdefault('name', prefix)
         except ClientError as ce:
             if ce.status in (404, ):
                 self._container_exists()
@@ -1246,29 +1275,37 @@ class file_download(_PithosContainer):
                         '  kamaki container download %s [LOCAL_PATH]' % (
                             self.container)])
             raise
-        rpath = self.path.strip('/')
-        if local_path and self.path and local_path.endswith('/'):
-            local_path = local_path.rstrip('/') or '/'
 
+        # We requested to download either a whole container or a directory
         if (not obj) or self.object_is_dir(obj):
             if self['recursive']:
-                if not (self.path or local_path.endswith('/')):
-                    #  Download the whole container
-                    local_path = '' if local_path in ('.', ) else local_path
-                    local_path = '%s/' % (local_path or self.container)
                 obj = obj or dict(
                     name='', content_type='application/directory')
                 dirs, files = [], []
-                objects = self.client.container_get(
-                    prefix=rpath,
+                result = self.client.container_get(
+                    prefix=prefix,
                     if_modified_since=self['modified_since_date'],
                     if_unmodified_since=self['unmodified_since_date'])
-                for o in objects.json:
-                    (dirs if self.object_is_dir(o) else files).append(o)
+
+                # Find the final local path for each remote object
+                # [(remote name, final local path),.]
+                for o in result.json:
+                    remote = o['name']
+                    # First find the relative path of the object
+                    # without the prefix and any leading '/'
+                    relative = remote[len(prefix):].lstrip('/')
+                    # Translate it to a valid path with proper separator
+                    norm = relative.replace('/', path.sep)
+                    # Append it to the desired local path
+                    final = path.join(local_path, norm)
+                    if self.object_is_dir(o):
+                        dirs.append((remote, final))
+                    else:
+                        files.append((remote, final))
+                    self.error(r"%s -> %s" % (remote, final))
 
                 #  Put the directories on top of the list
-                for dpath in sorted(['%s%s' % (
-                        local_path, d['name'][len(rpath):]) for d in dirs]):
+                for dpath in sorted(p for _, p in dirs):
                     if path.exists(dpath):
                         if path.isdir(dpath):
                             continue
@@ -1281,13 +1318,12 @@ class file_download(_PithosContainer):
                     ret.append((None, dpath, None))
 
                 #  Append the file objects
-                for opath in [o['name'] for o in files]:
-                    lpath = '%s%s' % (local_path, opath[len(rpath):])
+                for opath, lpath in files:
                     if self['resume']:
                         fxists = path.exists(lpath)
                         if fxists and path.isdir(lpath):
                             raise CLIError(
-                                'Cannot change local dir %s info file' % (
+                                'Cannot change local dir %s into a file' % (
                                     lpath),
                                 details=[
                                     'Either remove the file or specify a'
@@ -1300,10 +1336,10 @@ class file_download(_PithosContainer):
                                 self.arguments['resume'].lvalue)])
                     else:
                         ret.append((opath, lpath, None))
-            elif self.path:
+            elif prefix:
                 raise CLIError(
                     'Remote object /%s/%s is a directory' % (
-                        self.container, local_path),
+                        self.container, prefix),
                     details=['Use %s to download directories' % (
                         self.arguments['recursive'].lvalue)])
             else:
@@ -1316,33 +1352,25 @@ class file_download(_PithosContainer):
                             parsed_name, self.container)])
         else:
             #  Remote object is just a file
+            #  The local path to be stored already exists
             if path.exists(local_path):
                 if not self['resume']:
                     raise CLIError(
                         'Cannot overwrite local file %s' % (local_path),
                         details=['To overwrite/resume, use  %s' % (
                             self.arguments['resume'].lvalue)])
-            elif '/' in local_path[1:-1]:
-                dirs = [p for p in local_path.split('/') if p]
-                pref = '/' if local_path.startswith('/') else ''
-                for d in dirs[:-1]:
-                    pref += d
-                    if not path.exists(pref):
-                        ret.append((None, d, None))
-                    elif not path.isdir(pref):
-                        raise CLIError(
-                            'Failed to use %s as a destination' % local_path,
-                            importance=3,
-                            details=[
-                                'Local file %s is not a directory' % pref,
-                                'Destination prefix must consist of '
-                                'directories or non-existing names',
-                                'Either remove the file, or choose another '
-                                'destination'])
-            ret.append((rpath, local_path, self['resume']))
+            #  The local path does not exist.
+            elif path.sep in local_path:
+                # Delegate intermediate local dir cration
+                # to makedirs() inside _run()
+                d = path.dirname(local_path)
+                ret.append((None, d, None))
+            ret.append((prefix, local_path, self['resume']))
+
         for r, l, resume in ret:
             if r:
-                with open(l, 'rwb+' if resume else 'wb+') as f:
+                mode = 'rb+' if resume and path.exists(l) else 'wb+'
+                with open(l, mode) as f:
                     yield (r, f)
             else:
                 yield (r, l)
@@ -1357,11 +1385,17 @@ class file_download(_PithosContainer):
         self.client.MAX_THREADS = int(self['max_threads'] or 5)
         progress_bar = None
         try:
+            # From _src_dst():
+            # If rpath is None output_file is a directory.
+            # If rpath is not None output_file is a file descriptor.
             for rpath, output_file in self._src_dst(local_path):
+                # Create a directory
                 if not rpath:
-                    self.error('Create local directory %s' % output_file)
-                    makedirs(output_file)
+                    if not path.exists(output_file):
+                        self.error('Create local directory %s' % output_file)
+                        makedirs(output_file)
                     continue
+                # Download a file
                 self.error('/%s/%s --> %s' % (
                     self.container, rpath, output_file.name))
                 progress_bar, download_cb = self._safe_progress_bar(
@@ -1398,14 +1432,19 @@ class file_download(_PithosContainer):
         self.error('Download completed')
 
     def main(self, remote_path_or_url, local_path=None):
+        """ Dowload remote_path_or_url to local_path. """
         super(self.__class__, self)._run(remote_path_or_url)
-        local_path, rpath = local_path or '.', self.path or self.container
-        if local_path.endswith('.'):
-            local_path = local_path[:-1] + path.basename(self.path.rstrip('/'))
-        elif local_path.endswith(path.sep):
-            local_path += path.basename(rpath.rstrip('/'))
+        # Translate relative remote path to local path with proper separator
+        # and without trailing '/'. If not given use the name of the container
+        rpath = self.path.rstrip('/').replace('/', path.sep) or self.container
+        # If remote path is /pithos/dir1/dir2/ then here we download dir2
+        base = path.basename(rpath)
+        # If local_path is not given use current dir
+        if not local_path:
+            local_path = path.join('.', base)
+        # existing_dir/ -> existing_dir/base
         elif path.exists(local_path) and path.isdir(local_path):
-            local_path += path.sep + path.basename(rpath.rstrip('/'))
+            local_path = path.join(local_path, base)
         self._run(local_path=local_path)
 
 

kamaki/cli/config/__init__.py

@@ -168,6 +168,7 @@ DEFAULTS = {
         'config_cli': 'config',
         'history_cli': 'history',
         'ignore_ssl': 'off',
+        'scripts_cli': 'contrib.scripts',
         'ca_certs': CACERTS_DEFAULT_PATH,
         #  Optional command specs:
         #  'service_cli': 'astakos'

kamaki/cli/contrib/__init__.py

+# Copyright 2015 GRNET S.A. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or
+# without modification, are permitted provided that the following
+# conditions are met:
+#
+#   1. Redistributions of source code must retain the above
+#      copyright notice, this list of conditions and the following
+#      disclaimer.
+#
+#   2. Redistributions in binary form must reproduce the above
+#      copyright notice, this list of conditions and the following
+#      disclaimer in the documentation and/or other materials
+#      provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
+# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
+# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+#
+# The views and conclusions contained in the software and
+# documentation are those of the authors and should not be
+# interpreted as representing official policies, either expressed
+# or implied, of GRNET S.A.

kamaki/cli/contrib/scripts.py

+# Copyright 2015 GRNET S.A. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or
+# without modification, are permitted provided that the following
+# conditions are met:
+#
+#   1. Redistributions of source code must retain the above
+#      copyright notice, this list of conditions and the following
+#      disclaimer.
+#
+#   2. Redistributions in binary form must reproduce the above
+#      copyright notice, this list of conditions and the following
+#      disclaimer in the documentation and/or other materials
+#      provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
+# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
+# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+#
+# The views and conclusions contained in the software and
+# documentation are those of the authors and should not be
+# interpreted as representing official policies, either expressed
+# or implied, of GRNET S.A.
+
+from datetime import date
+
+from kamaki.cli import command
+from kamaki.cli.errors import CLIError
+from kamaki.cli.cmdtree import CommandTree
+from kamaki.cli.cmds import errors, OptionalOutput
+from kamaki.cli.cmds.pithos import _PithosAccount
+from kamaki.cli.argument import FlagArgument
+
+scripts_cmds = CommandTree('scripts', 'Useful scripts')
+namespaces = [scripts_cmds, ]
+
+
+@command(scripts_cmds)
+class scripts_verifyfs(_PithosAccount, OptionalOutput):
+    """Verify/Fix the structure of directory objects inside a container"""
+
+    arguments = dict(
+        fix_conflicts=FlagArgument(
+            'Fix conflicting names by renaming them '
+            '(prepare the structure of directory objects to be consistent)',
+            '--fix-conflicts'),
+        fix_names=FlagArgument(
+            'Rename directory objects containing \\',
+            '--fix-dir-names'),
+        fix_missing=FlagArgument(
+            'Create missing directories objects',
+            '--fix-missing-dirs'),
+        yes=FlagArgument('Do not prompt for permission', '--yes'),
+    )
+
+    @errors.Generic.all
+    @errors.Pithos.connection
+    @errors.Pithos.container
+    def _run(self):
+        dirs, files, empty_files = [], [], []
+        result = self.client.container_get()
+        for o in result.json:
+            name = o['name']
+            if self.object_is_dir(o):
+                dirs.append(name)
+            elif o['bytes'] == 0:
+                empty_files.append(name)
+            else:
+                files.append(name)
+
+        # Find all directories with backslashes
+        wrong = set()
+        for d in dirs:
+            if '\\' in d:
+                wrong.add(d)
+
+        # Find all intermediate directories and see if a missing directory
+        # exists or if an intermediate directory conflicts with an existing
+        # object name
+        missing = set()
+        conflicts = set()
+        for n in files + dirs:
+            inter = n.split('/')
+            inter.pop()
+            d = []
+            for i in inter:
+                d.append(i)
+                p = '/'.join(d)
+                if p not in dirs:
+                    missing.add(p)
+                if p in files + empty_files:
+                    conflicts.add(p)
+
+        # First try to resolve conflicts
+        if self['fix_conflicts']:
+            for c in conflicts:
+                if self['yes'] or self.ask_user('Rename %s?' % c):
+                    backup = '%s_orig_%s' % (c, date.today().isoformat())
+                    # TODO: check if backup name already exists
+                    self.error(' * Renaming %s to %s' % (c, backup))
+                    self.client.move_object(
+                        src_container=self.client.container,
+                        src_object=c,
+                        dst_container=self.client.container,
+                        dst_object=backup)
+
+        elif conflicts:
+            raise CLIError(
+                'Conflicting object names found: %s' % conflicts,
+                details=[
+                    'They should be directory objects instead',
+                    'Use --fix-conflicts to rename them and prepare'
+                    ' the directory structure for further fix actions'])
+
+        # renames should take place after fixing conflicts
+        elif self['fix_names']:
+            for w in wrong:
+                correct = w.replace('\\', '/')
+                if self['yes'] or self.ask_user('Rename %s?' % w):
+                    self.error(' * Renaming %s to %s' % (w, correct))
+                    self.client.move_object(
+                        src_container=self.client.container,
+                        src_object=w,
+                        dst_container=self.client.container,
+                        dst_object=correct)
+        elif wrong:
+            raise CLIError(
+                'Directory objects with backslashes found: %s' % wrong,
+                details=['Use --fix-dir-names to sanitize them'])
+
+        # missing dirs should be created after fixing names
+        elif self['fix_missing']:
+            for d in missing:
+                if self['yes'] or self.ask_user('Create %s?' % d):
+                    self.error(' * Creating directory object %s' % d)
+                    self.client.create_directory(d)
+
+        elif missing:
+            raise CLIError(
+                'Missing directory objects found: %s' % missing,
+                details=['Use --fix-missing-dirs to create them'])
+
+    def main(self, container):
+        super(self.__class__, self)._run()
+        self.container, self.client.container = container, container
+        self._run()

kamaki/cli/utils/__init__.py

@@ -387,8 +387,10 @@ def ask_user(msg, true_resp=('y', ), **kwargs):
     msg = escape_ctrl_chars(msg).encode(pref_enc, 'replace')
     yep = yep.encode(pref_enc, 'replace')
     nope = nope.encode(pref_enc, 'replace')
-    user_response = raw_input(
+    response = raw_input(
         '%s [%s/%s]: ' % (msg, yep, nope))
+    # Pressing just enter gives an empty response!
+    user_response = response if response else 'N'
     return user_response[0].lower() in [s.lower() for s in true_resp]
 
 

kamaki/clients/network/__init__.py

@@ -123,8 +123,9 @@ class NetworkClient(NetworkRestClient, Waiter):
         :param name: (str) The subnet name
         :param allocation_pools: (list of dicts) start/end addresses of
             allocation pools: [{'start': ..., 'end': ...}, ...]
-        :param gateway_ip: (str)
-        :param subnet_id: (str)
+        :param gateway_ip: (str) Special cases:
+            None: server applies the default policy
+            empty iterable: no gateway IP on this subnet
         :param ipv6: (bool) ip_version == 6 if true else 4 (default)
         :param enable_dhcp: (bool)
         """
@@ -134,8 +135,8 @@ class NetworkClient(NetworkRestClient, Waiter):
             subnet['name'] = name
         if allocation_pools:
             subnet['allocation_pools'] = allocation_pools
-        if gateway_ip:
-            subnet['gateway_ip'] = gateway_ip
+        if gateway_ip is not None:
+            subnet['gateway_ip'] = gateway_ip or None
         if subnet_id:
             subnet['id'] = subnet_id
         if enable_dhcp not in (None, ):

kamaki/version.py

 
-__version__ = "0.13.1next"
+__version__ = "0.13.2next"
 __version_vcs_info__ = {
-    'branch': 'feature-wait',
-    'revid': '68cc68c',
-    'revno': 2492}
-__version_user_email__ = "saxtouri@admin.grnet.gr"
-__version_user_name__ = "Stavros Sachtouris"
+    'branch': 'develop',
+    'revid': 'ce8d24b',
+    'revno': 2506}
+__version_user_email__ = "skalkoto@grnet.gr"
+__version_user_name__ = "Nikos Skalkotos"

setup.py

 #!/usr/bin/env python
 
-# Copyright 2011-2013 GRNET S.A. All rights reserved.
+# Copyright 2011-2015 GRNET S.A. All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or
 # without modification, are permitted provided that the following
@@ -67,6 +67,7 @@ setup(
         'kamaki.cli.argument',
         'kamaki.cli.cmds',
         'kamaki.cli.cmdtree',
+        'kamaki.cli.contrib',
         'kamaki.clients',
         'kamaki.clients.utils',
         'kamaki.clients.astakos',

version

-0.13.1next
+0.13.2next