Merge branch 'feature-image-meta-record-format' into develop

Changelog

@@ -23,7 +23,7 @@ Changes:
     move and copy
 - Rename file/server-meta commands to file/server-metadata
 - Rename image-[add|del]member commands to members-[add|delete]
-- Remove update option from imagre-register
+- Remove update option from image-register
 - In image-compute split properties to properties-list and properties-get
 - Add optional output to methods[#3756, #3732]:
     - file:
@@ -52,6 +52,9 @@ Changes:
     - image: members, member
     - image compute: properties
     - server: firewall, metadata
+- Add a _format_image_headers method and use it in image_register and get_meta
+    for uniform image meta output [#3797]
+
 Features:
 
 - A logger module container a set of basic loging method for kamaki [#3668]
@@ -71,5 +74,7 @@ Features:
 - Store image properties on remote location after image registration [#3769]
 - Add runtime args to image register for forcing or unsettitng property
     storage [#3769]
+- Expand runtime args of image register for managing metadata and metada file
+    dumps and loads [#3797]
 - Add server-firewall-get command to get a VMs firewall profile
 

kamaki/cli/commands/errors.py

@@ -358,22 +358,6 @@ class plankton(object):
         '* get a list of image ids: /image list',
         '* details of image: /flavor info <image id>']
 
-    remote_image_file = [
-        'Suggested usage:',
-        '  /image register <image container>:<uploaded image file path>',
-        'To set "image" as image container and "my_dir/img.diskdump" as',
-        'the remote image file path, try one of the following:',
-        '- <image container>:<remote path>',
-        '    e.g. image:/my_dir/img.diskdump',
-        '- <remote path> -C <image container>',
-        '    e.g. /my_dir/img.diskdump -C image',
-        'To check if the image file is accessible to current user:',
-        '  /file list <image container>',
-        'If the file is located under a different user id "us3r1d"',
-        ' use the --fileowner=us3r1d  argument e.g.:',
-        '  /image register "my" image:my_dir/img.diskdump --fileowner=us3r1d',
-        'Note: The form pithos://<userid>/<container>/<path> is deprecated']
-
     @classmethod
     def connection(this, foo):
         return generic._connection(foo, 'image.url')
@@ -412,20 +396,6 @@ class plankton(object):
                 raise
         return _raise
 
-    @classmethod
-    def image_file(this, foo):
-        def _raise(self, name, container_path):
-            try:
-                return foo(self, name, container_path)
-            except ClientError as ce:
-                if ce.status in (400,):
-                    raiseCLIError(
-                        ce,
-                        'Nonexistent location for %s' % container_path,
-                        importance=2, details=this.remote_image_file)
-                raise
-        return _raise
-
 
 class pithos(object):
     container_howto = [

kamaki/cli/commands/image.py

@@ -57,8 +57,15 @@ image_cmds = CommandTree(
 _commands = [image_cmds]
 
 
-about_image_id = [
-    'To see a list of available image ids: /image list']
+howto_image_file = [
+    'Kamaki commands to:',
+    ' get current user uuid: /user authenticate',
+    ' check available containers: /file list',
+    ' create a new container: /file create <container>',
+    ' check container contents: /file list <container>',
+    ' upload files: /file upload <image file> <container>']
+
+about_image_id = ['To see a list of available image ids: /image list']
 
 
 log = getLogger(__name__)
@@ -84,14 +91,14 @@ class _init_image(_command_init):
 # Plankton Image Commands
 
 
-def _validate_image_props(json_dict, return_str=False):
+def _validate_image_meta(json_dict, return_str=False):
     """
     :param json_dict" (dict) json-formated, of the form
         {"key1": "val1", "key2": "val2", ...}
 
     :param return_str: (boolean) if true, return a json dump
 
-    :returns: (dict)
+    :returns: (dict) if return_str is not True, else return str
 
     :raises TypeError, AttributeError: Invalid json format
 
@@ -99,15 +106,22 @@ def _validate_image_props(json_dict, return_str=False):
     """
     json_str = dumps(json_dict, indent=2)
     for k, v in json_dict.items():
-        dealbreaker = isinstance(v, dict) or isinstance(v, list)
-        assert not dealbreaker, 'Invalid property value for key %s' % k
-        dealbreaker = ' ' in k
-        assert not dealbreaker, 'Invalid key [%s]' % k
+        if k.lower() == 'properties':
+            for pk, pv in v.items():
+                prop_ok = not (isinstance(pv, dict) or isinstance(pv, list))
+                assert prop_ok, 'Invalid property value for key %s' % pk
+                key_ok = not (' ' in k or '-' in k)
+                assert key_ok, 'Invalid property key %s' % k
+            continue
+        meta_ok = not (isinstance(v, dict) or isinstance(v, list))
+        assert meta_ok, 'Invalid value for meta key %s' % k
+        meta_ok = ' ' not in k
+        assert meta_ok, 'Invalid meta key [%s]' % k
         json_dict[k] = '%s' % v
     return json_str if return_str else json_dict
 
 
-def _load_image_props(filepath):
+def _load_image_meta(filepath):
     """
     :param filepath: (str) the (relative) path of the metafile
 
@@ -120,12 +134,32 @@ def _load_image_props(filepath):
     with open(abspath(filepath)) as f:
         meta_dict = load(f)
         try:
-            return _validate_image_props(meta_dict)
+            return _validate_image_meta(meta_dict)
         except AssertionError:
             log.debug('Failed to load properties from file %s' % filepath)
             raise
 
 
+def _validate_image_location(location):
+    """
+    :param location: (str) pithos://<uuid>/<container>/<img-file-path>
+
+    :returns: (<uuid>, <container>, <img-file-path>)
+
+    :raises AssertionError: if location is invalid
+    """
+    prefix = 'pithos://'
+    msg = 'Invalid prefix for location %s , try: %s' % (location, prefix)
+    assert location.startswith(prefix), msg
+    service, sep, rest = location.partition('://')
+    assert sep and rest, 'Location %s is missing uuid' % location
+    uuid, sep, rest = rest.partition('/')
+    assert sep and rest, 'Location %s is missing container' % location
+    container, sep, img_path = rest.partition('/')
+    assert sep and img_path, 'Location %s is missing image path' % location
+    return uuid, container, img_path
+
+
 @command(image_cmds)
 class image_list(_init_image, _optional_json):
     """List images accessible by user"""
@@ -244,90 +278,64 @@ class image_register(_init_image, _optional_json):
             'set container format',
             '--container-format'),
         disk_format=ValueArgument('set disk format', '--disk-format'),
-        #id=ValueArgument('set image ID', '--id'),
         owner=ValueArgument('set image owner (admin only)', '--owner'),
         properties=KeyValueArgument(
             'add property in key=value form (can be repeated)',
             ('-p', '--property')),
         is_public=FlagArgument('mark image as public', '--public'),
         size=IntArgument('set image size', '--size'),
-        property_file=ValueArgument(
-            'Load properties from a json-formated file <img-file>.meta :'
-            '{"key1": "val1", "key2": "val2", ...}',
-            ('--property-file')),
-        prop_file_force=FlagArgument(
-            'Store remote property object, even it already exists',
-            ('-f', '--force-upload-property-file')),
-        no_prop_file_upload=FlagArgument(
-            'Do not store properties in remote property file',
-            ('--no-property-file-upload')),
-        container=ValueArgument(
-            'Remote image container', ('-C', '--container')),
-        fileowner=ValueArgument(
-            'UUID of the user who owns the image file', ('--fileowner'))
+        metafile=ValueArgument(
+            'Load metadata from a json-formated file <img-file>.meta :'
+            '{"key1": "val1", "key2": "val2", ..., "properties: {...}"}',
+            ('--metafile')),
+        metafile_force=FlagArgument(
+            'Store remote metadata object, even if it already exists',
+            ('-f', '--force')),
+        no_metafile_upload=FlagArgument(
+            'Do not store metadata in remote meta file',
+            ('--no-metafile-upload')),
 
     )
 
     def _get_uuid(self):
-        uuid = self['fileowner'] or self.config.get('image', 'fileowner')
-        if uuid:
-            return uuid
         atoken = self.client.token
         user = AstakosClient(self.config.get('user', 'url'), atoken)
         return user.term('uuid')
 
-    def _get_pithos_client(self, uuid, container):
+    def _get_pithos_client(self, container):
+        if self['no_metafile_upload']:
+            return None
         purl = self.config.get('file', 'url')
         ptoken = self.client.token
-        return PithosClient(purl, ptoken, uuid, container)
+        return PithosClient(purl, ptoken, self._get_uuid(), container)
 
-    def _store_remote_property_file(self, pclient, remote_path, properties):
+    def _store_remote_metafile(self, pclient, remote_path, metadata):
         return pclient.upload_from_string(
-            remote_path, _validate_image_props(properties, return_str=True))
-
-    def _get_container_path(self, container_path):
-        container = self['container'] or self.config.get('image', 'container')
-        if container:
-            return container, container_path
-
-        container, sep, path = container_path.partition(':')
-        if not sep or not container or not path:
-            raiseCLIError(
-                '%s is not a valid pithos+ remote location' % container_path,
-                importance=2,
-                details=[
-                    'To set "image" as container and "my_dir/img.diskdump" as',
-                    'the image path, try one of the following as '
-                    'container:path',
-                    '- <image container>:<remote path>',
-                    '    e.g. image:/my_dir/img.diskdump',
-                    '- <remote path> -C <image container>',
-                    '    e.g. /my_dir/img.diskdump -C image'])
-        return container, path
+            remote_path, _validate_image_meta(metadata, return_str=True))
 
-    @errors.generic.all
-    @errors.plankton.image_file
-    @errors.plankton.connection
-    def _run(self, name, container_path):
-        container, path = self._get_container_path(container_path)
-        uuid = self._get_uuid()
-        prop_path = '%s.meta' % path
-
-        pclient = None if (
-            self['no_prop_file_upload']) else self._get_pithos_client(
-                uuid, container)
-        if pclient and not self['prop_file_force']:
+    def _load_params_from_file(self, location):
+        params, properties = dict(), dict()
+        pfile = self['metafile']
+        if pfile:
             try:
-                pclient.get_object_info(prop_path)
-                raiseCLIError('Property file %s: %s already exists' % (
-                    container, prop_path))
-            except ClientError as ce:
-                if ce.status != 404:
-                    raise
-
-        location = 'pithos://%s/%s/%s' % (uuid, container, path)
+                for k, v in _load_image_meta(pfile).items():
+                    key = k.lower().replace('-', '_')
+                    if k == 'properties':
+                        for pk, pv in v.items():
+                            properties[pk.upper().replace('-', '_')] = pv
+                    elif key == 'name':
+                            continue
+                    elif key == 'location':
+                        if location:
+                            continue
+                        location = v
+                    else:
+                        params[key] = v
+            except Exception as e:
+                raiseCLIError(e, 'Invalid json metadata config file')
+        return params, properties, location
 
-        params = {}
+    def _load_params_from_args(self, params, properties):
         for key in set([
                 'checksum',
                 'container_format',
@@ -336,47 +344,78 @@ class image_register(_init_image, _optional_json):
                 'size',
                 'is_public']).intersection(self.arguments):
             params[key] = self[key]
-        properties = self['properties']
+        for k, v in self['properties'].items():
+            properties[k.upper().replace('-', '_')] = v
 
-        #load properties
-        properties = dict()
-        pfile = self['property_file']
-        if pfile:
+    def _validate_location(self, location):
+        if not location:
+            raiseCLIError(
+                'No image file location provided',
+                importance=2, details=[
+                    'An image location is needed. Image location format:',
+                    '  pithos://<uuid>/<container>/<path>',
+                    ' an image file at the above location must exist.'
+                    ] + howto_image_file)
+        try:
+            return _validate_image_location(location)
+        except AssertionError as ae:
+            raiseCLIError(
+                ae, 'Invalid image location format',
+                importance=1, details=[
+                    'Valid image location format:',
+                    '  pithos://<uuid>/<container>/<img-file-path>'
+                    ] + howto_image_file)
+
+    @errors.generic.all
+    @errors.plankton.connection
+    def _run(self, name, location):
+        (params, properties, location) = self._load_params_from_file(location)
+        uuid, container, img_path = self._validate_location(location)
+        self._load_params_from_args(params, properties)
+        pclient = self._get_pithos_client(container)
+
+        #check if metafile exists
+        meta_path = '%s.meta' % img_path
+        if pclient and not self['metafile_force']:
             try:
-                for k, v in _load_image_props(pfile).items():
-                    properties[k.lower()] = v
-            except Exception as e:
+                pclient.get_object_info(meta_path)
+                raiseCLIError('Metadata file %s:%s already exists' % (
+                    container, meta_path))
+            except ClientError as ce:
+                if ce.status != 404:
+                    raise
+
+        #register the image
+        try:
+            r = self.client.register(name, location, params, properties)
+        except ClientError as ce:
+            if ce.status in (400, ):
                 raiseCLIError(
-                    e, 'Format error in property file %s' % pfile,
+                    ce, 'Nonexistent image file location %s' % location,
                     details=[
-                        'Expected content format:',
-                        '  {',
-                        '    "key1": "value1",',
-                        '    "key2": "value2",',
-                        '    ...',
-                        '  }',
-                        '',
-                        'Parser:'
-                    ])
-        for k, v in self['properties'].items():
-            properties[k.lower()] = v
-
-        self._print([self.client.register(name, location, params, properties)])
+                        'Make sure the image file exists'] + howto_image_file)
+            raise
+        self._print(r, print_dict)
 
+        #upload the metadata file
         if pclient:
-            prop_headers = pclient.upload_from_string(
-                prop_path, _validate_image_props(properties, return_str=True))
+            try:
+                meta_headers = pclient.upload_from_string(
+                    meta_path, dumps(r, indent=2))
+            except TypeError:
+                print('Failed to dump metafile %s:%s' % (container, meta_path))
+                return
             if self['json_output']:
                 print_json(dict(
-                    property_file_location='%s:%s' % (container, prop_path),
-                    headers=prop_headers))
+                    metafile_location='%s:%s' % (container, meta_path),
+                    headers=meta_headers))
             else:
-                print('Property file uploaded as %s:%s (version %s)' % (
-                    container, prop_path, prop_headers['x-object-version']))
+                print('Metadata file uploaded as %s:%s (version %s)' % (
+                    container, meta_path, meta_headers['x-object-version']))
 
-    def main(self, name, container___path):
+    def main(self, name, location=None):
         super(self.__class__, self)._run()
-        self._run(name, container___path)
+        self._run(name, location)
 
 
 @command(image_cmds)

kamaki/clients/image/__init__.py

@@ -35,6 +35,22 @@ from kamaki.clients import Client, ClientError
 from kamaki.clients.utils import path4url, filter_in
 
 
+def _format_image_headers(headers):
+    reply = dict(properties=dict())
+    meta_prefix = 'x-image-meta-'
+    property_prefix = 'x-image-meta-property-'
+
+    for key, val in headers.items():
+        key = key.lower()
+        if key.startswith(property_prefix):
+            key = key[len(property_prefix):].upper().replace('-', '_')
+            reply['properties'][key] = val
+        elif key.startswith(meta_prefix):
+            key = key[len(meta_prefix):]
+            reply[key] = val
+    return reply
+
+
 class ImageClient(Client):
     """Synnefo Plankton API client"""
 
@@ -80,23 +96,7 @@ class ImageClient(Client):
         path = path4url('images', image_id)
         r = self.head(path, success=200)
 
-        reply = {}
-        properties = {}
-        meta_prefix = 'x-image-meta-'
-        property_prefix = 'x-image-meta-property-'
-
-        for key, val in r.headers.items():
-            key = key.lower()
-            if key.startswith(property_prefix):
-                key = key[len(property_prefix):]
-                properties[key] = val
-            elif key.startswith(meta_prefix):
-                key = key[len(meta_prefix):]
-                reply[key] = val
-
-        if properties:
-            reply['properties'] = properties
-        return reply
+        return _format_image_headers(r.headers)
 
     def register(self, name, location, params={}, properties={}):
         """Register an image that is uploaded at location
@@ -110,7 +110,7 @@ class ImageClient(Client):
 
         :param properties: (dict) image properties (X-Image-Meta-Property)
 
-        :returns: (dict) details of the created image
+        :returns: (dict) metadata of the created image
         """
         path = path4url('images') + '/'
         self.set_header('X-Image-Meta-Name', name)
@@ -127,7 +127,8 @@ class ImageClient(Client):
             async_headers['x-image-meta-property-%s' % key] = val
 
         r = self.post(path, success=200, async_headers=async_headers)
-        return filter_in(r.headers, 'X-Image-')
+
+        return _format_image_headers(r.headers)
 
     def unregister(self, image_id):
         """Unregister an image

kamaki/clients/image/test.py

@@ -228,7 +228,8 @@ class ImageClient(TestCase):
             params=params, properties=props)
         expectedict = dict(example_image_headers)
         expectedict.pop('extraheaders')
-        self.assert_dicts_are_equal(expectedict, r)
+        from kamaki.clients.image import _format_image_headers
+        self.assert_dicts_are_equal(_format_image_headers(expectedict), r)
         self.assertEqual(
             post.mock_calls[-1],
             call('/images/', async_headers=async_headers, success=200))

kamaki/clients/livetest/image.py

@@ -85,7 +85,7 @@ class Image(livetest.Generic):
             self.location,
             params=dict(is_public=True))
         self._imglist[self.imgname] = dict(
-            name=r['x-image-meta-name'], id=r['x-image-meta-id'])
+            name=r['name'], id=r['id'])
         self._imgdetails[self.imgname] = r
 
     def tearDown(self):
@@ -154,7 +154,7 @@ class Image(livetest.Generic):
                     'properties',
                     'size'):
                 self.assertTrue(term in img)
-                if img['properties']:
+                if len(img['properties']):
                     for interm in ('osfamily', 'users', 'root_partition'):
                         self.assertTrue(interm in img['properties'])
         size_max = 1000000000
@@ -185,13 +185,14 @@ class Image(livetest.Generic):
                 'container-format'):
             self.assertTrue(term in r)
             for interm in (
-                    'kernel',
-                    'osfamily',
-                    'users',
-                    'gui', 'sortorder',
-                    'root-partition',
-                    'os',
-                    'description'):
+                    'KERNEL',
+                    'OSFAMILY',
+                    'USERS',
+                    'GUI',
+                    'SORTORDER',
+                    'ROOT_PARTITION',
+                    'OS',
+                    'DESCRIPTION'):
                 self.assertTrue(interm in r['properties'])
 
     def test_register(self):
@@ -204,8 +205,7 @@ class Image(livetest.Generic):
         for img in self._imglist.values():
             self.assertTrue(img is not None)
             r = set(self._imgdetails[img['name']].keys())
-            self.assertTrue(
-                r.issubset(['x-image-meta-%s' % k for k in IMGMETA]))
+            self.assertTrue(r.issubset(IMGMETA.union(['properties'])))
 
     def test_unregister(self):
         """Test unregister"""