Commit fb3998bd authored by Stavros Sachtouris's avatar Stavros Sachtouris
Browse files

Merge branch 'feature-image-meta-record-format' into develop

parents 4e424eaa aa82dd5a
......@@ -23,7 +23,7 @@ Changes:
move and copy
- Rename file/server-meta commands to file/server-metadata
- Rename image-[add|del]member commands to members-[add|delete]
- Remove update option from imagre-register
- Remove update option from image-register
- In image-compute split properties to properties-list and properties-get
- Add optional output to methods[#3756, #3732]:
- file:
......@@ -52,6 +52,9 @@ Changes:
- image: members, member
- image compute: properties
- server: firewall, metadata
- Add a _format_image_headers method and use it in image_register and get_meta
for uniform image meta output [#3797]
- A logger module container a set of basic loging method for kamaki [#3668]
......@@ -71,5 +74,7 @@ Features:
- Store image properties on remote location after image registration [#3769]
- Add runtime args to image register for forcing or unsettitng property
storage [#3769]
- Expand runtime args of image register for managing metadata and metada file
dumps and loads [#3797]
- Add server-firewall-get command to get a VMs firewall profile
......@@ -358,22 +358,6 @@ class plankton(object):
'* get a list of image ids: /image list',
'* details of image: /flavor info <image id>']
remote_image_file = [
'Suggested usage:',
' /image register <image container>:<uploaded image file path>',
'To set "image" as image container and "my_dir/img.diskdump" as',
'the remote image file path, try one of the following:',
'- <image container>:<remote path>',
' e.g. image:/my_dir/img.diskdump',
'- <remote path> -C <image container>',
' e.g. /my_dir/img.diskdump -C image',
'To check if the image file is accessible to current user:',
' /file list <image container>',
'If the file is located under a different user id "us3r1d"',
' use the --fileowner=us3r1d argument e.g.:',
' /image register "my" image:my_dir/img.diskdump --fileowner=us3r1d',
'Note: The form pithos://<userid>/<container>/<path> is deprecated']
def connection(this, foo):
return generic._connection(foo, 'image.url')
......@@ -412,20 +396,6 @@ class plankton(object):
return _raise
def image_file(this, foo):
def _raise(self, name, container_path):
return foo(self, name, container_path)
except ClientError as ce:
if ce.status in (400,):
'Nonexistent location for %s' % container_path,
importance=2, details=this.remote_image_file)
return _raise
class pithos(object):
container_howto = [
......@@ -57,8 +57,15 @@ image_cmds = CommandTree(
_commands = [image_cmds]
about_image_id = [
'To see a list of available image ids: /image list']
howto_image_file = [
'Kamaki commands to:',
' get current user uuid: /user authenticate',
' check available containers: /file list',
' create a new container: /file create <container>',
' check container contents: /file list <container>',
' upload files: /file upload <image file> <container>']
about_image_id = ['To see a list of available image ids: /image list']
log = getLogger(__name__)
......@@ -84,14 +91,14 @@ class _init_image(_command_init):
# Plankton Image Commands
def _validate_image_props(json_dict, return_str=False):
def _validate_image_meta(json_dict, return_str=False):
:param json_dict" (dict) json-formated, of the form
{"key1": "val1", "key2": "val2", ...}
:param return_str: (boolean) if true, return a json dump
:returns: (dict)
:returns: (dict) if return_str is not True, else return str
:raises TypeError, AttributeError: Invalid json format
......@@ -99,15 +106,22 @@ def _validate_image_props(json_dict, return_str=False):
json_str = dumps(json_dict, indent=2)
for k, v in json_dict.items():
dealbreaker = isinstance(v, dict) or isinstance(v, list)
assert not dealbreaker, 'Invalid property value for key %s' % k
dealbreaker = ' ' in k
assert not dealbreaker, 'Invalid key [%s]' % k
if k.lower() == 'properties':
for pk, pv in v.items():
prop_ok = not (isinstance(pv, dict) or isinstance(pv, list))
assert prop_ok, 'Invalid property value for key %s' % pk
key_ok = not (' ' in k or '-' in k)
assert key_ok, 'Invalid property key %s' % k
meta_ok = not (isinstance(v, dict) or isinstance(v, list))
assert meta_ok, 'Invalid value for meta key %s' % k
meta_ok = ' ' not in k
assert meta_ok, 'Invalid meta key [%s]' % k
json_dict[k] = '%s' % v
return json_str if return_str else json_dict
def _load_image_props(filepath):
def _load_image_meta(filepath):
:param filepath: (str) the (relative) path of the metafile
......@@ -120,12 +134,32 @@ def _load_image_props(filepath):
with open(abspath(filepath)) as f:
meta_dict = load(f)
return _validate_image_props(meta_dict)
return _validate_image_meta(meta_dict)
except AssertionError:
log.debug('Failed to load properties from file %s' % filepath)
def _validate_image_location(location):
:param location: (str) pithos://<uuid>/<container>/<img-file-path>
:returns: (<uuid>, <container>, <img-file-path>)
:raises AssertionError: if location is invalid
prefix = 'pithos://'
msg = 'Invalid prefix for location %s , try: %s' % (location, prefix)
assert location.startswith(prefix), msg
service, sep, rest = location.partition('://')
assert sep and rest, 'Location %s is missing uuid' % location
uuid, sep, rest = rest.partition('/')
assert sep and rest, 'Location %s is missing container' % location
container, sep, img_path = rest.partition('/')
assert sep and img_path, 'Location %s is missing image path' % location
return uuid, container, img_path
class image_list(_init_image, _optional_json):
"""List images accessible by user"""
......@@ -244,90 +278,64 @@ class image_register(_init_image, _optional_json):
'set container format',
disk_format=ValueArgument('set disk format', '--disk-format'),
#id=ValueArgument('set image ID', '--id'),
owner=ValueArgument('set image owner (admin only)', '--owner'),
'add property in key=value form (can be repeated)',
('-p', '--property')),
is_public=FlagArgument('mark image as public', '--public'),
size=IntArgument('set image size', '--size'),
'Load properties from a json-formated file <img-file>.meta :'
'{"key1": "val1", "key2": "val2", ...}',
'Store remote property object, even it already exists',
('-f', '--force-upload-property-file')),
'Do not store properties in remote property file',
'Remote image container', ('-C', '--container')),
'UUID of the user who owns the image file', ('--fileowner'))
'Load metadata from a json-formated file <img-file>.meta :'
'{"key1": "val1", "key2": "val2", ..., "properties: {...}"}',
'Store remote metadata object, even if it already exists',
('-f', '--force')),
'Do not store metadata in remote meta file',
def _get_uuid(self):
uuid = self['fileowner'] or self.config.get('image', 'fileowner')
if uuid:
return uuid
atoken = self.client.token
user = AstakosClient(self.config.get('user', 'url'), atoken)
return user.term('uuid')
def _get_pithos_client(self, uuid, container):
def _get_pithos_client(self, container):
if self['no_metafile_upload']:
return None
purl = self.config.get('file', 'url')
ptoken = self.client.token
return PithosClient(purl, ptoken, uuid, container)
return PithosClient(purl, ptoken, self._get_uuid(), container)
def _store_remote_property_file(self, pclient, remote_path, properties):
def _store_remote_metafile(self, pclient, remote_path, metadata):
return pclient.upload_from_string(
remote_path, _validate_image_props(properties, return_str=True))
def _get_container_path(self, container_path):
container = self['container'] or self.config.get('image', 'container')
if container:
return container, container_path
container, sep, path = container_path.partition(':')
if not sep or not container or not path:
'%s is not a valid pithos+ remote location' % container_path,
'To set "image" as container and "my_dir/img.diskdump" as',
'the image path, try one of the following as '
'- <image container>:<remote path>',
' e.g. image:/my_dir/img.diskdump',
'- <remote path> -C <image container>',
' e.g. /my_dir/img.diskdump -C image'])
return container, path
remote_path, _validate_image_meta(metadata, return_str=True))
def _run(self, name, container_path):
container, path = self._get_container_path(container_path)
uuid = self._get_uuid()
prop_path = '%s.meta' % path
pclient = None if (
self['no_prop_file_upload']) else self._get_pithos_client(
uuid, container)
if pclient and not self['prop_file_force']:
def _load_params_from_file(self, location):
params, properties = dict(), dict()
pfile = self['metafile']
if pfile:
raiseCLIError('Property file %s: %s already exists' % (
container, prop_path))
except ClientError as ce:
if ce.status != 404:
location = 'pithos://%s/%s/%s' % (uuid, container, path)
for k, v in _load_image_meta(pfile).items():
key = k.lower().replace('-', '_')
if k == 'properties':
for pk, pv in v.items():
properties[pk.upper().replace('-', '_')] = pv
elif key == 'name':
elif key == 'location':
if location:
location = v
params[key] = v
except Exception as e:
raiseCLIError(e, 'Invalid json metadata config file')
return params, properties, location
params = {}
def _load_params_from_args(self, params, properties):
for key in set([
......@@ -336,47 +344,78 @@ class image_register(_init_image, _optional_json):
params[key] = self[key]
properties = self['properties']
for k, v in self['properties'].items():
properties[k.upper().replace('-', '_')] = v
#load properties
properties = dict()
pfile = self['property_file']
if pfile:
def _validate_location(self, location):
if not location:
'No image file location provided',
importance=2, details=[
'An image location is needed. Image location format:',
' pithos://<uuid>/<container>/<path>',
' an image file at the above location must exist.'
] + howto_image_file)
return _validate_image_location(location)
except AssertionError as ae:
ae, 'Invalid image location format',
importance=1, details=[
'Valid image location format:',
' pithos://<uuid>/<container>/<img-file-path>'
] + howto_image_file)
def _run(self, name, location):
(params, properties, location) = self._load_params_from_file(location)
uuid, container, img_path = self._validate_location(location)
self._load_params_from_args(params, properties)
pclient = self._get_pithos_client(container)
#check if metafile exists
meta_path = '%s.meta' % img_path
if pclient and not self['metafile_force']:
for k, v in _load_image_props(pfile).items():
properties[k.lower()] = v
except Exception as e:
raiseCLIError('Metadata file %s:%s already exists' % (
container, meta_path))
except ClientError as ce:
if ce.status != 404:
#register the image
r = self.client.register(name, location, params, properties)
except ClientError as ce:
if ce.status in (400, ):
e, 'Format error in property file %s' % pfile,
ce, 'Nonexistent image file location %s' % location,
'Expected content format:',
' {',
' "key1": "value1",',
' "key2": "value2",',
' ...',
' }',
for k, v in self['properties'].items():
properties[k.lower()] = v
self._print([self.client.register(name, location, params, properties)])
'Make sure the image file exists'] + howto_image_file)
self._print(r, print_dict)
#upload the metadata file
if pclient:
prop_headers = pclient.upload_from_string(
prop_path, _validate_image_props(properties, return_str=True))
meta_headers = pclient.upload_from_string(
meta_path, dumps(r, indent=2))
except TypeError:
print('Failed to dump metafile %s:%s' % (container, meta_path))
if self['json_output']:
property_file_location='%s:%s' % (container, prop_path),
metafile_location='%s:%s' % (container, meta_path),
print('Property file uploaded as %s:%s (version %s)' % (
container, prop_path, prop_headers['x-object-version']))
print('Metadata file uploaded as %s:%s (version %s)' % (
container, meta_path, meta_headers['x-object-version']))
def main(self, name, container___path):
def main(self, name, location=None):
super(self.__class__, self)._run()
self._run(name, container___path)
self._run(name, location)
......@@ -35,6 +35,22 @@ from kamaki.clients import Client, ClientError
from kamaki.clients.utils import path4url, filter_in
def _format_image_headers(headers):
reply = dict(properties=dict())
meta_prefix = 'x-image-meta-'
property_prefix = 'x-image-meta-property-'
for key, val in headers.items():
key = key.lower()
if key.startswith(property_prefix):
key = key[len(property_prefix):].upper().replace('-', '_')
reply['properties'][key] = val
elif key.startswith(meta_prefix):
key = key[len(meta_prefix):]
reply[key] = val
return reply
class ImageClient(Client):
"""Synnefo Plankton API client"""
......@@ -80,23 +96,7 @@ class ImageClient(Client):
path = path4url('images', image_id)
r = self.head(path, success=200)
reply = {}
properties = {}
meta_prefix = 'x-image-meta-'
property_prefix = 'x-image-meta-property-'
for key, val in r.headers.items():
key = key.lower()
if key.startswith(property_prefix):
key = key[len(property_prefix):]
properties[key] = val
elif key.startswith(meta_prefix):
key = key[len(meta_prefix):]
reply[key] = val
if properties:
reply['properties'] = properties
return reply
return _format_image_headers(r.headers)
def register(self, name, location, params={}, properties={}):
"""Register an image that is uploaded at location
......@@ -110,7 +110,7 @@ class ImageClient(Client):
:param properties: (dict) image properties (X-Image-Meta-Property)
:returns: (dict) details of the created image
:returns: (dict) metadata of the created image
path = path4url('images') + '/'
self.set_header('X-Image-Meta-Name', name)
......@@ -127,7 +127,8 @@ class ImageClient(Client):
async_headers['x-image-meta-property-%s' % key] = val
r =, success=200, async_headers=async_headers)
return filter_in(r.headers, 'X-Image-')
return _format_image_headers(r.headers)
def unregister(self, image_id):
"""Unregister an image
......@@ -228,7 +228,8 @@ class ImageClient(TestCase):
params=params, properties=props)
expectedict = dict(example_image_headers)
self.assert_dicts_are_equal(expectedict, r)
from kamaki.clients.image import _format_image_headers
self.assert_dicts_are_equal(_format_image_headers(expectedict), r)
call('/images/', async_headers=async_headers, success=200))
......@@ -85,7 +85,7 @@ class Image(livetest.Generic):
self._imglist[self.imgname] = dict(
name=r['x-image-meta-name'], id=r['x-image-meta-id'])
name=r['name'], id=r['id'])
self._imgdetails[self.imgname] = r
def tearDown(self):
......@@ -154,7 +154,7 @@ class Image(livetest.Generic):
self.assertTrue(term in img)
if img['properties']:
if len(img['properties']):
for interm in ('osfamily', 'users', 'root_partition'):
self.assertTrue(interm in img['properties'])
size_max = 1000000000
......@@ -185,13 +185,14 @@ class Image(livetest.Generic):
self.assertTrue(term in r)
for interm in (
'gui', 'sortorder',
self.assertTrue(interm in r['properties'])
def test_register(self):
......@@ -204,8 +205,7 @@ class Image(livetest.Generic):
for img in self._imglist.values():
self.assertTrue(img is not None)
r = set(self._imgdetails[img['name']].keys())
r.issubset(['x-image-meta-%s' % k for k in IMGMETA]))
def test_unregister(self):
"""Test unregister"""
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment