Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
itminedu
kamaki
Commits
fb3998bd
Commit
fb3998bd
authored
May 23, 2013
by
Stavros Sachtouris
Browse files
Merge branch 'feature-image-meta-record-format' into develop
parents
4e424eaa
aa82dd5a
Changes
6
Hide whitespace changes
Inline
Side-by-side
Changelog
View file @
fb3998bd
...
...
@@ -23,7 +23,7 @@ Changes:
move and copy
- Rename file/server-meta commands to file/server-metadata
- Rename image-[add|del]member commands to members-[add|delete]
- Remove update option from imag
r
e-register
- Remove update option from image-register
- In image-compute split properties to properties-list and properties-get
- Add optional output to methods[#3756, #3732]:
- file:
...
...
@@ -52,6 +52,9 @@ Changes:
- image: members, member
- image compute: properties
- server: firewall, metadata
- Add a _format_image_headers method and use it in image_register and get_meta
for uniform image meta output [#3797]
Features:
- A logger module container a set of basic loging method for kamaki [#3668]
...
...
@@ -71,5 +74,7 @@ Features:
- Store image properties on remote location after image registration [#3769]
- Add runtime args to image register for forcing or unsettitng property
storage [#3769]
- Expand runtime args of image register for managing metadata and metada file
dumps and loads [#3797]
- Add server-firewall-get command to get a VMs firewall profile
kamaki/cli/commands/errors.py
View file @
fb3998bd
...
...
@@ -358,22 +358,6 @@ class plankton(object):
'* get a list of image ids: /image list'
,
'* details of image: /flavor info <image id>'
]
remote_image_file
=
[
'Suggested usage:'
,
' /image register <image container>:<uploaded image file path>'
,
'To set "image" as image container and "my_dir/img.diskdump" as'
,
'the remote image file path, try one of the following:'
,
'- <image container>:<remote path>'
,
' e.g. image:/my_dir/img.diskdump'
,
'- <remote path> -C <image container>'
,
' e.g. /my_dir/img.diskdump -C image'
,
'To check if the image file is accessible to current user:'
,
' /file list <image container>'
,
'If the file is located under a different user id "us3r1d"'
,
' use the --fileowner=us3r1d argument e.g.:'
,
' /image register "my" image:my_dir/img.diskdump --fileowner=us3r1d'
,
'Note: The form pithos://<userid>/<container>/<path> is deprecated'
]
@
classmethod
def
connection
(
this
,
foo
):
return
generic
.
_connection
(
foo
,
'image.url'
)
...
...
@@ -412,20 +396,6 @@ class plankton(object):
raise
return
_raise
@
classmethod
def
image_file
(
this
,
foo
):
def
_raise
(
self
,
name
,
container_path
):
try
:
return
foo
(
self
,
name
,
container_path
)
except
ClientError
as
ce
:
if
ce
.
status
in
(
400
,):
raiseCLIError
(
ce
,
'Nonexistent location for %s'
%
container_path
,
importance
=
2
,
details
=
this
.
remote_image_file
)
raise
return
_raise
class
pithos
(
object
):
container_howto
=
[
...
...
kamaki/cli/commands/image.py
View file @
fb3998bd
...
...
@@ -57,8 +57,15 @@ image_cmds = CommandTree(
_commands
=
[
image_cmds
]
about_image_id
=
[
'To see a list of available image ids: /image list'
]
howto_image_file
=
[
'Kamaki commands to:'
,
' get current user uuid: /user authenticate'
,
' check available containers: /file list'
,
' create a new container: /file create <container>'
,
' check container contents: /file list <container>'
,
' upload files: /file upload <image file> <container>'
]
about_image_id
=
[
'To see a list of available image ids: /image list'
]
log
=
getLogger
(
__name__
)
...
...
@@ -84,14 +91,14 @@ class _init_image(_command_init):
# Plankton Image Commands
def
_validate_image_
props
(
json_dict
,
return_str
=
False
):
def
_validate_image_
meta
(
json_dict
,
return_str
=
False
):
"""
:param json_dict" (dict) json-formated, of the form
{"key1": "val1", "key2": "val2", ...}
:param return_str: (boolean) if true, return a json dump
:returns: (dict)
:returns: (dict)
if return_str is not True, else return str
:raises TypeError, AttributeError: Invalid json format
...
...
@@ -99,15 +106,22 @@ def _validate_image_props(json_dict, return_str=False):
"""
json_str
=
dumps
(
json_dict
,
indent
=
2
)
for
k
,
v
in
json_dict
.
items
():
dealbreaker
=
isinstance
(
v
,
dict
)
or
isinstance
(
v
,
list
)
assert
not
dealbreaker
,
'Invalid property value for key %s'
%
k
dealbreaker
=
' '
in
k
assert
not
dealbreaker
,
'Invalid key [%s]'
%
k
if
k
.
lower
()
==
'properties'
:
for
pk
,
pv
in
v
.
items
():
prop_ok
=
not
(
isinstance
(
pv
,
dict
)
or
isinstance
(
pv
,
list
))
assert
prop_ok
,
'Invalid property value for key %s'
%
pk
key_ok
=
not
(
' '
in
k
or
'-'
in
k
)
assert
key_ok
,
'Invalid property key %s'
%
k
continue
meta_ok
=
not
(
isinstance
(
v
,
dict
)
or
isinstance
(
v
,
list
))
assert
meta_ok
,
'Invalid value for meta key %s'
%
k
meta_ok
=
' '
not
in
k
assert
meta_ok
,
'Invalid meta key [%s]'
%
k
json_dict
[
k
]
=
'%s'
%
v
return
json_str
if
return_str
else
json_dict
def
_load_image_
props
(
filepath
):
def
_load_image_
meta
(
filepath
):
"""
:param filepath: (str) the (relative) path of the metafile
...
...
@@ -120,12 +134,32 @@ def _load_image_props(filepath):
with
open
(
abspath
(
filepath
))
as
f
:
meta_dict
=
load
(
f
)
try
:
return
_validate_image_
props
(
meta_dict
)
return
_validate_image_
meta
(
meta_dict
)
except
AssertionError
:
log
.
debug
(
'Failed to load properties from file %s'
%
filepath
)
raise
def
_validate_image_location
(
location
):
"""
:param location: (str) pithos://<uuid>/<container>/<img-file-path>
:returns: (<uuid>, <container>, <img-file-path>)
:raises AssertionError: if location is invalid
"""
prefix
=
'pithos://'
msg
=
'Invalid prefix for location %s , try: %s'
%
(
location
,
prefix
)
assert
location
.
startswith
(
prefix
),
msg
service
,
sep
,
rest
=
location
.
partition
(
'://'
)
assert
sep
and
rest
,
'Location %s is missing uuid'
%
location
uuid
,
sep
,
rest
=
rest
.
partition
(
'/'
)
assert
sep
and
rest
,
'Location %s is missing container'
%
location
container
,
sep
,
img_path
=
rest
.
partition
(
'/'
)
assert
sep
and
img_path
,
'Location %s is missing image path'
%
location
return
uuid
,
container
,
img_path
@
command
(
image_cmds
)
class
image_list
(
_init_image
,
_optional_json
):
"""List images accessible by user"""
...
...
@@ -244,90 +278,64 @@ class image_register(_init_image, _optional_json):
'set container format'
,
'--container-format'
),
disk_format
=
ValueArgument
(
'set disk format'
,
'--disk-format'
),
#id=ValueArgument('set image ID', '--id'),
owner
=
ValueArgument
(
'set image owner (admin only)'
,
'--owner'
),
properties
=
KeyValueArgument
(
'add property in key=value form (can be repeated)'
,
(
'-p'
,
'--property'
)),
is_public
=
FlagArgument
(
'mark image as public'
,
'--public'
),
size
=
IntArgument
(
'set image size'
,
'--size'
),
property_file
=
ValueArgument
(
'Load properties from a json-formated file <img-file>.meta :'
'{"key1": "val1", "key2": "val2", ...}'
,
(
'--property-file'
)),
prop_file_force
=
FlagArgument
(
'Store remote property object, even it already exists'
,
(
'-f'
,
'--force-upload-property-file'
)),
no_prop_file_upload
=
FlagArgument
(
'Do not store properties in remote property file'
,
(
'--no-property-file-upload'
)),
container
=
ValueArgument
(
'Remote image container'
,
(
'-C'
,
'--container'
)),
fileowner
=
ValueArgument
(
'UUID of the user who owns the image file'
,
(
'--fileowner'
))
metafile
=
ValueArgument
(
'Load metadata from a json-formated file <img-file>.meta :'
'{"key1": "val1", "key2": "val2", ..., "properties: {...}"}'
,
(
'--metafile'
)),
metafile_force
=
FlagArgument
(
'Store remote metadata object, even if it already exists'
,
(
'-f'
,
'--force'
)),
no_metafile_upload
=
FlagArgument
(
'Do not store metadata in remote meta file'
,
(
'--no-metafile-upload'
)),
)
def
_get_uuid
(
self
):
uuid
=
self
[
'fileowner'
]
or
self
.
config
.
get
(
'image'
,
'fileowner'
)
if
uuid
:
return
uuid
atoken
=
self
.
client
.
token
user
=
AstakosClient
(
self
.
config
.
get
(
'user'
,
'url'
),
atoken
)
return
user
.
term
(
'uuid'
)
def
_get_pithos_client
(
self
,
uuid
,
container
):
def
_get_pithos_client
(
self
,
container
):
if
self
[
'no_metafile_upload'
]:
return
None
purl
=
self
.
config
.
get
(
'file'
,
'url'
)
ptoken
=
self
.
client
.
token
return
PithosClient
(
purl
,
ptoken
,
uuid
,
container
)
return
PithosClient
(
purl
,
ptoken
,
self
.
_get_
uuid
()
,
container
)
def
_store_remote_
property_
file
(
self
,
pclient
,
remote_path
,
properties
):
def
_store_remote_
meta
file
(
self
,
pclient
,
remote_path
,
metadata
):
return
pclient
.
upload_from_string
(
remote_path
,
_validate_image_props
(
properties
,
return_str
=
True
))
def
_get_container_path
(
self
,
container_path
):
container
=
self
[
'container'
]
or
self
.
config
.
get
(
'image'
,
'container'
)
if
container
:
return
container
,
container_path
container
,
sep
,
path
=
container_path
.
partition
(
':'
)
if
not
sep
or
not
container
or
not
path
:
raiseCLIError
(
'%s is not a valid pithos+ remote location'
%
container_path
,
importance
=
2
,
details
=
[
'To set "image" as container and "my_dir/img.diskdump" as'
,
'the image path, try one of the following as '
'container:path'
,
'- <image container>:<remote path>'
,
' e.g. image:/my_dir/img.diskdump'
,
'- <remote path> -C <image container>'
,
' e.g. /my_dir/img.diskdump -C image'
])
return
container
,
path
remote_path
,
_validate_image_meta
(
metadata
,
return_str
=
True
))
@
errors
.
generic
.
all
@
errors
.
plankton
.
image_file
@
errors
.
plankton
.
connection
def
_run
(
self
,
name
,
container_path
):
container
,
path
=
self
.
_get_container_path
(
container_path
)
uuid
=
self
.
_get_uuid
()
prop_path
=
'%s.meta'
%
path
pclient
=
None
if
(
self
[
'no_prop_file_upload'
])
else
self
.
_get_pithos_client
(
uuid
,
container
)
if
pclient
and
not
self
[
'prop_file_force'
]:
def
_load_params_from_file
(
self
,
location
):
params
,
properties
=
dict
(),
dict
()
pfile
=
self
[
'metafile'
]
if
pfile
:
try
:
pclient
.
get_object_info
(
prop_path
)
raiseCLIError
(
'Property file %s: %s already exists'
%
(
container
,
prop_path
))
except
ClientError
as
ce
:
if
ce
.
status
!=
404
:
raise
location
=
'pithos://%s/%s/%s'
%
(
uuid
,
container
,
path
)
for
k
,
v
in
_load_image_meta
(
pfile
).
items
():
key
=
k
.
lower
().
replace
(
'-'
,
'_'
)
if
k
==
'properties'
:
for
pk
,
pv
in
v
.
items
():
properties
[
pk
.
upper
().
replace
(
'-'
,
'_'
)]
=
pv
elif
key
==
'name'
:
continue
elif
key
==
'location'
:
if
location
:
continue
location
=
v
else
:
params
[
key
]
=
v
except
Exception
as
e
:
raiseCLIError
(
e
,
'Invalid json metadata config file'
)
return
params
,
properties
,
location
params
=
{}
def
_load_params_from_args
(
self
,
params
,
properties
):
for
key
in
set
([
'checksum'
,
'container_format'
,
...
...
@@ -336,47 +344,78 @@ class image_register(_init_image, _optional_json):
'size'
,
'is_public'
]).
intersection
(
self
.
arguments
):
params
[
key
]
=
self
[
key
]
properties
=
self
[
'properties'
]
for
k
,
v
in
self
[
'properties'
].
items
():
properties
[
k
.
upper
().
replace
(
'-'
,
'_'
)]
=
v
#load properties
properties
=
dict
()
pfile
=
self
[
'property_file'
]
if
pfile
:
def
_validate_location
(
self
,
location
):
if
not
location
:
raiseCLIError
(
'No image file location provided'
,
importance
=
2
,
details
=
[
'An image location is needed. Image location format:'
,
' pithos://<uuid>/<container>/<path>'
,
' an image file at the above location must exist.'
]
+
howto_image_file
)
try
:
return
_validate_image_location
(
location
)
except
AssertionError
as
ae
:
raiseCLIError
(
ae
,
'Invalid image location format'
,
importance
=
1
,
details
=
[
'Valid image location format:'
,
' pithos://<uuid>/<container>/<img-file-path>'
]
+
howto_image_file
)
@
errors
.
generic
.
all
@
errors
.
plankton
.
connection
def
_run
(
self
,
name
,
location
):
(
params
,
properties
,
location
)
=
self
.
_load_params_from_file
(
location
)
uuid
,
container
,
img_path
=
self
.
_validate_location
(
location
)
self
.
_load_params_from_args
(
params
,
properties
)
pclient
=
self
.
_get_pithos_client
(
container
)
#check if metafile exists
meta_path
=
'%s.meta'
%
img_path
if
pclient
and
not
self
[
'metafile_force'
]:
try
:
for
k
,
v
in
_load_image_props
(
pfile
).
items
():
properties
[
k
.
lower
()]
=
v
except
Exception
as
e
:
pclient
.
get_object_info
(
meta_path
)
raiseCLIError
(
'Metadata file %s:%s already exists'
%
(
container
,
meta_path
))
except
ClientError
as
ce
:
if
ce
.
status
!=
404
:
raise
#register the image
try
:
r
=
self
.
client
.
register
(
name
,
location
,
params
,
properties
)
except
ClientError
as
ce
:
if
ce
.
status
in
(
400
,
):
raiseCLIError
(
e
,
'
Format error in property file %s'
%
pfile
,
c
e
,
'
Nonexistent image file location %s'
%
location
,
details
=
[
'Expected content format:'
,
' {'
,
' "key1": "value1",'
,
' "key2": "value2",'
,
' ...'
,
' }'
,
''
,
'Parser:'
])
for
k
,
v
in
self
[
'properties'
].
items
():
properties
[
k
.
lower
()]
=
v
self
.
_print
([
self
.
client
.
register
(
name
,
location
,
params
,
properties
)])
'Make sure the image file exists'
]
+
howto_image_file
)
raise
self
.
_print
(
r
,
print_dict
)
#upload the metadata file
if
pclient
:
prop_headers
=
pclient
.
upload_from_string
(
prop_path
,
_validate_image_props
(
properties
,
return_str
=
True
))
try
:
meta_headers
=
pclient
.
upload_from_string
(
meta_path
,
dumps
(
r
,
indent
=
2
))
except
TypeError
:
print
(
'Failed to dump metafile %s:%s'
%
(
container
,
meta_path
))
return
if
self
[
'json_output'
]:
print_json
(
dict
(
property_
file_location
=
'%s:%s'
%
(
container
,
prop
_path
),
headers
=
prop
_headers
))
meta
file_location
=
'%s:%s'
%
(
container
,
meta
_path
),
headers
=
meta
_headers
))
else
:
print
(
'
Property
file uploaded as %s:%s (version %s)'
%
(
container
,
prop
_path
,
prop
_headers
[
'x-object-version'
]))
print
(
'
Metadata
file uploaded as %s:%s (version %s)'
%
(
container
,
meta
_path
,
meta
_headers
[
'x-object-version'
]))
def
main
(
self
,
name
,
container___path
):
def
main
(
self
,
name
,
location
=
None
):
super
(
self
.
__class__
,
self
).
_run
()
self
.
_run
(
name
,
container___path
)
self
.
_run
(
name
,
location
)
@
command
(
image_cmds
)
...
...
kamaki/clients/image/__init__.py
View file @
fb3998bd
...
...
@@ -35,6 +35,22 @@ from kamaki.clients import Client, ClientError
from
kamaki.clients.utils
import
path4url
,
filter_in
def
_format_image_headers
(
headers
):
reply
=
dict
(
properties
=
dict
())
meta_prefix
=
'x-image-meta-'
property_prefix
=
'x-image-meta-property-'
for
key
,
val
in
headers
.
items
():
key
=
key
.
lower
()
if
key
.
startswith
(
property_prefix
):
key
=
key
[
len
(
property_prefix
):].
upper
().
replace
(
'-'
,
'_'
)
reply
[
'properties'
][
key
]
=
val
elif
key
.
startswith
(
meta_prefix
):
key
=
key
[
len
(
meta_prefix
):]
reply
[
key
]
=
val
return
reply
class
ImageClient
(
Client
):
"""Synnefo Plankton API client"""
...
...
@@ -80,23 +96,7 @@ class ImageClient(Client):
path
=
path4url
(
'images'
,
image_id
)
r
=
self
.
head
(
path
,
success
=
200
)
reply
=
{}
properties
=
{}
meta_prefix
=
'x-image-meta-'
property_prefix
=
'x-image-meta-property-'
for
key
,
val
in
r
.
headers
.
items
():
key
=
key
.
lower
()
if
key
.
startswith
(
property_prefix
):
key
=
key
[
len
(
property_prefix
):]
properties
[
key
]
=
val
elif
key
.
startswith
(
meta_prefix
):
key
=
key
[
len
(
meta_prefix
):]
reply
[
key
]
=
val
if
properties
:
reply
[
'properties'
]
=
properties
return
reply
return
_format_image_headers
(
r
.
headers
)
def
register
(
self
,
name
,
location
,
params
=
{},
properties
=
{}):
"""Register an image that is uploaded at location
...
...
@@ -110,7 +110,7 @@ class ImageClient(Client):
:param properties: (dict) image properties (X-Image-Meta-Property)
:returns: (dict)
details
of the created image
:returns: (dict)
metadata
of the created image
"""
path
=
path4url
(
'images'
)
+
'/'
self
.
set_header
(
'X-Image-Meta-Name'
,
name
)
...
...
@@ -127,7 +127,8 @@ class ImageClient(Client):
async_headers
[
'x-image-meta-property-%s'
%
key
]
=
val
r
=
self
.
post
(
path
,
success
=
200
,
async_headers
=
async_headers
)
return
filter_in
(
r
.
headers
,
'X-Image-'
)
return
_format_image_headers
(
r
.
headers
)
def
unregister
(
self
,
image_id
):
"""Unregister an image
...
...
kamaki/clients/image/test.py
View file @
fb3998bd
...
...
@@ -228,7 +228,8 @@ class ImageClient(TestCase):
params
=
params
,
properties
=
props
)
expectedict
=
dict
(
example_image_headers
)
expectedict
.
pop
(
'extraheaders'
)
self
.
assert_dicts_are_equal
(
expectedict
,
r
)
from
kamaki.clients.image
import
_format_image_headers
self
.
assert_dicts_are_equal
(
_format_image_headers
(
expectedict
),
r
)
self
.
assertEqual
(
post
.
mock_calls
[
-
1
],
call
(
'/images/'
,
async_headers
=
async_headers
,
success
=
200
))
...
...
kamaki/clients/livetest/image.py
View file @
fb3998bd
...
...
@@ -85,7 +85,7 @@ class Image(livetest.Generic):
self
.
location
,
params
=
dict
(
is_public
=
True
))
self
.
_imglist
[
self
.
imgname
]
=
dict
(
name
=
r
[
'
x-image-meta-
name'
],
id
=
r
[
'
x-image-meta-
id'
])
name
=
r
[
'name'
],
id
=
r
[
'id'
])
self
.
_imgdetails
[
self
.
imgname
]
=
r
def
tearDown
(
self
):
...
...
@@ -154,7 +154,7 @@ class Image(livetest.Generic):
'properties'
,
'size'
):
self
.
assertTrue
(
term
in
img
)
if
img
[
'properties'
]:
if
len
(
img
[
'properties'
]
)
:
for
interm
in
(
'osfamily'
,
'users'
,
'root_partition'
):
self
.
assertTrue
(
interm
in
img
[
'properties'
])
size_max
=
1000000000
...
...
@@ -185,13 +185,14 @@ class Image(livetest.Generic):
'container-format'
):
self
.
assertTrue
(
term
in
r
)
for
interm
in
(
'kernel'
,
'osfamily'
,
'users'
,
'gui'
,
'sortorder'
,
'root-partition'
,
'os'
,
'description'
):
'KERNEL'
,
'OSFAMILY'
,
'USERS'
,
'GUI'
,
'SORTORDER'
,
'ROOT_PARTITION'
,
'OS'
,
'DESCRIPTION'
):
self
.
assertTrue
(
interm
in
r
[
'properties'
])
def
test_register
(
self
):
...
...
@@ -204,8 +205,7 @@ class Image(livetest.Generic):
for
img
in
self
.
_imglist
.
values
():
self
.
assertTrue
(
img
is
not
None
)
r
=
set
(
self
.
_imgdetails
[
img
[
'name'
]].
keys
())
self
.
assertTrue
(
r
.
issubset
([
'x-image-meta-%s'
%
k
for
k
in
IMGMETA
]))
self
.
assertTrue
(
r
.
issubset
(
IMGMETA
.
union
([
'properties'
])))
def
test_unregister
(
self
):
"""Test unregister"""
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment