Commit 3dce53e9 authored by Georgios D. Tsoukalas's avatar Georgios D. Tsoukalas
Browse files

Merge branch 'origin/devel-0.13' into quota

Conflicts:
	snf-cyclades-app/synnefo/api/servers.py
parents 969d149d d1202ac5
......@@ -31,6 +31,9 @@
# interpreted as representing official policies, either expressed
# or implied, of GRNET S.A.
from base64 import b64decode
from django import dispatch
from django.conf import settings
from django.conf.urls.defaults import patterns
from django.db import transaction
......@@ -48,6 +51,8 @@ from synnefo.logic.rapi import GanetiApiError
from synnefo.logic.backend_allocator import BackendAllocator
from synnefo import quotas
# server creation signal
server_created = dispatch.Signal(providing_args=["created_vm_params"])
from logging import getLogger
log = getLogger('synnefo.api')
......@@ -274,10 +279,32 @@ def create_server(serials, request):
if len(personality) > settings.MAX_PERSONALITY:
raise faults.OverLimit("Maximum number of personalities exceeded")
util.verify_personality(personality)
image = util.get_image_dict(image_id, user_id)
flavor = util.get_flavor(flavor_id)
password = util.random_password()
for p in personality:
# Verify that personalities are well-formed
try:
assert isinstance(p, dict)
keys = set(p.keys())
allowed = set(['contents', 'group', 'mode', 'owner', 'path'])
assert keys.issubset(allowed)
contents = p['contents']
if len(contents) > settings.MAX_PERSONALITY_SIZE:
# No need to decode if contents already exceed limit
raise faults.OverLimit("Maximum size of personality exceeded")
if len(b64decode(contents)) > settings.MAX_PERSONALITY_SIZE:
raise faults.OverLimit("Maximum size of personality exceeded")
except AssertionError:
raise faults.BadRequest("Malformed personality in request")
image = {}
img = util.get_image(image_id, request.user_uniq)
properties = img.get('properties', {})
image['backend_id'] = img['location']
image['format'] = img['disk_format']
image['metadata'] = dict((key.upper(), val) \
for key, val in properties.items())
# Ensure that request if for active flavor
flavor = util.get_flavor(flavor_id, include_deleted=False)
backend_allocator = BackendAllocator()
backend = backend_allocator.allocate(request.user_uniq, flavor)
......@@ -321,9 +348,29 @@ def create_server(serials, request):
action="CREATE",
serial=serial)
password = util.random_password()
# TODO: Just copied code from backend.py to fix the images backend_id
# for archipelagos. Find a better way and remove double checks
img_id = image['backend_id']
provider = None
disk_template = flavor.disk_template
if flavor.disk_template.startswith("ext"):
disk_template, provider = flavor.disk_template.split("_", 1)
if provider == 'vlmc':
img_id = 'null'
# dispatch server created signal
server_created.send(sender=vm, created_vm_params={
'img_id': img_id,
'img_passwd': password,
'img_format': str(image['format']),
'img_personality': str(personality),
'img_properties': str(image['metadata']),
})
try:
jobID = create_instance(vm, nic, flavor, image, password,
personality)
jobID = create_instance(vm, nic, flavor, image)
except GanetiApiError:
vm.delete()
raise
......
......@@ -64,7 +64,7 @@ from synnefo.db.models import (Flavor, VirtualMachine, VirtualMachineMetadata,
from synnefo.db.pools import EmptyPool
from synnefo.lib.astakos import get_user
from synnefo.plankton.backend import ImageBackend
from synnefo.plankton.backend import ImageBackend, NotAllowedError
from synnefo.settings import MAX_CIDR_BLOCK
......@@ -442,6 +442,10 @@ def api_method(http_method=None, atom_allowed=False):
except VirtualMachine.BuildingError:
fault = BuildInProgress('Server is being built.')
return render_fault(request, fault)
except NotAllowedError:
# Image Backend Unathorized
fault = Forbidden('Request not allowed.')
return render_fault(request, fault)
except Fault, fault:
if fault.code >= 500:
log.exception('API fault')
......
......@@ -5,6 +5,7 @@ synnefo_web_apps = [
'synnefo.db',
'synnefo.logic',
'synnefo.plankton',
'synnefo.vmapi',
'synnefo.helpdesk',
'synnefo.ui.userdata',
'synnefo.helpdesk',
......
......@@ -92,12 +92,6 @@ DEFAULT_FIREWALL_PROFILE = 'DISABLED'
# our REST API would prefer to be explicit about trailing slashes
APPEND_SLASH = False
# Ignore disk size specified by flavor, always build the
# machine with a 4GB (in the case of Windows: 14GB) disk.
# This setting is helpful in development setups.
#
IGNORE_FLAVOR_DISK_SIZES = False
# Fixed mapping of user VMs to a specific backend.
# e.g. BACKEND_PER_USER = {'example@okeanos.grnet.gr': 2}
BACKEND_PER_USER = {}
......
......@@ -39,6 +39,7 @@ urlpatterns = patterns('',
name='ui_machines_console'),
url(r'^machines/connect$', 'synnefo.ui.views.machines_connect',
name='ui_machines_connect'),
(r'^vmapi/', include('synnefo.vmapi.urls')),
(r'^api/', include('synnefo.api.urls')),
(r'^plankton/', include('synnefo.plankton.urls')),
(r'^helpdesk/', include('synnefo.helpdesk.urls')),
......
......@@ -349,21 +349,13 @@ def create_instance_diagnostic(vm, message, source, level="DEBUG", etime=None,
source_date=etime, message=message, details=details)
def create_instance(vm, public_nic, flavor, image, password, personality):
def create_instance(vm, public_nic, flavor, image, password=None):
"""`image` is a dictionary which should contain the keys:
'backend_id', 'format' and 'metadata'
metadata value should be a dictionary.
"""
if settings.IGNORE_FLAVOR_DISK_SIZES:
if image['backend_id'].find("windows") >= 0:
sz = 14000
else:
sz = 4000
else:
sz = flavor.disk * 1024
# Handle arguments to CreateInstance() as a dictionary,
# initialize it based on a deployment-specific value.
# This enables the administrator to override deployment-specific
......@@ -385,7 +377,7 @@ def create_instance(vm, public_nic, flavor, image, password, personality):
disk_template, provider = flavor.disk_template.split("_", 1)
kw['disk_template'] = disk_template
kw['disks'] = [{"size": sz}]
kw['disks'] = [{"size": flavor.disk * 1024}]
if provider:
kw['disks'][0]['provider'] = provider
......@@ -399,31 +391,36 @@ def create_instance(vm, public_nic, flavor, image, password, personality):
# kw['os'] = settings.GANETI_OS_PROVIDER
kw['ip_check'] = False
kw['name_check'] = False
# Do not specific a node explicitly, have
# Ganeti use an iallocator instead
#
#kw['pnode'] = rapi.GetNodes()[0]
kw['dry_run'] = settings.TEST
kw['beparams'] = {
'auto_balance': True,
'vcpus': flavor.cpu,
'memory': flavor.ram}
'auto_balance': True,
'vcpus': flavor.cpu,
'memory': flavor.ram}
if provider == 'vlmc':
image_id = 'null'
else:
image_id = image['backend_id']
kw['osparams'] = {
'img_id': image['backend_id'],
'img_passwd': password,
'config_url': vm.config_url,
# Store image id and format to Ganeti
'img_id': image_id,
'img_format': image['format']}
if personality:
kw['osparams']['img_personality'] = json.dumps(personality)
if provider != None and provider == 'vlmc':
kw['osparams']['img_id'] = 'null'
kw['osparams']['img_properties'] = json.dumps(image['metadata'])
if password:
# Only for admin created VMs !!
kw['osparams']['img_passwd'] = password
# Defined in settings.GANETI_CREATEINSTANCE_KWARGS
# kw['hvparams'] = dict(serial_console=False)
log.debug("Creating instance %s", utils.hide_pass(kw))
with pooled_rapi_client(vm) as client:
return client.CreateInstance(**kw)
......
......@@ -54,11 +54,12 @@ import json
import warnings
from operator import itemgetter
from time import gmtime, strftime, time
from time import gmtime, strftime
from functools import wraps
from django.conf import settings
from pithos.backends.base import NotAllowedError
from pithos.backends.base import NotAllowedError as PithosNotAllowedError
PLANKTON_DOMAIN = 'plankton'
......@@ -86,6 +87,10 @@ class BackendException(Exception):
pass
class NotAllowedError(BackendException):
pass
from pithos.backends.util import PithosBackendPool
POOL_SIZE = 8
_pithos_backend_pool = \
......@@ -98,6 +103,16 @@ def get_pithos_backend():
return _pithos_backend_pool.pool_get()
def handle_backend_exceptions(func):
@wraps(func)
def wrapper(*args, **kwargs):
try:
return func(*args, **kwargs)
except PithosNotAllowedError:
raise NotAllowedError()
return wrapper
class ImageBackend(object):
"""A wrapper arround the pithos backend to simplify image handling."""
......@@ -109,6 +124,7 @@ class ImageBackend(object):
self.backend = get_pithos_backend()
warnings.filters = original_filters # Restore warnings
@handle_backend_exceptions
def _get_image(self, location):
def format_timestamp(t):
return strftime('%Y-%m-%d %H:%M:%S', gmtime(t))
......@@ -159,6 +175,7 @@ class ImageBackend(object):
return image
@handle_backend_exceptions
def _get_meta(self, location, version=None):
account, container, object = split_location(location)
try:
......@@ -167,12 +184,14 @@ class ImageBackend(object):
except NameError:
return None
@handle_backend_exceptions
def _get_permissions(self, location):
account, container, object = split_location(location)
action, path, permissions = self.backend.get_object_permissions(
self.user, account, container, object)
return permissions
@handle_backend_exceptions
def _store(self, f, size=None):
"""Breaks data into blocks and stores them in the backend"""
......@@ -193,6 +212,7 @@ class ImageBackend(object):
return hashmap, bytes
@handle_backend_exceptions
def _update(self, location, size, hashmap, meta, permissions):
account, container, object = split_location(location)
self.backend.update_object_hashmap(self.user, account, container,
......@@ -200,6 +220,7 @@ class ImageBackend(object):
permissions=permissions)
self._update_meta(location, meta, replace=True)
@handle_backend_exceptions
def _update_meta(self, location, meta, replace=False):
account, container, object = split_location(location)
......@@ -213,11 +234,13 @@ class ImageBackend(object):
self.backend.update_object_meta(self.user, account, container, object,
PLANKTON_DOMAIN, prefixed, replace)
@handle_backend_exceptions
def _update_permissions(self, location, permissions):
account, container, object = split_location(location)
self.backend.update_object_permissions(self.user, account, container,
object, permissions)
@handle_backend_exceptions
def add_user(self, image_id, user):
image = self.get_image(image_id)
assert image, "Image not found"
......@@ -232,11 +255,13 @@ class ImageBackend(object):
def close(self):
self.backend.close()
@handle_backend_exceptions
def delete(self, image_id):
image = self.get_image(image_id)
account, container, object = split_location(image['location'])
self.backend.delete_object(self.user, account, container, object)
@handle_backend_exceptions
def get_data(self, location):
account, container, object = split_location(location)
size, hashmap = self.backend.get_object_hashmap(self.user, account,
......@@ -245,6 +270,7 @@ class ImageBackend(object):
assert len(data) == size
return data
@handle_backend_exceptions
def get_image(self, image_id):
try:
account, container, object = self.backend.get_uuid(self.user,
......@@ -255,6 +281,7 @@ class ImageBackend(object):
location = get_location(account, container, object)
return self._get_image(location)
@handle_backend_exceptions
def _iter(self, public=False, filters=None, shared_from=None):
filters = filters or {}
......@@ -301,7 +328,7 @@ class ImageBackend(object):
def iter_shared(self, filters=None, member=None):
"""Iter over images shared to member"""
return self._iter(filters=filters)
return self._iter(filters=filters, shared_from=member)
def list(self, filters=None, params={}):
"""Return all images available to the user"""
......@@ -326,6 +353,7 @@ class ImageBackend(object):
permissions = self._get_permissions(image['location'])
return [user for user in permissions.get('read', []) if user != '*']
@handle_backend_exceptions
def put(self, name, f, params):
assert 'checksum' not in params, "Passing a checksum is not supported"
assert 'id' not in params, "Passing an ID is not supported"
......@@ -353,6 +381,7 @@ class ImageBackend(object):
self._update(location, size, hashmap, meta, permissions)
return self._get_image(location)
@handle_backend_exceptions
def register(self, name, location, params):
assert 'id' not in params, "Passing an ID is not supported"
assert location.startswith('pithos://'), "Invalid location"
......@@ -392,6 +421,7 @@ class ImageBackend(object):
self._update_permissions(location, permissions)
return self._get_image(location)
@handle_backend_exceptions
def remove_user(self, image_id, user):
image = self.get_image(image_id)
assert image, "Image not found"
......@@ -404,6 +434,7 @@ class ImageBackend(object):
return # User did not have access anyway
self._update_permissions(location, permissions)
@handle_backend_exceptions
def replace_users(self, image_id, users):
image = self.get_image(image_id)
assert image, "Image not found"
......@@ -415,6 +446,7 @@ class ImageBackend(object):
permissions['read'].append('*')
self._update_permissions(location, permissions)
@handle_backend_exceptions
def update(self, image_id, params):
image = self.get_image(image_id)
assert image, "Image not found"
......@@ -437,4 +469,3 @@ class ImageBackend(object):
self._update_meta(location, meta)
return self.get_image(image_id)
......@@ -36,9 +36,12 @@ from synnefo.plankton.backend import ImageBackend
class Command(BaseCommand):
help = "List public images or images available to a user."
option_list = BaseCommand.option_list + (
make_option('--user-id', dest='userid',
help="List all images available to that user"),
help="List all images available to that user."\
" If no user is specified, only public images"\
" are displayed."),
)
def handle(self, **options):
......
......@@ -29,6 +29,7 @@
#
from django.core.management.base import BaseCommand, CommandError
from optparse import make_option
from synnefo.plankton.backend import ImageBackend
from pprint import pprint
......@@ -37,6 +38,10 @@ from pprint import pprint
class Command(BaseCommand):
args = "<image_id>"
help = "Display available information about an image"
option_list = BaseCommand.option_list + (
make_option('--user-id', dest='userid',
help="The ID of the owner of the Image."),
)
def handle(self, *args, **options):
......@@ -44,7 +49,8 @@ class Command(BaseCommand):
raise CommandError("Please provide an image ID")
image_id = args[0]
c = ImageBackend("")
userid = options['userid']
c = ImageBackend(userid) if userid else ImageBackend("")
image = c.get_image(image_id)
if not image:
......
......@@ -39,7 +39,7 @@ from synnefo.plankton import views
def demux(request):
if request.method == 'GET':
return views.list_public_images(request)
return views.list_images(request)
elif request.method == 'POST':
return views.add_image(request)
else:
......@@ -77,11 +77,9 @@ def demux_members(request, image_id, member):
urlpatterns = patterns('',
(r'^images/$', demux),
(r'^images/detail$', views.list_public_images, {'detail': True}),
(r'^images/detail$', views.list_images, {'detail': True}),
(r'^images/([\w-]+)$', demux_image),
(r'^images/([\w-]+)/members$', demux_image_members),
(r'^images/([\w-]+)/members/([\w@._-]+)$', demux_members),
(r'^shared-images/$', views.list_shared_images),
(r'^shared-images/detail', views.list_shared_images, {'detail': True}),
(r'^shared-images/([\w@._-]+)$', views.list_shared_images_with)
(r'^shared-images/([\w@._-]+)$', views.list_shared_images)
)
......@@ -31,8 +31,6 @@
# interpreted as representing official policies, either expressed
# or implied, of GRNET S.A.
import datetime
from functools import wraps
from logging import getLogger
from traceback import format_exc
......@@ -42,9 +40,8 @@ from django.http import (HttpResponse, HttpResponseBadRequest,
HttpResponseServerError, HttpResponseForbidden)
from synnefo.lib.astakos import get_user
from synnefo.plankton.backend import ImageBackend, BackendException
from pithos.backends.base import NotAllowedError
from synnefo.plankton.backend import (ImageBackend, BackendException,
NotAllowedError)
log = getLogger('synnefo.plankton')
......
......@@ -226,16 +226,12 @@ def list_image_members(request, image_id):
@plankton_method('GET')
def list_public_images(request, detail=False):
"""Return a list of public VM images.
def list_images(request, detail=False):
"""Return a list of available images.
Described in:
3.1. Requesting a List of Public VM Images
3.2. Requesting Detailed Metadata on Public VM Images
3.3. Filtering Images Returned via GET /images andGET /images/detail
This includes images owned by the user, images shared with the user and
public images.
Extensions:
* Image ID is returned in both compact and detail listings
"""
def get_request_params(keys):
......@@ -257,7 +253,7 @@ def list_public_images(request, detail=False):
assert params['sort_key'] in SORT_KEY_OPTIONS
assert params['sort_dir'] in SORT_DIR_OPTIONS
images = request.backend.list_public(filters, params)
images = request.backend.list(filters, params)
# Remove keys that should not be returned
fields = DETAIL_FIELDS if detail else LIST_FIELDS
......@@ -271,7 +267,7 @@ def list_public_images(request, detail=False):
@plankton_method('GET')
def list_shared_images_with(request, member):
def list_shared_images(request, member):
"""Request shared images
Described in:
......@@ -282,10 +278,10 @@ def list_shared_images_with(request, member):
the users's images that are accessible by `member`.
"""
log.debug('list_shared_images_with %s', member)
log.debug('list_shared_images %s', member)
images = []
for image in request.backend.iter_shared(member):
for image in request.backend.iter_shared(member=member):
image_id = image['id']
images.append({'image_id': image_id, 'can_share': False})
......@@ -350,38 +346,3 @@ def update_image_members(request, image_id):
request.backend.replace_users(image_id, members)
return HttpResponse(status=204)
@plankton_method('GET')
def list_shared_images(request, detail=False):
def get_request_params(keys):
params = {}
for key in keys:
val = request.GET.get(key, None)
if val is not None:
params[key] = val
return params
log.debug('list_shared_images detail=%s, request %s', detail, request)
filters = get_request_params(FILTERS)
params = get_request_params(PARAMS)
params.setdefault('sort_key', 'created_at')
params.setdefault('sort_dir', 'desc')
assert params['sort_key'] in SORT_KEY_OPTIONS
assert params['sort_dir'] in SORT_DIR_OPTIONS
images = request.backend.list(filters, params)
images = filter(lambda x: not x['is_public'], images)
# Remove keys that should not be returned
fields = DETAIL_FIELDS if detail else LIST_FIELDS
for image in images:
for key in image.keys():
if key not in fields:
del image[key]
data = json.dumps(images, indent=settings.DEBUG)
return HttpResponse(data)
......@@ -1100,8 +1100,8 @@
},
// fix left border position
fix_left_border: function() {
if (!this.nics_visible) { return };
fix_left_border: function(force) {
if (!this.nics_visible && !force) { return };
var imgheight = 2783;
var opened_vm_height = 133 + 18;
......@@ -1137,9 +1137,14 @@
initialize: function(network, view) {
views.PublicNetworkView.__super__.initialize.call(this, network, view);
this.fix_left_border(1);
},
init_handlers: function(vm) {}
init_handlers: function(vm) {
$(window).bind("resize", _.bind(function() {
this.fix_left_border();
}, this));
}
});
views.GroupedPublicNetworkView = views.PublicNetworkView.extend({
......