Commit fc96f8c6 authored by Christos Stavrakakis's avatar Christos Stavrakakis
Browse files

cyclades: Create servers with volumes

Extend Cyclades POST /servers API endpoint to support specifying volumes
when creating a server. The call is extended with the
'block_device_mapping_v2' field that contains a list of dictionaries,
each of which contains the following information about the devices that
the server must have:

* source_type: One of'image', 'snapshot', 'volume' or 'blank'
* uuid: The UUID of the object that is specified by source_type (if
        source_type is not blank)
* size: The size of the volume
* delete_on_termination: Whether the volume will be preserved or
                         automaticaly deleted when the server is deleted

If no volume is specified, then a volume is automatically created with
the size of the flavor and with source the image that is specified in
the 'imageRef' field of the API call.

The follow restrictions apply:
* Source_type can be other than 'blank', only if the disk template is
  ext_ or if the volume is created as a root volume of a server, so
  it will be filled with data by snf-image.
* The root volume cannot be blank.
* In order to use a volume, it must be in AVAILABLE status. This will
  never happen until detachable volumes are implemented. However, the
  user can "clone" a user volume either by creating a new volume from
  cinder and then using this new volume, or by taking a snapshot and then
  use the snapshot as a source.
* The size field is required if the source is image, snapshot or blank.
  Also, the size of the volume must be equal or bigger from the source
  size (if any).

Finally, move some code from 'servers' module to 'server_attachments'
and 'commands' to avoid cyclic imports.
parent cbdfd23a
......@@ -66,6 +66,13 @@ urlpatterns = patterns(
(r'^/(\d+)/os-volume_attachments/(\d+)(?:.json)?$', 'demux_volumes_item'),
)
VOLUME_SOURCE_TYPES = [
"snapshot",
"image",
"volume",
"blank"
]
def demux(request):
if request.method == 'GET':
......@@ -414,10 +421,13 @@ def create_server(request):
except (KeyError, AssertionError):
raise faults.BadRequest("Malformed request")
volumes = None
dev_map = server.get("block_device_mapping_v2")
if dev_map is not None:
volumes = parse_block_device_mapping(dev_map)
# Verify that personalities are well-formed
util.verify_personality(personality)
# Get image information
image = util.get_image_dict(image_id, user_id)
# Get flavor (ensure it is active)
flavor = util.get_flavor(flavor_id, include_deleted=False)
if not flavor.allow_create:
......@@ -427,9 +437,9 @@ def create_server(request):
# Generate password
password = util.random_password()
vm = servers.create(user_id, name, password, flavor, image,
vm = servers.create(user_id, name, password, flavor, image_id,
metadata=metadata, personality=personality,
project=project, networks=networks)
project=project, networks=networks, volumes=volumes)
server = vm_to_dict(vm, detail=True)
server['status'] = 'BUILD'
......@@ -440,6 +450,65 @@ def create_server(request):
return response
def parse_block_device_mapping(dev_map):
"""Parse 'block_device_mapping_v2' attribute"""
if not isinstance(dev_map, list):
raise faults.BadRequest("Block Device Mapping is Invalid")
return [_parse_block_device(device) for device in dev_map]
def _parse_block_device(device):
"""Parse and validate a block device mapping"""
if not isinstance(device, dict):
raise faults.BadRequest("Block Device Mapping is Invalid")
# Validate source type
source_type = device.get("source_type")
if source_type is None:
raise faults.BadRequest("Block Device Mapping is Invalid: Invalid"
" source_type field")
elif source_type not in VOLUME_SOURCE_TYPES:
raise faults.BadRequest("Block Device Mapping is Invalid: source_type"
" must be on of %s"
% ", ".join(VOLUME_SOURCE_TYPES))
# Validate source UUID
uuid = device.get("uuid")
if uuid is None and source_type != "blank":
raise faults.BadRequest("Block Device Mapping is Invalid: uuid of"
" %s is missing" % source_type)
# Validate volume size
size = device.get("volume_size")
if size is not None:
try:
size = int(size)
except (TypeError, ValueError):
raise faults.BadRequest("Block Device Mapping is Invalid: Invalid"
" size field")
# Validate 'delete_on_termination'
delete_on_termination = device.get("delete_on_termination")
if delete_on_termination is not None:
if not isinstance(delete_on_termination, bool):
raise faults.BadRequest("Block Device Mapping is Invalid: Invalid"
" delete_on_termination field")
else:
if source_type == "volume":
delete_on_termination = False
else:
delete_on_termination = True
# Unused API Attributes
# boot_index = device.get("boot_index")
# destination_type = device.get("destination_type")
return {"source_type": source_type,
"source_uuid": uuid,
"size": size,
"delete_on_termination": delete_on_termination}
@api.api_method(http_method='GET', user_required=True, logger=log)
def get_server_details(request, server_id):
# Normal Response Codes: 200, 203
......@@ -994,7 +1063,7 @@ def volume_to_attachment(volume):
return {"id": volume.id,
"volumeId": volume.id,
"serverId": volume.machine_id,
"device": ""} # TODO: What device to return?
"device": ""} # TODO: What device to return?
@api.api_method(http_method='GET', user_required=True, logger=log)
......
......@@ -38,7 +38,7 @@ from copy import deepcopy
from snf_django.utils.testing import (BaseAPITest, mocked_quotaholder,
override_settings)
from synnefo.db.models import (VirtualMachine, VirtualMachineMetadata,
IPAddress, NetworkInterface)
IPAddress, NetworkInterface, Volume)
from synnefo.db import models_factory as mfactory
from synnefo.logic.utils import get_rsapi_state
from synnefo.cyclades_settings import cyclades_services
......@@ -329,11 +329,13 @@ fixed_image.return_value = {'location': 'pithos://foo',
'checksum': '1234',
"id": 1,
"name": "test_image",
"size": "41242",
"size": 1024,
"is_snapshot": False,
'disk_format': 'diskdump'}
@patch('synnefo.api.util.get_image', fixed_image)
@patch('synnefo.volume.util.get_snapshot', fixed_image)
@patch('synnefo.logic.rapi_pool.GanetiRapiClient')
class ServerCreateAPITest(ComputeAPITest):
def setUp(self):
......@@ -594,6 +596,89 @@ class ServerCreateAPITest(ComputeAPITest):
json.dumps(request), 'json')
self.assertEqual(response.status_code, 404)
def test_create_server_with_volumes(self, mrapi):
user = "test_user"
mrapi().CreateInstance.return_value = 42
# Test creation without any volumes. Server will use flavor+image
request = deepcopy(self.request)
request["server"]["block_device_mapping_v2"] = []
with mocked_quotaholder():
response = self.mypost("servers", user,
json.dumps(request), 'json')
self.assertEqual(response.status_code, 202, msg=response.content)
vm_id = json.loads(response.content)["server"]["id"]
volume = Volume.objects.get(machine_id=vm_id)
self.assertEqual(volume.disk_template, self.flavor.disk_template)
self.assertEqual(volume.size, self.flavor.disk)
self.assertEqual(volume.source, "image:%s" % fixed_image()["id"])
self.assertEqual(volume.delete_on_termination, True)
self.assertEqual(volume.userid, user)
# Test using an image
request["server"]["block_device_mapping_v2"] = [
{"source_type": "image",
"uuid": fixed_image()["id"],
"volume_size": 10,
"delete_on_termination": False}
]
with mocked_quotaholder():
response = self.mypost("servers", user,
json.dumps(request), 'json')
self.assertEqual(response.status_code, 202, msg=response.content)
vm_id = json.loads(response.content)["server"]["id"]
volume = Volume.objects.get(machine_id=vm_id)
self.assertEqual(volume.disk_template, self.flavor.disk_template)
self.assertEqual(volume.size, 10)
self.assertEqual(volume.source, "image:%s" % fixed_image()["id"])
self.assertEqual(volume.delete_on_termination, False)
self.assertEqual(volume.userid, user)
self.assertEqual(volume.origin, "pithos:" + fixed_image()["checksum"])
# Test using a snapshot
request["server"]["block_device_mapping_v2"] = [
{"source_type": "snapshot",
"uuid": fixed_image()["id"],
"volume_size": 10,
"delete_on_termination": False}
]
with mocked_quotaholder():
response = self.mypost("servers", user,
json.dumps(request), 'json')
self.assertEqual(response.status_code, 202, msg=response.content)
vm_id = json.loads(response.content)["server"]["id"]
volume = Volume.objects.get(machine_id=vm_id)
self.assertEqual(volume.disk_template, self.flavor.disk_template)
self.assertEqual(volume.size, 10)
self.assertEqual(volume.source, "snapshot:%s" % fixed_image()["id"])
self.assertEqual(volume.origin, fixed_image()["checksum"])
self.assertEqual(volume.delete_on_termination, False)
self.assertEqual(volume.userid, user)
source_volume = volume
# Test using source volume
request["server"]["block_device_mapping_v2"] = [
{"source_type": "volume",
"uuid": source_volume.id,
"volume_size": source_volume.size,
"delete_on_termination": False}
]
with mocked_quotaholder():
response = self.mypost("servers", user,
json.dumps(request), 'json')
# This will fail because the volume is not AVAILABLE.
self.assertBadRequest(response)
# Test using a blank volume
request["server"]["block_device_mapping_v2"] = [
{"source_type": "blank",
"volume_size": 10,
"delete_on_termination": True}
]
with mocked_quotaholder():
response = self.mypost("servers", user,
json.dumps(request), 'json')
self.assertBadRequest(response)
@patch('synnefo.logic.rapi_pool.GanetiRapiClient')
class ServerDestroyAPITest(ComputeAPITest):
......
......@@ -711,7 +711,7 @@ def create_instance(vm, nics, volumes, flavor, image):
kw['name'] = vm.backend_vm_id
# Defined in settings.GANETI_CREATEINSTANCE_KWARGS
kw['disk_template'] = flavor.disk_template
kw['disk_template'] = volumes[0].template
disks = []
for volume in volumes:
disk = {"name": volume.backend_volume_uuid,
......
# Copyright 2011, 2012, 2013 GRNET S.A. All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
# 1. Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
# SUCH DAMAGE.
#
# The views and conclusions contained in the software and documentation are
# those of the authors and should not be interpreted as representing official
# policies, either expressed or implied, of GRNET S.A.
import logging
from functools import wraps
from django.db import transaction
from django.conf import settings
from snf_django.lib.api import faults
from synnefo import quotas
from synnefo.db.models import VirtualMachine
log = logging.getLogger(__name__)
def validate_server_action(vm, action):
if vm.deleted:
raise faults.BadRequest("Server '%s' has been deleted." % vm.id)
# Destroyin a server should always be permitted
if action == "DESTROY":
return
# Check that there is no pending action
pending_action = vm.task
if pending_action:
if pending_action == "BUILD":
raise faults.BuildInProgress("Server '%s' is being build." % vm.id)
raise faults.BadRequest("Cannot perform '%s' action while there is a"
" pending '%s'." % (action, pending_action))
# Check if action can be performed to VM's operstate
operstate = vm.operstate
if operstate == "ERROR":
raise faults.BadRequest("Cannot perform '%s' action while server is"
" in 'ERROR' state." % action)
elif operstate == "BUILD" and action != "BUILD":
raise faults.BuildInProgress("Server '%s' is being build." % vm.id)
elif (action == "START" and operstate != "STOPPED") or\
(action == "STOP" and operstate != "STARTED") or\
(action == "RESIZE" and operstate != "STOPPED") or\
(action in ["CONNECT", "DISCONNECT"]
and operstate != "STOPPED"
and not settings.GANETI_USE_HOTPLUG) or \
(action in ["ATTACH_VOLUME", "DETACH_VOLUME"]
and operstate != "STOPPED"
and not settings.GANETI_USE_HOTPLUG):
raise faults.BadRequest("Cannot perform '%s' action while server is"
" in '%s' state." % (action, operstate))
return
def server_command(action, action_fields=None):
"""Handle execution of a server action.
Helper function to validate and execute a server action, handle quota
commission and update the 'task' of the VM in the DB.
1) Check if action can be performed. If it can, then there must be no
pending task (with the exception of DESTROY).
2) Handle previous commission if unresolved:
* If it is not pending and it to accept, then accept
* If it is not pending and to reject or is pending then reject it. Since
the action can be performed only if there is no pending task, then there
can be no pending commission. The exception is DESTROY, but in this case
the commission can safely be rejected, and the dispatcher will generate
the correct ones!
3) Issue new commission and associate it with the VM. Also clear the task.
4) Send job to ganeti
5) Update task and commit
"""
def decorator(func):
@wraps(func)
@transaction.commit_on_success
def wrapper(vm, *args, **kwargs):
user_id = vm.userid
validate_server_action(vm, action)
vm.action = action
commission_name = "client: api, resource: %s" % vm
quotas.handle_resource_commission(vm, action=action,
action_fields=action_fields,
commission_name=commission_name)
vm.save()
# XXX: Special case for server creation!
if action == "BUILD":
# Perform a commit, because the VirtualMachine must be saved to
# DB before the OP_INSTANCE_CREATE job in enqueued in Ganeti.
# Otherwise, messages will arrive from snf-dispatcher about
# this instance, before the VM is stored in DB.
transaction.commit()
# After committing the locks are released. Refetch the instance
# to guarantee x-lock.
vm = VirtualMachine.objects.select_for_update().get(id=vm.id)
# Send the job to Ganeti and get the associated jobID
try:
job_id = func(vm, *args, **kwargs)
except Exception as e:
if vm.serial is not None:
# Since the job never reached Ganeti, reject the commission
log.debug("Rejecting commission: '%s', could not perform"
" action '%s': %s" % (vm.serial, action, e))
transaction.rollback()
quotas.reject_resource_serial(vm)
transaction.commit()
raise
if action == "BUILD" and vm.serial is not None:
# XXX: Special case for server creation: we must accept the
# commission because the VM has been stored in DB. Also, if
# communication with Ganeti fails, the job will never reach
# Ganeti, and the commission will never be resolved.
quotas.accept_resource_serial(vm)
log.info("user: %s, vm: %s, action: %s, job_id: %s, serial: %s",
user_id, vm.id, action, job_id, vm.serial)
# store the new task in the VM
if job_id is not None:
vm.task = action
vm.task_job_id = job_id
vm.save()
return vm
return wrapper
return decorator
......@@ -118,7 +118,7 @@ class Command(SynnefoCommand):
backend = None
connection_list = parse_connections(options["connections"])
server = servers.create(user_id, name, password, flavor, image,
server = servers.create(user_id, name, password, flavor, image["id"],
networks=connection_list,
use_backend=backend)
pprint.pprint_server(server, stdout=self.stdout)
......
# Copyright 2011, 2012, 2013 GRNET S.A. All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
# 1. Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
# SUCH DAMAGE.
#
# The views and conclusions contained in the software and documentation are
# those of the authors and should not be interpreted as representing official
# policies, either expressed or implied, of GRNET S.A.
import logging
from snf_django.lib.api import faults
from django.conf import settings
from synnefo.logic import backend, commands
log = logging.getLogger(__name__)
@commands.server_command("ATTACH_VOLUME")
def attach_volume(vm, volume):
"""Attach a volume to a server.
The volume must be in 'AVAILABLE' status in order to be attached. Also,
number of the volumes that are attached to the server must remain less
than 'GANETI_MAX_DISKS_PER_INSTANCE' setting. This function will send
the corresponding job to Ganeti backend and update the status of the
volume to 'ATTACHING'.
"""
# Check volume state
if volume.status not in ["AVAILABLE", "CREATING"]:
raise faults.BadRequest("Cannot attach volume while volume is in"
" '%s' status." % volume.status)
# Check that disk templates are the same
if volume.disk_template != vm.flavor.disk_template:
msg = ("Volume and server must have the same disk template. Volume has"
" disk template '%s' while server has '%s'"
% (volume.disk_template, vm.flavor.disk_template))
raise faults.BadRequest(msg)
# Check maximum disk per instance hard limit
if vm.volumes.filter(deleted=False).count() == settings.GANETI_MAX_DISKS_PER_INSTANCE:
raise faults.BadRequest("Maximum volumes per server limit reached")
jobid = backend.attach_volume(vm, volume)
log.info("Attached volume '%s' to server '%s'. JobID: '%s'", volume.id,
volume.machine_id, jobid)
volume.backendjobid = jobid
volume.machine = vm
volume.status = "ATTACHING"
volume.save()
return jobid
@commands.server_command("DETACH_VOLUME")
def detach_volume(vm, volume):
"""Detach a volume to a server.
The volume must be in 'IN_USE' status in order to be detached. Also,
the root volume of the instance (index=0) can not be detached. This
function will send the corresponding job to Ganeti backend and update the
status of the volume to 'DETACHING'.
"""
_check_attachment(vm, volume)
if volume.status != "IN_USE":
#TODO: Maybe allow other statuses as well ?
raise faults.BadRequest("Cannot detach volume while volume is in"
" '%s' status." % volume.status)
if volume.index == 0:
raise faults.BadRequest("Cannot detach the root volume of a server")
jobid = backend.detach_volume(vm, volume)
log.info("Detached volume '%s' from server '%s'. JobID: '%s'", volume.id,
volume.machine_id, jobid)
volume.backendjobid = jobid
volume.status = "DETACHING"
volume.save()
return jobid
def _check_attachment(vm, volume):
"""Check that volume is attached to vm."""
if volume.machine_id != vm.id:
raise faults.BadRequest("Volume '%s' is not attached to server '%s'"
% volume.id, vm.id)
......@@ -31,22 +31,24 @@ import logging
from datetime import datetime
from socket import getfqdn
from functools import wraps
from django import dispatch
from django.db import transaction
from django.utils import simplejson as json
from snf_django.lib.api import faults
from django.conf import settings
from synnefo import quotas
from synnefo.api import util
from synnefo.logic import backend, ips, utils
from synnefo.logic.backend_allocator import BackendAllocator
from synnefo.db.models import (NetworkInterface, VirtualMachine,
VirtualMachineMetadata, IPAddressLog, Network,
Volume, pooled_rapi_client)
pooled_rapi_client)
from vncauthproxy.client import request_forwarding as request_vnc_forwarding
from synnefo.logic import rapi
from synnefo.volume.volumes import _create_volume
from synnefo.volume.util import get_volume
from synnefo.logic import commands
from synnefo import quotas
log = logging.getLogger(__name__)
......@@ -54,129 +56,20 @@ log = logging.getLogger(__name__)
server_created = dispatch.Signal(providing_args=["created_vm_params"])
def validate_server_action(vm, action):
if vm.deleted:
raise faults.BadRequest("Server '%s' has been deleted." % vm.id)
# Destroyin a server should always be permitted
if action == "DESTROY":
return
# Check that there is no pending action
pending_action = vm.task
if pending_action:
if pending_action == "BUILD":
raise faults.BuildInProgress("Server '%s' is being build." % vm.id)
raise faults.BadRequest("Cannot perform '%s' action while there is a"
" pending '%s'." % (action, pending_action))
# Check if action can be performed to VM's operstate
operstate = vm.operstate
if operstate == "ERROR":
raise faults.BadRequest("Cannot perform '%s' action while server is"
" in 'ERROR' state." % action)
elif operstate == "BUILD" and action != "BUILD":
raise faults.BuildInProgress("Server '%s' is being build." % vm.id)
elif (action == "START" and operstate != "STOPPED") or\
(action == "STOP" and operstate != "STARTED") or\
(action == "RESIZE" and operstate != "STOPPED") or\
(action in ["CONNECT", "DISCONNECT"]
and operstate != "STOPPED"
and not settings.GANETI_USE_HOTPLUG) or \
(action in ["ATTACH_VOLUME", "DETACH_VOLUME"]
and operstate != "STOPPED"
and not settings.GANETI_USE_HOTPLUG):
raise faults.BadRequest("Cannot perform '%s' action while server is"
" in '%s' state." % (action, operstate))
return
def server_command(action, action_fields=None):
"""Handle execution of a server action.
Helper function to validate and execute a server action, handle quota
commission and update the 'task' of the VM in the DB.
1) Check if action can be performed. If it can, then there must be no
pending task (with the exception of DESTROY).
2) Handle previous commission if unresolved:
* If it is not pending and it to accept, then accept
* If it is not pending and to reject or is pending then reject it. Since
the action can be performed only if there is no pending task, then there
can be no pending commission. The exception is DESTROY, but in this case
the commission can safely be rejected, and the dispatcher will generate