Commit cc723ba9 authored by Christos Stavrakakis's avatar Christos Stavrakakis

cyclades: Fix deadlock when creating servers

Fix deadlock between snf-dispatcher and backend allocator, by making
snf-dispatcher take an exclusive lock to the backend which hosts the vm,
before trying to get the exclusive locks to the IP pools in order to
reserve and release the instance IPs.
parent b4e359fe
......@@ -322,21 +322,6 @@ def create_server(request):
@transaction.commit_manually
def do_create_server(userid, name, password, flavor, image, metadata={},
personality=[], network=None, backend=None):
if backend is None:
# Allocate backend to host the server. Commit after allocation to
# release the locks hold by the backend allocator.
try:
backend_allocator = BackendAllocator()
backend = backend_allocator.allocate(userid, flavor)
if backend is None:
log.error("No available backend for VM with flavor %s", flavor)
raise faults.ServiceUnavailable("No available backends")
except:
transaction.rollback()
raise
else:
transaction.commit()
# Fix flavor for archipelago
disk_template, provider = util.get_flavor_provider(flavor)
if provider:
......@@ -349,6 +334,14 @@ def do_create_server(userid, name, password, flavor, image, metadata={},
flavor.disk_origin = None
try:
if backend is None:
# Allocate backend to host the server.
backend_allocator = BackendAllocator()
backend = backend_allocator.allocate(userid, flavor)
if backend is None:
log.error("No available backend for VM with flavor %s", flavor)
raise faults.ServiceUnavailable("No available backends")
if network is None:
# Allocate IP from public network
(network, address) = util.get_public_ip(backend)
......
......@@ -97,8 +97,7 @@ def process_op_status(vm, etime, jobid, opcode, status, logmsg, nics=None):
#
if status == 'success' or (status == 'error' and
vm.operstate == 'ERROR'):
release_instance_nics(vm)
vm.nics.all().delete()
_process_net_status(vm, etime, nics=[])
vm.deleted = True
vm.operstate = state_for_success
vm.backendtime = etime
......@@ -133,6 +132,11 @@ def _process_net_status(vm, etime, nics):
ganeti_nics = process_ganeti_nics(nics)
if not nics_changed(vm.nics.order_by('index'), ganeti_nics):
log.debug("NICs for VM %s have not changed", vm)
return
# Get X-Lock on backend before getting X-Lock on network IP pools, to
# guarantee that no deadlock will occur with Backend allocator.
Backend.objects.select_for_update().get(id=vm.backend_id)
release_instance_nics(vm)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment