Commit c99c44cb authored by Christos Stavrakakis's avatar Christos Stavrakakis
Browse files

cyclades: Handle errors during vm creation

Commit #f2080d1 mocked an OP_INSTANCE_REMOVE when enqueuing an
OP_INSTANCE_CREATE job to Ganeti failed. However, this may result in
releasing IPs that may get reserved in Ganeti, because we can never
really know if the job has reached Ganeti or not (e.g.  timeout).  For
this reason, we should *never* delete the VM. Instead we mark the VM as
being in error state.  The user must explicitly delete the server.  This
has the downside, that we return 202 with the resulting VM being in
ERROR state. However, this is the only way to be really secure.
parent fb2eb3c3
......@@ -519,14 +519,13 @@ class ServerCreateAPITest(ComputeAPITest):
with mocked_quotaholder():
response = self.mypost('servers', 'test_user',
json.dumps(request), 'json')
self.assertEqual(response.status_code, 500)
self.assertEqual(response.status_code, 202)
mrapi().CreateInstance.assert_called_once()
vm = VirtualMachine.objects.get()
# The VM has been deleted
self.assertTrue(vm.deleted)
# and it has no nics
self.assertEqual(len(vm.nics.all()), 0)
self.assertEqual(vm.backendjobid, 0)
# The VM has not been deleted
self.assertFalse(vm.deleted)
# but is in "ERROR" operstate
self.assertEqual(vm.operstate, "ERROR")
@patch('synnefo.logic.rapi_pool.GanetiRapiClient')
......
......@@ -692,6 +692,7 @@ class NetworkInterface(models.Model):
STATES = (
("ACTIVE", "Active"),
("BUILDING", "Building"),
("ERROR", "Error"),
)
machine = models.ForeignKey(VirtualMachine, related_name='nics')
......
import logging
import datetime
from socket import getfqdn
from functools import wraps
......@@ -171,16 +170,11 @@ def create(userid, name, password, flavor, image, metadata={},
# Create the server in Ganeti.
create_server(vm, nics, flavor, image, personality, password)
except:
# If an exception is raised, then the user will never get the VM id.
# In order to delete it from DB and release it's resources, we
# mock a successful OP_INSTANCE_REMOVE job.
backend.process_op_status(vm=vm, etime=datetime.datetime.now(),
jobid=-0,
opcode="OP_INSTANCE_REMOVE",
status="success",
logmsg="Reconciled eventd: VM creation"
" failed.")
raise
log.exception("Failed create instance '%s'", vm)
vm.operstate = "ERROR"
vm.backendlogmsg = "Failed to send job to Ganeti."
vm.save()
vm.nics.all().update(state="ERROR")
return vm
......
......@@ -72,11 +72,13 @@ class ServerCreationTest(TransactionTestCase):
# error in enqueue. check the vm is deleted and resources released
mrapi().CreateInstance.side_effect = Exception("ganeti is down")
with mocked_quotaholder():
self.assertRaises(Exception, servers.create, **kwargs)
servers.create(**kwargs)
vm = models.VirtualMachine.objects.get()
self.assertTrue(vm.deleted)
self.assertEqual(len(vm.nics.all()), 0)
vm.delete()
self.assertFalse(vm.deleted)
self.assertEqual(vm.operstate, "ERROR")
self.assertEqual(len(vm.nics.all()), 1)
for nic in vm.nics.all():
self.assertEqual(nic.state, "ERROR")
# success with no nics
mrapi().CreateInstance.side_effect = None
......@@ -85,7 +87,6 @@ class ServerCreationTest(TransactionTestCase):
DEFAULT_INSTANCE_NETWORKS=[]):
with mocked_quotaholder():
vm = servers.create(**kwargs)
self.assertEqual(models.VirtualMachine.objects.count(), 1)
vm = models.VirtualMachine.objects.get(id=vm.id)
self.assertEqual(vm.nics.count(), 0)
self.assertEqual(vm.backendjobid, 42)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment