Commit a399119b authored by Christos Stavrakakis's avatar Christos Stavrakakis

cyclades: Fix bug in snf-dispatcher logic

This commit reverts commits #07602322 and #30514662 which introduced a
bug in dispatcher's logic. These commit fixed the issue that, in a case
the Cyclades DB is unsynced with Ganeti, a resource that does not exist
in the Ganeti backend, it  will not be deleted from DB in case the
remove job fails.

This commits fixes this issue, by quering Ganeti each time a failed
OP_INSTANCE_REMOVE or OP_NETWORK_REMOVE is processed.
parent e0015520
......@@ -743,7 +743,7 @@ class IPPoolTable(PoolTable):
@contextmanager
def pooled_rapi_client(obj):
if isinstance(obj, VirtualMachine):
if isinstance(obj, (VirtualMachine, BackendNetwork)):
backend = obj.backend
else:
backend = obj
......
......@@ -41,6 +41,7 @@ from synnefo.logic import utils
from synnefo import quotas
from synnefo.api.util import release_resource
from synnefo.util.mac2eui64 import mac2eui64
from synnefo.logic.rapi import GanetiApiError
from logging import getLogger
log = getLogger(__name__)
......@@ -87,14 +88,10 @@ def process_op_status(vm, etime, jobid, opcode, status, logmsg, nics=None):
vm.operstate = 'ERROR'
vm.backendtime = etime
elif opcode == 'OP_INSTANCE_REMOVE':
# Set the deleted flag explicitly, cater for admin-initiated removals
# Special case: OP_INSTANCE_REMOVE fails for machines in ERROR,
# when no instance exists at the Ganeti backend.
# See ticket #799 for all the details.
#
if (status == 'success' or
(status == 'error' and (vm.operstate == 'ERROR' or
vm.action == 'DESTROY'))):
if status == "success" or (status == "error" and
not vm_exists_in_backend(vm)):
_process_net_status(vm, etime, nics=[])
vm.deleted = True
vm.operstate = state_for_success
......@@ -233,9 +230,9 @@ def process_network_status(back_network, etime, jobid, opcode, status, logmsg):
back_network.backendtime = etime
if opcode == 'OP_NETWORK_REMOVE':
if (status == 'success' or
(status == 'error' and (back_network.operstate == 'ERROR' or
network.action == 'DESTROY'))):
network_is_deleted = (status == "success")
if network_is_deleted or (status == "error" and not
network_exists_in_backend(back_network)):
back_network.operstate = state_for_success
back_network.deleted = True
back_network.backendtime = etime
......@@ -489,7 +486,31 @@ def get_instance_console(vm):
def get_instance_info(vm):
with pooled_rapi_client(vm) as client:
return client.GetInstanceInfo(vm.backend_vm_id)
return client.GetInstance(vm.backend_vm_id)
def vm_exists_in_backend(vm):
try:
get_instance_info(vm)
return True
except GanetiApiError as e:
if e.code == 404:
return False
raise e
def get_network_info(backend_network):
with pooled_rapi_client(backend_network) as client:
return client.GetNetwork(backend_network.network.backend_id)
def network_exists_in_backend(backend_network):
try:
get_network_info(backend_network)
return True
except GanetiApiError as e:
if e.code == 404:
return False
def create_network(network, backend, connect=True):
......
......@@ -44,6 +44,7 @@ from synnefo.api.util import allocate_resource
from synnefo.logic.callbacks import (update_db, update_network,
update_build_progress)
from snf_django.utils.testing import mocked_quotaholder
from synnefo.logic.rapi import GanetiApiError
now = datetime.now
from time import time
......@@ -145,6 +146,31 @@ class UpdateDBTest(TestCase):
# Check that nics are deleted
self.assertFalse(db_vm.nics.all())
@patch("synnefo.logic.rapi_pool.GanetiRapiClient")
def test_remove_error(self, rapi, client):
vm = mfactory.VirtualMachineFactory()
# Also create a NIC
msg = self.create_msg(operation='OP_INSTANCE_REMOVE',
status="error",
instance=vm.backend_vm_id)
rapi().GetInstance.return_value = {}
update_db(client, msg)
db_vm = VirtualMachine.objects.get(id=vm.id)
self.assertFalse(db_vm.deleted)
rapi().GetInstance.side_effect = GanetiApiError(msg="msg",
code=503)
update_db(client, msg)
db_vm = VirtualMachine.objects.get(id=vm.id)
self.assertFalse(db_vm.deleted)
rapi().GetInstance.side_effect = GanetiApiError(msg="msg",
code=404)
with mocked_quotaholder():
update_db(client, msg)
db_vm = VirtualMachine.objects.get(id=vm.id)
self.assertTrue(db_vm.deleted)
def test_create(self, client):
vm = mfactory.VirtualMachineFactory()
msg = self.create_msg(operation='OP_INSTANCE_CREATE',
......@@ -418,6 +444,26 @@ class UpdateNetworkTest(TestCase):
pool = MacPrefixPoolTable.get_pool()
self.assertTrue(pool.is_available(net.mac_prefix))
@patch("synnefo.logic.rapi_pool.GanetiRapiClient")
def test_remove_error(self, rapi, client):
mfactory.MacPrefixPoolTableFactory()
mfactory.BridgePoolTableFactory()
bn = mfactory.BackendNetworkFactory(operstate='ACTIVE')
network = bn.network
msg = self.create_msg(operation='OP_NETWORK_REMOVE',
network=network.backend_id,
status="error",
cluster=bn.backend.clustername)
rapi().GetNetwork.return_value = {}
update_network(client, msg)
bn = BackendNetwork.objects.get(id=bn.id)
self.assertNotEqual(bn.operstate, "DELETED")
rapi().GetNetwork.side_effect = GanetiApiError(msg="foo", code=404)
with mocked_quotaholder():
update_network(client, msg)
bn = BackendNetwork.objects.get(id=bn.id)
self.assertEqual(bn.operstate, "DELETED")
def test_remove_offline_backend(self, client):
"""Test network removing when a backend is offline"""
mfactory.BridgePoolTableFactory()
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment