Commit c682d231 authored by Ioannis Tsafaras's avatar Ioannis Tsafaras
Browse files

Merge pull request #284 from gouzouni625/fix-destroy-lambda-instance

Check if resource exists on ~okeanos before destroying it
parents 19bb4ecb a253f9ba
......@@ -103,9 +103,8 @@ def run_playbook(ansible_manager, playbook, only_tags=None, skip_tags=None, extr
return ansible_result
def lambda_instance_destroy(instance_uuid, auth_token,
master_id, slave_ids,
public_ip_id, private_network_id):
def lambda_instance_destroy(instance_uuid, auth_token, master_id, slave_ids, public_ip_id,
private_network_id):
"""
Destroys the specified lambda instance. The VMs of the lambda instance, along with the public
ip and the private network used are destroyed and the status of the lambda instance gets
......@@ -127,47 +126,55 @@ def lambda_instance_destroy(instance_uuid, auth_token,
# Retrieve cyclades network client
cyclades_network_client = provisioner.network_client
server_ids = list()
if master_id:
server_ids.append(master_id)
if slave_ids:
server_ids.extend(slave_ids)
# Detach the public ip
port_id = cyclades_network_client.get_floatingip_details(public_ip_id)['port_id']
if port_id:
port_status = cyclades_network_client.get_port_details(port_id)['status']
cyclades_network_client.delete_port(port_id)
try:
cyclades_network_client.wait_port_while(port_id, port_status)
except ClientError as ce:
if ce.status != 404:
raise
# Check if the public ip is attached
attached_vm = cyclades_network_client.get_floatingip_details(public_ip_id)['instance_id']
if not attached_vm:
# Destroy the public ip
cyclades_network_client.delete_floatingip(public_ip_id)
else:
if attached_vm in [master_id] + slave_ids:
raise ClientError("Destroy failed. Reason: The public IP was not detached")
if public_ip_id:
port_id = cyclades_network_client.get_floatingip_details(public_ip_id)['port_id']
if port_id:
port_status = cyclades_network_client.get_port_details(port_id)['status']
cyclades_network_client.delete_port(port_id)
try:
cyclades_network_client.wait_port_while(port_id, port_status)
except ClientError as ce:
if ce.status != 404:
raise
# Check if the public ip is attached
attached_vm = cyclades_network_client.get_floatingip_details(public_ip_id)['instance_id']
if not attached_vm:
# Destroy the public ip
cyclades_network_client.delete_floatingip(public_ip_id)
else:
if attached_vm in server_ids:
raise ClientError("Destroy failed. Reason: The public IP was not detached")
# Detach the VMs from the private network
for server_id in [master_id] + slave_ids:
ports = [port for port in
cyclades_compute_client.get_server_details(server_id)['attachments'] if
(int(port['network_id']) == private_network_id)]
if ports:
for port in ports:
port['status'] = cyclades_network_client.get_port_details(port['id'])['status']
cyclades_network_client.delete_port(port['id'])
try:
cyclades_network_client.wait_port_while(port['id'], port['status'])
except ClientError as ce:
if ce.status != 404:
raise
# Destroy the private network
cyclades_network_client.delete_network(private_network_id)
if private_network_id:
for server_id in server_ids:
ports = [port for port in
cyclades_compute_client.get_server_details(server_id)['attachments'] if
(int(port['network_id']) == private_network_id)]
if ports:
for port in ports:
port['status'] = cyclades_network_client.get_port_details(port['id'])['status']
cyclades_network_client.delete_port(port['id'])
try:
cyclades_network_client.wait_port_while(port['id'], port['status'])
except ClientError as ce:
if ce.status != 404:
raise
# Destroy the private network
cyclades_network_client.delete_network(private_network_id)
# Destroy all the VMs without caring for properly stopping the lambda services.
# Destroy master node.
if cyclades_compute_client.get_server_details(master_id)["status"] != "DELETED":
if master_id and cyclades_compute_client.get_server_details(master_id)["status"] != "DELETED":
cyclades_compute_client.delete_server(master_id)
# Destroy all slave nodes.
......@@ -176,7 +183,9 @@ def lambda_instance_destroy(instance_uuid, auth_token,
cyclades_compute_client.delete_server(slave_id)
# Wait for all VMs to be destroyed
cyclades_compute_client.wait_server_until(master_id, target_status="DELETED", max_wait=300)
if master_id:
cyclades_compute_client.wait_server_until(master_id, target_status="DELETED", max_wait=300)
for slave_id in slave_ids:
cyclades_compute_client.wait_server_until(slave_id, target_status="DELETED", max_wait=300)
......
......@@ -2,6 +2,8 @@ from mock import patch, MagicMock, call
from fokia.lambda_instance_manager import lambda_instance_destroy, create_cluster
from uuid import uuid4
test_flavors = [{
u'SNF:allow_create': True, u'SNF:disk_template': u'drbd', u'SNF:volume_type': 1, u'disk': 20,
u'id': 1, u'links': [{
......@@ -229,6 +231,187 @@ def test_destroy_cluster():
]
def test_lambda_instance_destroy():
lambda_instance_id = uuid4()
master_id = "master_id"
slave_ids = ["slave_1_id", "slave_2_id"]
public_ip_id = "public_ip_id"
private_network_id = 123456789
master_server_details = "ACTIVE"
master_port = {
'id': "master_port_id",
'network_id': private_network_id,
'status': "master_port_status"
}
slave_servers_details = ["ACTIVE", "ACTIVE"]
slave_ports = [
{
'id': "slave_1_port_id",
'network_id': private_network_id,
'status': "slave_1_port_status"
},
{
'id': "slave_2_port_id",
'network_id': private_network_id,
'status': "slave_2_port_status"
}
]
checks_list = [
{'master': True, 'slaves': True, 'public_ip': True, 'private_network': True},
{'master': False, 'slaves': True, 'public_ip': True, 'private_network': True},
{'master': True, 'slaves': False, 'public_ip': True, 'private_network': True},
{'master': True, 'slaves': True, 'public_ip': False, 'private_network': True},
{'master': True, 'slaves': True, 'public_ip': True, 'private_network': False},
{'master': False, 'slaves': False, 'public_ip': True, 'private_network': True},
{'master': False, 'slaves': True, 'public_ip': False, 'private_network': True},
{'master': False, 'slaves': True, 'public_ip': True, 'private_network': False},
{'master': True, 'slaves': False, 'public_ip': False, 'private_network': True},
{'master': True, 'slaves': False, 'public_ip': True, 'private_network': False},
{'master': True, 'slaves': True, 'public_ip': False, 'private_network': False},
{'master': False, 'slaves': False, 'public_ip': False, 'private_network': True},
{'master': False, 'slaves': False, 'public_ip': True, 'private_network': False},
{'master': True, 'slaves': False, 'public_ip': False, 'private_network': False},
{'master': False, 'slaves': False, 'public_ip': False, 'private_network': False}
]
for checks in checks_list:
with patch('fokia.lambda_instance_manager.Provisioner') as provisioner,\
patch('fokia.lambda_instance_manager.__delete_private_key') as delete_private_key:
# Setup the mocks that will be used
provisioner_obj = provisioner.return_value
cyclades_compute_client = provisioner_obj.cyclades
cyclades_network_client = provisioner_obj.network_client
def get_server_details(server_id):
if server_id == master_id:
return {'attachments': [master_port], 'status': master_server_details}
elif server_id in slave_ids:
slave_index = slave_ids.index(server_id)
return {'attachments': [slave_ports[slave_index]],
'status': slave_servers_details[slave_index]}
cyclades_compute_client.get_server_details.side_effect = get_server_details
cyclades_network_client.get_floatingip_details.return_value = {
'port_id': master_port['id'],
'instance_id': None
}
def port_details(port_id):
for port in [master_port] + slave_ports:
if port['id'] == port_id:
return {'status': port['status']}
cyclades_network_client.get_port_details.side_effect = port_details
# Make a call to lambda_instance_destroy method
lambda_instance_destroy(lambda_instance_id, 'token',
master_id if checks['master'] else None,
slave_ids if checks['slaves'] else [],
public_ip_id if checks['public_ip'] else None,
private_network_id if checks['private_network'] else None)
# Assertions
if checks['master'] and checks['slaves']:
delete_private_key.assert_called_with(lambda_instance_id, master_id, slave_ids)
elif checks['master']:
delete_private_key.assert_called_with(lambda_instance_id, master_id, [])
elif checks['slaves']:
delete_private_key.assert_called_with(lambda_instance_id, None, slave_ids)
else:
delete_private_key.assert_called_with(lambda_instance_id, None, [])
if checks['master']:
assert cyclades_compute_client.wait_server_until.call_args_list.\
count(call(master_id, target_status="DELETED", max_wait=300)) == 1
if checks['private_network']:
assert cyclades_compute_client.get_server_details.\
call_args_list.count(call(master_id)) == 2
else:
assert cyclades_compute_client.get_server_details.\
call_args_list.count(call(master_id)) == 1
else:
assert call(master_id, target_status="DELETED", max_wait=300) not in \
cyclades_compute_client.wait_server_until.call_args_list
assert cyclades_compute_client.get_server_details.\
call_args_list.count(call(master_id)) == 0
if checks['slaves']:
for slave_id in slave_ids:
assert cyclades_compute_client.wait_server_until.call_args_list.\
count(call(slave_id, target_status="DELETED", max_wait=300)) == 1
if checks['private_network']:
assert cyclades_compute_client.get_server_details.\
call_args_list.count(call(slave_id)) == 2
else:
assert cyclades_compute_client.get_server_details.\
call_args_list.count(call(slave_id)) == 1
else:
for slave_id in slave_ids:
assert call(slave_id, target_status="DELETED", max_wait=300) not in \
cyclades_compute_client.wait_server_until.call_args_list
assert cyclades_compute_client.get_server_details.\
call_args_list.count(call(slave_id)) == 0
if checks['public_ip']:
assert cyclades_network_client.get_floatingip_details.\
call_args_list.count(call(public_ip_id)) == 2
if checks['master'] and checks['private_network']:
assert cyclades_network_client.get_port_details.\
call_args_list.count(call(master_port['id'])) == 2
assert cyclades_network_client.delete_port.\
call_args_list.count(call(master_port['id'])) == 2
assert cyclades_network_client.wait_port_while.\
call_args_list.count(call(master_port['id'], master_port['status'])) == 2
else:
assert cyclades_network_client.get_port_details.\
call_args_list.count(call(master_port['id'])) == 1
assert cyclades_network_client.delete_port.\
call_args_list.count(call(master_port['id'])) == 1
assert cyclades_network_client.wait_port_while.\
call_args_list.count(call(master_port['id'], master_port['status'])) == 1
assert cyclades_network_client.delete_floatingip.call_args_list.\
count(call(public_ip_id)) == 1
else:
assert call(public_ip_id) not in \
cyclades_network_client.get_floatingip_details.call_args_list
if checks['master'] and checks['private_network']:
assert cyclades_network_client.get_port_details.\
call_args_list.count(call(master_port['id'])) == 1
assert cyclades_network_client.delete_port.\
call_args_list.count(call(master_port['id'])) == 1
assert cyclades_network_client.wait_port_while.\
call_args_list.count(call(master_port['id'], master_port['status'])) == 1
else:
assert cyclades_network_client.get_port_details.\
call_args_list.count(call(master_port['id'])) == 0
assert cyclades_network_client.delete_port.\
call_args_list.count(call(master_port['id'])) == 0
assert cyclades_network_client.wait_port_while.\
call_args_list.count(call(master_port['id'], master_port['status'])) == 0
assert call(public_ip_id) not in \
cyclades_network_client.delete_floatingip.call_args_list
if __name__ == '__main__':
test_create_cluster()
test_destroy_cluster()
test_lambda_instance_destroy()
......@@ -985,23 +985,6 @@ class LambdaInstanceViewSet(mixins.ListModelMixin, viewsets.GenericViewSet):
.messages['lambda_instance_already']
.format(state="destroyed"))
if lambda_instance_data['status'] == LambdaInstance.CLUSTER_FAILED:
if 'out of limit' in lambda_instance_data['failure_message']:
events.set_lambda_instance_status.delay(lambda_instance_data['uuid'],
LambdaInstance.DESTROYED)
status_code = status.HTTP_202_ACCEPTED
return Response({
"status": {
'code': status_code,
'short_description':
ResponseMessages.short_descriptions['lambda_instance_destroy']
}}, status=status_code)
else:
raise CustomCantDoError(CustomCantDoError.messages['cant_do'].
format(action="destroy", object="a lambda instance",
status=LambdaInstance.status_choices[
int(lambda_instance_data['status'])][1]))
# Get the id of the master node and the ids of the slave nodes.
master_id = None
public_ip_id = None
......@@ -1019,9 +1002,12 @@ class LambdaInstanceViewSet(mixins.ListModelMixin, viewsets.GenericViewSet):
auth_token = request.META.get("HTTP_AUTHORIZATION").split()[-1]
# auth_url = "https://accounts.okeanos.grnet.gr/identity/v2.0"
tasks.lambda_instance_destroy.delay(lambda_instance_data['uuid'], auth_token,
master_id, slave_ids, public_ip_id,
lambda_instance_data['private_network'][0]['id'])
private_network_id = None
if lambda_instance_data['private_network']:
private_network_id = lambda_instance_data['private_network'][0]['id']
tasks.lambda_instance_destroy.delay(lambda_instance_data['uuid'], auth_token, master_id,
slave_ids, public_ip_id, private_network_id)
# Create event to update the database.
events.set_lambda_instance_status.delay(lambda_instance_data['uuid'],
......
......@@ -812,36 +812,6 @@ class LambdaInstanceDestroy(APITestCase):
CustomAlreadyDoneError.messages['lambda_instance_already'].
format(state="destroyed"))
# Test for destroying a lambda instance when its status is CLUSTER_FAILED.
def test_cluster_failed_status(self):
# Change the status of the lambda instance to CLUSTER_FAILED.
self.lambda_instance.status = LambdaInstance.CLUSTER_FAILED
self.lambda_instance.save()
# Make a request to destroy the lambda instance.
response = self.client.delete("/api/lambda-instances/{id}/".
format(id=self.lambda_instance_uuid))
# Assert the response code.
self.assertEqual(response.status_code, status.HTTP_409_CONFLICT)
# Assert the structure of the response.
self.assertIn('errors', response.data)
self.assertEqual(len(response.data['errors']), 1)
for error in response.data['errors']:
self.assertIn('status', error)
self.assertIn('detail', error)
# Assert the contents of the response.
self.assertEqual(response.data['errors'][0]['status'], status.HTTP_409_CONFLICT)
self.assertEqual(response.data['errors'][0]['detail'],
CustomCantDoError.messages['cant_do'].
format(action="destroy", object="a lambda instance",
status="CLUSTER_FAILED"))
class TestLambdaInstanceStart(APITestCase):
"""
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment