Skip to content

GitLab

Commit 9edd4150 authored by Christos Stavrakakis's avatar Christos Stavrakakis
Browse files

cyclades: Move NetworkReconciler to reconciliation 

Move NetworkReconciler class from network-create command to
reconciliation module.
parent 15c31b56
Hide whitespace changes
Inline Side-by-side
snf-cyclades-app/synnefo/logic/management/commands/reconcile-networks.py
View file @ 9edd4150
# Copyright 2011-2012 GRNET S.A. All rights reserved. # Copyright 2011-2013 GRNET S.A. All rights reserved.
# #
# Redistribution and use in source and binary forms, with or without # Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions # modification, are permitted provided that the following conditions
...@@ -34,19 +34,10 @@ the state of the Ganeti backend. See docstring on top of ...@@ -34,19 +34,10 @@ the state of the Ganeti backend. See docstring on top of
logic/reconciliation.py for a description of reconciliation rules. logic/reconciliation.py for a description of reconciliation rules.
""" """
import datetime
import bitarray
import logging import logging
from optparse import make_option from optparse import make_option
from django.core.management.base import BaseCommand from django.core.management.base import BaseCommand
from django.db import transaction from synnefo.logic import reconciliation
from synnefo.db.models import Backend, Network, BackendNetwork
from synnefo.db.pools import IPPool
from synnefo.logic import reconciliation, utils
from synnefo.logic import backend as backend_mod
class Command(BaseCommand): class Command(BaseCommand):
...@@ -93,254 +84,8 @@ Network reconciliation can detect and fix the following cases: ...@@ -93,254 +84,8 @@ Network reconciliation can detect and fix the following cases:
logger.setLevel(logging.WARNING) logger.setLevel(logging.WARNING)
logger.addHandler(log_handler) logger.addHandler(log_handler)
reconciler = NetworkReconciler(logger=logger, fix=fix, reconciler = reconciliation.NetworkReconciler(
conflicting_ips=conflicting_ips) logger=logger,
fix=fix,
conflicting_ips=conflicting_ips)
reconciler.reconcile_networks() reconciler.reconcile_networks()
class NetworkReconciler(object):
def __init__(self, logger, fix=False, conflicting_ips=False):
self.log = logger
self.conflicting_ips = conflicting_ips
self.fix = fix
@transaction.commit_on_success
def reconcile_networks(self):
# Get models from DB
backends = Backend.objects.exclude(offline=True)
networks = Network.objects.filter(deleted=False)
# Get info from all ganeti backends
ganeti_networks = {}
ganeti_hanging_networks = {}
for b in backends:
g_nets = reconciliation.get_networks_from_ganeti(b)
ganeti_networks[b] = g_nets
g_hanging_nets = reconciliation.hanging_networks(b, g_nets)
ganeti_hanging_networks[b] = g_hanging_nets
# Perform reconciliation for each network
for network in networks:
ip_available_maps = []
ip_reserved_maps = []
for bend in backends:
bnet = get_backend_network(network, bend)
gnet = ganeti_networks[bend].get(network.id)
if not bnet:
if network.floating_ip_pool:
# Network is a floating IP pool and does not exist in
# backend. We need to create it
bnet = self.reconcile_parted_network(network, bend)
elif not gnet:
# Network does not exist either in Ganeti nor in BD.
continue
else:
# Network exists in Ganeti and not in DB.
if network.action != "DESTROY" and not network.public:
bnet = self.reconcile_parted_network(network, bend)
if not gnet:
# Network does not exist in Ganeti. If the network action
# is DESTROY, we have to mark as deleted in DB, else we
# have to create it in Ganeti.
if network.action == "DESTROY":
if bnet.operstate != "DELETED":
self.reconcile_stale_network(bnet)
else:
self.reconcile_missing_network(network, bend)
# Skip rest reconciliation!
continue
try:
hanging_groups = ganeti_hanging_networks[bend][network.id]
except KeyError:
# Network is connected to all nodegroups
hanging_groups = []
if hanging_groups:
# CASE-3: Ganeti networks not connected to all nodegroups
self.reconcile_hanging_groups(network, bend,
hanging_groups)
continue
if bnet.operstate != 'ACTIVE':
# CASE-4: Unsynced network state. At this point the network
# exists and is connected to all nodes so is must be
# active!
self.reconcile_unsynced_network(network, bend, bnet)
# Get ganeti IP Pools
available_map, reserved_map = get_network_pool(gnet)
ip_available_maps.append(available_map)
ip_reserved_maps.append(reserved_map)
if ip_available_maps or ip_reserved_maps:
# CASE-5: Unsynced IP Pools
self.reconcile_ip_pools(network, ip_available_maps,
ip_reserved_maps)
if self.conflicting_ips:
self.detect_conflicting_ips()
# CASE-6: Orphan networks
self.reconcile_orphan_networks(networks, ganeti_networks)
def reconcile_parted_network(self, network, backend):
self.log.info("D: Missing DB entry for network %s in backend %s",
network, backend)
if self.fix:
network.create_backend_network(backend)
self.log.info("F: Created DB entry")
bnet = get_backend_network(network, backend)
return bnet
def reconcile_stale_network(self, backend_network):
self.log.info("D: Stale DB entry for network %s in backend %s",
backend_network.network, backend_network.backend)
if self.fix:
etime = datetime.datetime.now()
backend_mod.process_network_status(
backend_network, etime, 0,
"OP_NETWORK_REMOVE",
"success",
"Reconciliation simulated event")
self.log.info("F: Reconciled event: OP_NETWORK_REMOVE")
def reconcile_missing_network(self, network, backend):
self.log.info("D: Missing Ganeti network %s in backend %s",
network, backend)
if self.fix:
backend_mod.create_network(network, backend)
self.log.info("F: Issued OP_NETWORK_CONNECT")
def reconcile_hanging_groups(self, network, backend, hanging_groups):
self.log.info('D: Network %s in backend %s is not connected to '
'the following groups:', network, backend)
self.log.info('- ' + '\n- '.join(hanging_groups))
if self.fix:
for group in hanging_groups:
self.log.info('F: Connecting network %s to nodegroup %s',
network, group)
backend_mod.connect_network(network, backend, depends=[],
group=group)
def reconcile_unsynced_network(self, network, backend, backend_network):
self.log.info("D: Unsynced network %s in backend %s", network, backend)
if self.fix:
self.log.info("F: Issuing OP_NETWORK_CONNECT")
etime = datetime.datetime.now()
backend_mod.process_network_status(
backend_network, etime, 0,
"OP_NETWORK_CONNECT",
"success",
"Reconciliation simulated eventd")
def reconcile_ip_pools(self, network, available_maps, reserved_maps):
available_map = reduce(lambda x, y: x & y, available_maps)
reserved_map = reduce(lambda x, y: x & y, reserved_maps)
pool = network.get_pool()
# Temporary release unused floating IPs
temp_pool = network.get_pool()
used_ips = network.nics.values_list("ipv4", flat=True)
unused_static_ips = network.floating_ips.exclude(ipv4__in=used_ips)
map(lambda ip: temp_pool.put(ip.ipv4), unused_static_ips)
if temp_pool.available != available_map:
self.log.info("D: Unsynced available map of network %s:\n"
"\tDB: %r\n\tGB: %r", network,
temp_pool.available.to01(),
available_map.to01())
if self.fix:
pool.available = available_map
# Release unsued floating IPs, as they are not included in the
# available map
map(lambda ip: pool.reserve(ip.ipv4), unused_static_ips)
pool.save()
if pool.reserved != reserved_map:
self.log.info("D: Unsynced reserved map of network %s:\n"
"\tDB: %r\n\tGB: %r", network, pool.reserved.to01(),
reserved_map.to01())
if self.fix:
pool.reserved = reserved_map
pool.save()
def detect_conflicting_ips(self, network):
"""Detect NIC's that have the same IP in the same network."""
machine_ips = network.nics.all().values_list('ipv4', 'machine')
ips = map(lambda x: x[0], machine_ips)
distinct_ips = set(ips)
if len(distinct_ips) < len(ips):
for i in distinct_ips:
ips.remove(i)
for i in ips:
machines = [utils.id_to_instance_name(x[1])
for x in machine_ips if x[0] == i]
self.log.info('D: Conflicting IP:%s Machines: %s',
i, ', '.join(machines))
def reconcile_orphan_networks(self, db_networks, ganeti_networks):
# Detect Orphan Networks in Ganeti
db_network_ids = set([net.id for net in db_networks])
for back_end, ganeti_networks in ganeti_networks.items():
ganeti_network_ids = set(ganeti_networks.keys())
orphans = ganeti_network_ids - db_network_ids
if len(orphans) > 0:
self.log.info('D: Orphan Networks in backend %s:',
back_end.clustername)
self.log.info('- ' + '\n- '.join([str(o) for o in orphans]))
if self.fix:
for net_id in orphans:
self.log.info('Disconnecting and deleting network %d',
net_id)
try:
network = Network.objects.get(id=net_id)
backend_mod.delete_network(network,
backend=back_end)
except Network.DoesNotExist:
self.log.info("Not entry for network %s in DB !!",
net_id)
def get_backend_network(network, backend):
try:
return BackendNetwork.objects.get(network=network, backend=backend)
except BackendNetwork.DoesNotExist:
return None
def get_network_pool(gnet):
"""Return available and reserved IP maps.
Extract the available and reserved IP map from the info return from Ganeti
for a network.
"""
converter = IPPool(Foo(gnet['network']))
a_map = bitarray_from_map(gnet['map'])
a_map.invert()
reserved = gnet['external_reservations']
r_map = a_map.copy()
r_map.setall(True)
for address in reserved.split(','):
index = converter.value_to_index(address.strip())
a_map[index] = True
r_map[index] = False
return a_map, r_map
def bitarray_from_map(bitmap):
return bitarray.bitarray(bitmap.replace("X", "1").replace(".", "0"))
class Foo():
def __init__(self, subnet):
self.available_map = ''
self.reserved_map = ''
self.size = 0
self.network = Foo.Foo1(subnet)
class Foo1():
def __init__(self, subnet):
self.subnet = subnet
self.gateway = None
snf-cyclades-app/synnefo/logic/reconciliation.py
View file @ 9edd4150
...@@ -67,11 +67,14 @@ setup_environ(settings) ...@@ -67,11 +67,14 @@ setup_environ(settings)
import logging import logging
import itertools import itertools
import bitarray
from datetime import datetime, timedelta from datetime import datetime, timedelta
from django.db import transaction from django.db import transaction
from synnefo.db.models import (Backend, VirtualMachine, Flavor, from synnefo.db.models import (Backend, VirtualMachine, Flavor,
pooled_rapi_client) pooled_rapi_client, Network,
BackendNetwork)
from synnefo.db.pools import IPPool
from synnefo.logic import utils, backend as backend_mod from synnefo.logic import utils, backend as backend_mod
from synnefo.logic.rapi import GanetiApiError from synnefo.logic.rapi import GanetiApiError
...@@ -435,3 +438,254 @@ def nics_from_instance(i): ...@@ -435,3 +438,254 @@ def nics_from_instance(i):
def disks_from_instance(i): def disks_from_instance(i):
return dict([(index, {"size": size}) return dict([(index, {"size": size})
for index, size in enumerate(i["disk.sizes"])]) for index, size in enumerate(i["disk.sizes"])])
class NetworkReconciler(object):
def __init__(self, logger, fix=False, conflicting_ips=False):
self.log = logger
self.conflicting_ips = conflicting_ips
self.fix = fix
@transaction.commit_on_success
def reconcile_networks(self):
# Get models from DB
backends = Backend.objects.exclude(offline=True)
networks = Network.objects.filter(deleted=False)
self.event_time = datetime.now()
# Get info from all ganeti backends
ganeti_networks = {}
ganeti_hanging_networks = {}
for b in backends:
g_nets = get_networks_from_ganeti(b)
ganeti_networks[b] = g_nets
g_hanging_nets = hanging_networks(b, g_nets)
ganeti_hanging_networks[b] = g_hanging_nets
# Perform reconciliation for each network
for network in networks:
ip_available_maps = []
ip_reserved_maps = []
for bend in backends:
bnet = get_backend_network(network, bend)
gnet = ganeti_networks[bend].get(network.id)
if not bnet:
if network.floating_ip_pool:
# Network is a floating IP pool and does not exist in
# backend. We need to create it
bnet = self.reconcile_parted_network(network, bend)
elif not gnet:
# Network does not exist either in Ganeti nor in BD.
continue
else:
# Network exists in Ganeti and not in DB.
if network.action != "DESTROY" and not network.public:
bnet = self.reconcile_parted_network(network, bend)
else:
continue
if not gnet:
# Network does not exist in Ganeti. If the network action
# is DESTROY, we have to mark as deleted in DB, else we
# have to create it in Ganeti.
if network.action == "DESTROY":
if bnet.operstate != "DELETED":
self.reconcile_stale_network(bnet)
else:
self.reconcile_missing_network(network, bend)
# Skip rest reconciliation!
continue
try:
hanging_groups = ganeti_hanging_networks[bend][network.id]
except KeyError:
# Network is connected to all nodegroups
hanging_groups = []
if hanging_groups:
# CASE-3: Ganeti networks not connected to all nodegroups
self.reconcile_hanging_groups(network, bend,
hanging_groups)
continue
if bnet.operstate != 'ACTIVE':
# CASE-4: Unsynced network state. At this point the network
# exists and is connected to all nodes so is must be
# active!
self.reconcile_unsynced_network(network, bend, bnet)
# Get ganeti IP Pools
available_map, reserved_map = get_network_pool(gnet)
ip_available_maps.append(available_map)
ip_reserved_maps.append(reserved_map)
if ip_available_maps or ip_reserved_maps:
# CASE-5: Unsynced IP Pools
self.reconcile_ip_pools(network, ip_available_maps,
ip_reserved_maps)
if self.conflicting_ips:
self.detect_conflicting_ips()
# CASE-6: Orphan networks
self.reconcile_orphan_networks(networks, ganeti_networks)
def reconcile_parted_network(self, network, backend):
self.log.info("D: Missing DB entry for network %s in backend %s",
network, backend)
if self.fix:
network.create_backend_network(backend)
self.log.info("F: Created DB entry")
bnet = get_backend_network(network, backend)
return bnet
def reconcile_stale_network(self, backend_network):
self.log.info("D: Stale DB entry for network %s in backend %s",
backend_network.network, backend_network.backend)
if self.fix:
backend_mod.process_network_status(
backend_network, self.event_time, 0,
"OP_NETWORK_REMOVE",
"success",
"Reconciliation simulated event")
self.log.info("F: Reconciled event: OP_NETWORK_REMOVE")
def reconcile_missing_network(self, network, backend):
self.log.info("D: Missing Ganeti network %s in backend %s",
network, backend)
if self.fix:
backend_mod.create_network(network, backend)
self.log.info("F: Issued OP_NETWORK_CONNECT")
def reconcile_hanging_groups(self, network, backend, hanging_groups):
self.log.info('D: Network %s in backend %s is not connected to '
'the following groups:', network, backend)
self.log.info('- ' + '\n- '.join(hanging_groups))
if self.fix:
for group in hanging_groups:
self.log.info('F: Connecting network %s to nodegroup %s',
network, group)
backend_mod.connect_network(network, backend, depends=[],
group=group)
def reconcile_unsynced_network(self, network, backend, backend_network):
self.log.info("D: Unsynced network %s in backend %s", network, backend)
if self.fix:
self.log.info("F: Issuing OP_NETWORK_CONNECT")
backend_mod.process_network_status(
backend_network, self.event_time, 0,
"OP_NETWORK_CONNECT",
"success",
"Reconciliation simulated eventd")
def reconcile_ip_pools(self, network, available_maps, reserved_maps):
available_map = reduce(lambda x, y: x & y, available_maps)
reserved_map = reduce(lambda x, y: x & y, reserved_maps)
pool = network.get_pool()
# Temporary release unused floating IPs
temp_pool = network.get_pool()
used_ips = network.nics.values_list("ipv4", flat=True)
unused_static_ips = network.floating_ips.exclude(ipv4__in=used_ips)
map(lambda ip: temp_pool.put(ip.ipv4), unused_static_ips)
if temp_pool.available != available_map:
self.log.info("D: Unsynced available map of network %s:\n"
"\tDB: %r\n\tGB: %r", network,
temp_pool.available.to01(),
available_map.to01())
if self.fix:
pool.available = available_map
# Release unsued floating IPs, as they are not included in the
# available map
map(lambda ip: pool.reserve(ip.ipv4), unused_static_ips)
pool.save()
if pool.reserved != reserved_map:
self.log.info("D: Unsynced reserved map of network %s:\n"
"\tDB: %r\n\tGB: %r", network, pool.reserved.to01(),
reserved_map.to01())
if self.fix:
pool.reserved = reserved_map
pool.save()
def detect_conflicting_ips(self, network):
"""Detect NIC's that have the same IP in the same network."""
machine_ips = network.nics.all().values_list('ipv4', 'machine')
ips = map(lambda x: x[0], machine_ips)
distinct_ips = set(ips)
if len(distinct_ips) < len(ips):
for i in distinct_ips:
ips.remove(i)
for i in ips:
machines = [utils.id_to_instance_name(x[1])
for x in machine_ips if x[0] == i]
self.log.info('D: Conflicting IP:%s Machines: %s',
i, ', '.join(machines))
def reconcile_orphan_networks(self, db_networks, ganeti_networks):
# Detect Orphan Networks in Ganeti
db_network_ids = set([net.id for net in db_networks])
for back_end, ganeti_networks in ganeti_networks.items():
ganeti_network_ids = set(ganeti_networks.keys())
orphans = ganeti_network_ids - db_network_ids
if len(orphans) > 0:
self.log.info('D: Orphan Networks in backend %s:',
back_end.clustername)
self.log.info('- ' + '\n- '.join([str(o) for o in orphans]))
if self.fix:
for net_id in orphans:
self.log.info('Disconnecting and deleting network %d',
net_id)
try:
network = Network.objects.get(id=net_id)
backend_mod.delete_network(network,
backend=back_end)
except Network.DoesNotExist:
self.log.info("Not entry for network %s in DB !!",
net_id)
def get_backend_network(network, backend):
try:
return BackendNetwork.objects.get(network=network, backend=backend)
except BackendNetwork.DoesNotExist:
return None
def get_network_pool(gnet):
"""Return available and reserved IP maps.
Extract the available and reserved IP map from the info return from Ganeti
for a network.
"""
converter = IPPool(Foo(gnet['network']))
a_map = bitarray_from_map(gnet['map'])
a_map.invert()
reserved = gnet['external_reservations']
r_map = a_map.copy()
r_map.setall(True)
if reserved:
for address in reserved.split(','):
index = converter.value_to_index(address.strip())
a_map[index] = True
r_map[index] = False
return a_map, r_map
def bitarray_from_map(bitmap):
return bitarray.bitarray(bitmap.replace("X", "1").replace(".", "0"))
class Foo():
def __init__(self, subnet):
self.available_map = ''
self.reserved_map = ''
self.size = 0
self.network = Foo.Foo1(subnet)
class Foo1():
def __init__(self, subnet):
self.subnet = subnet
self.gateway = None
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment