Commit 9edd4150 authored by Christos Stavrakakis's avatar Christos Stavrakakis
Browse files

cyclades: Move NetworkReconciler to reconciliation

Move NetworkReconciler class from network-create command to
reconciliation module.
parent 15c31b56
# Copyright 2011-2012 GRNET S.A. All rights reserved.
# Copyright 2011-2013 GRNET S.A. All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
......@@ -34,19 +34,10 @@ the state of the Ganeti backend. See docstring on top of
logic/reconciliation.py for a description of reconciliation rules.
"""
import datetime
import bitarray
import logging
from optparse import make_option
from django.core.management.base import BaseCommand
from django.db import transaction
from synnefo.db.models import Backend, Network, BackendNetwork
from synnefo.db.pools import IPPool
from synnefo.logic import reconciliation, utils
from synnefo.logic import backend as backend_mod
from synnefo.logic import reconciliation
class Command(BaseCommand):
......@@ -93,254 +84,8 @@ Network reconciliation can detect and fix the following cases:
logger.setLevel(logging.WARNING)
logger.addHandler(log_handler)
reconciler = NetworkReconciler(logger=logger, fix=fix,
reconciler = reconciliation.NetworkReconciler(
logger=logger,
fix=fix,
conflicting_ips=conflicting_ips)
reconciler.reconcile_networks()
class NetworkReconciler(object):
def __init__(self, logger, fix=False, conflicting_ips=False):
self.log = logger
self.conflicting_ips = conflicting_ips
self.fix = fix
@transaction.commit_on_success
def reconcile_networks(self):
# Get models from DB
backends = Backend.objects.exclude(offline=True)
networks = Network.objects.filter(deleted=False)
# Get info from all ganeti backends
ganeti_networks = {}
ganeti_hanging_networks = {}
for b in backends:
g_nets = reconciliation.get_networks_from_ganeti(b)
ganeti_networks[b] = g_nets
g_hanging_nets = reconciliation.hanging_networks(b, g_nets)
ganeti_hanging_networks[b] = g_hanging_nets
# Perform reconciliation for each network
for network in networks:
ip_available_maps = []
ip_reserved_maps = []
for bend in backends:
bnet = get_backend_network(network, bend)
gnet = ganeti_networks[bend].get(network.id)
if not bnet:
if network.floating_ip_pool:
# Network is a floating IP pool and does not exist in
# backend. We need to create it
bnet = self.reconcile_parted_network(network, bend)
elif not gnet:
# Network does not exist either in Ganeti nor in BD.
continue
else:
# Network exists in Ganeti and not in DB.
if network.action != "DESTROY" and not network.public:
bnet = self.reconcile_parted_network(network, bend)
if not gnet:
# Network does not exist in Ganeti. If the network action
# is DESTROY, we have to mark as deleted in DB, else we
# have to create it in Ganeti.
if network.action == "DESTROY":
if bnet.operstate != "DELETED":
self.reconcile_stale_network(bnet)
else:
self.reconcile_missing_network(network, bend)
# Skip rest reconciliation!
continue
try:
hanging_groups = ganeti_hanging_networks[bend][network.id]
except KeyError:
# Network is connected to all nodegroups
hanging_groups = []
if hanging_groups:
# CASE-3: Ganeti networks not connected to all nodegroups
self.reconcile_hanging_groups(network, bend,
hanging_groups)
continue
if bnet.operstate != 'ACTIVE':
# CASE-4: Unsynced network state. At this point the network
# exists and is connected to all nodes so is must be
# active!
self.reconcile_unsynced_network(network, bend, bnet)
# Get ganeti IP Pools
available_map, reserved_map = get_network_pool(gnet)
ip_available_maps.append(available_map)
ip_reserved_maps.append(reserved_map)
if ip_available_maps or ip_reserved_maps:
# CASE-5: Unsynced IP Pools
self.reconcile_ip_pools(network, ip_available_maps,
ip_reserved_maps)
if self.conflicting_ips:
self.detect_conflicting_ips()
# CASE-6: Orphan networks
self.reconcile_orphan_networks(networks, ganeti_networks)
def reconcile_parted_network(self, network, backend):
self.log.info("D: Missing DB entry for network %s in backend %s",
network, backend)
if self.fix:
network.create_backend_network(backend)
self.log.info("F: Created DB entry")
bnet = get_backend_network(network, backend)
return bnet
def reconcile_stale_network(self, backend_network):
self.log.info("D: Stale DB entry for network %s in backend %s",
backend_network.network, backend_network.backend)
if self.fix:
etime = datetime.datetime.now()
backend_mod.process_network_status(
backend_network, etime, 0,
"OP_NETWORK_REMOVE",
"success",
"Reconciliation simulated event")
self.log.info("F: Reconciled event: OP_NETWORK_REMOVE")
def reconcile_missing_network(self, network, backend):
self.log.info("D: Missing Ganeti network %s in backend %s",
network, backend)
if self.fix:
backend_mod.create_network(network, backend)
self.log.info("F: Issued OP_NETWORK_CONNECT")
def reconcile_hanging_groups(self, network, backend, hanging_groups):
self.log.info('D: Network %s in backend %s is not connected to '
'the following groups:', network, backend)
self.log.info('- ' + '\n- '.join(hanging_groups))
if self.fix:
for group in hanging_groups:
self.log.info('F: Connecting network %s to nodegroup %s',
network, group)
backend_mod.connect_network(network, backend, depends=[],
group=group)
def reconcile_unsynced_network(self, network, backend, backend_network):
self.log.info("D: Unsynced network %s in backend %s", network, backend)
if self.fix:
self.log.info("F: Issuing OP_NETWORK_CONNECT")
etime = datetime.datetime.now()
backend_mod.process_network_status(
backend_network, etime, 0,
"OP_NETWORK_CONNECT",
"success",
"Reconciliation simulated eventd")
def reconcile_ip_pools(self, network, available_maps, reserved_maps):
available_map = reduce(lambda x, y: x & y, available_maps)
reserved_map = reduce(lambda x, y: x & y, reserved_maps)
pool = network.get_pool()
# Temporary release unused floating IPs
temp_pool = network.get_pool()
used_ips = network.nics.values_list("ipv4", flat=True)
unused_static_ips = network.floating_ips.exclude(ipv4__in=used_ips)
map(lambda ip: temp_pool.put(ip.ipv4), unused_static_ips)
if temp_pool.available != available_map:
self.log.info("D: Unsynced available map of network %s:\n"
"\tDB: %r\n\tGB: %r", network,
temp_pool.available.to01(),
available_map.to01())
if self.fix:
pool.available = available_map
# Release unsued floating IPs, as they are not included in the
# available map
map(lambda ip: pool.reserve(ip.ipv4), unused_static_ips)
pool.save()
if pool.reserved != reserved_map:
self.log.info("D: Unsynced reserved map of network %s:\n"
"\tDB: %r\n\tGB: %r", network, pool.reserved.to01(),
reserved_map.to01())
if self.fix:
pool.reserved = reserved_map
pool.save()
def detect_conflicting_ips(self, network):
"""Detect NIC's that have the same IP in the same network."""
machine_ips = network.nics.all().values_list('ipv4', 'machine')
ips = map(lambda x: x[0], machine_ips)
distinct_ips = set(ips)
if len(distinct_ips) < len(ips):
for i in distinct_ips:
ips.remove(i)
for i in ips:
machines = [utils.id_to_instance_name(x[1])
for x in machine_ips if x[0] == i]
self.log.info('D: Conflicting IP:%s Machines: %s',
i, ', '.join(machines))
def reconcile_orphan_networks(self, db_networks, ganeti_networks):
# Detect Orphan Networks in Ganeti
db_network_ids = set([net.id for net in db_networks])
for back_end, ganeti_networks in ganeti_networks.items():
ganeti_network_ids = set(ganeti_networks.keys())
orphans = ganeti_network_ids - db_network_ids
if len(orphans) > 0:
self.log.info('D: Orphan Networks in backend %s:',
back_end.clustername)
self.log.info('- ' + '\n- '.join([str(o) for o in orphans]))
if self.fix:
for net_id in orphans:
self.log.info('Disconnecting and deleting network %d',
net_id)
try:
network = Network.objects.get(id=net_id)
backend_mod.delete_network(network,
backend=back_end)
except Network.DoesNotExist:
self.log.info("Not entry for network %s in DB !!",
net_id)
def get_backend_network(network, backend):
try:
return BackendNetwork.objects.get(network=network, backend=backend)
except BackendNetwork.DoesNotExist:
return None
def get_network_pool(gnet):
"""Return available and reserved IP maps.
Extract the available and reserved IP map from the info return from Ganeti
for a network.
"""
converter = IPPool(Foo(gnet['network']))
a_map = bitarray_from_map(gnet['map'])
a_map.invert()
reserved = gnet['external_reservations']
r_map = a_map.copy()
r_map.setall(True)
for address in reserved.split(','):
index = converter.value_to_index(address.strip())
a_map[index] = True
r_map[index] = False
return a_map, r_map
def bitarray_from_map(bitmap):
return bitarray.bitarray(bitmap.replace("X", "1").replace(".", "0"))
class Foo():
def __init__(self, subnet):
self.available_map = ''
self.reserved_map = ''
self.size = 0
self.network = Foo.Foo1(subnet)
class Foo1():
def __init__(self, subnet):
self.subnet = subnet
self.gateway = None
......@@ -67,11 +67,14 @@ setup_environ(settings)
import logging
import itertools
import bitarray
from datetime import datetime, timedelta
from django.db import transaction
from synnefo.db.models import (Backend, VirtualMachine, Flavor,
pooled_rapi_client)
pooled_rapi_client, Network,
BackendNetwork)
from synnefo.db.pools import IPPool
from synnefo.logic import utils, backend as backend_mod
from synnefo.logic.rapi import GanetiApiError
......@@ -435,3 +438,254 @@ def nics_from_instance(i):
def disks_from_instance(i):
return dict([(index, {"size": size})
for index, size in enumerate(i["disk.sizes"])])
class NetworkReconciler(object):
def __init__(self, logger, fix=False, conflicting_ips=False):
self.log = logger
self.conflicting_ips = conflicting_ips
self.fix = fix
@transaction.commit_on_success
def reconcile_networks(self):
# Get models from DB
backends = Backend.objects.exclude(offline=True)
networks = Network.objects.filter(deleted=False)
self.event_time = datetime.now()
# Get info from all ganeti backends
ganeti_networks = {}
ganeti_hanging_networks = {}
for b in backends:
g_nets = get_networks_from_ganeti(b)
ganeti_networks[b] = g_nets
g_hanging_nets = hanging_networks(b, g_nets)
ganeti_hanging_networks[b] = g_hanging_nets
# Perform reconciliation for each network
for network in networks:
ip_available_maps = []
ip_reserved_maps = []
for bend in backends:
bnet = get_backend_network(network, bend)
gnet = ganeti_networks[bend].get(network.id)
if not bnet:
if network.floating_ip_pool:
# Network is a floating IP pool and does not exist in
# backend. We need to create it
bnet = self.reconcile_parted_network(network, bend)
elif not gnet:
# Network does not exist either in Ganeti nor in BD.
continue
else:
# Network exists in Ganeti and not in DB.
if network.action != "DESTROY" and not network.public:
bnet = self.reconcile_parted_network(network, bend)
else:
continue
if not gnet:
# Network does not exist in Ganeti. If the network action
# is DESTROY, we have to mark as deleted in DB, else we
# have to create it in Ganeti.
if network.action == "DESTROY":
if bnet.operstate != "DELETED":
self.reconcile_stale_network(bnet)
else:
self.reconcile_missing_network(network, bend)
# Skip rest reconciliation!
continue
try:
hanging_groups = ganeti_hanging_networks[bend][network.id]
except KeyError:
# Network is connected to all nodegroups
hanging_groups = []
if hanging_groups:
# CASE-3: Ganeti networks not connected to all nodegroups
self.reconcile_hanging_groups(network, bend,
hanging_groups)
continue
if bnet.operstate != 'ACTIVE':
# CASE-4: Unsynced network state. At this point the network
# exists and is connected to all nodes so is must be
# active!
self.reconcile_unsynced_network(network, bend, bnet)
# Get ganeti IP Pools
available_map, reserved_map = get_network_pool(gnet)
ip_available_maps.append(available_map)
ip_reserved_maps.append(reserved_map)
if ip_available_maps or ip_reserved_maps:
# CASE-5: Unsynced IP Pools
self.reconcile_ip_pools(network, ip_available_maps,
ip_reserved_maps)
if self.conflicting_ips:
self.detect_conflicting_ips()
# CASE-6: Orphan networks
self.reconcile_orphan_networks(networks, ganeti_networks)
def reconcile_parted_network(self, network, backend):
self.log.info("D: Missing DB entry for network %s in backend %s",
network, backend)
if self.fix:
network.create_backend_network(backend)
self.log.info("F: Created DB entry")
bnet = get_backend_network(network, backend)
return bnet
def reconcile_stale_network(self, backend_network):
self.log.info("D: Stale DB entry for network %s in backend %s",
backend_network.network, backend_network.backend)
if self.fix:
backend_mod.process_network_status(
backend_network, self.event_time, 0,
"OP_NETWORK_REMOVE",
"success",
"Reconciliation simulated event")
self.log.info("F: Reconciled event: OP_NETWORK_REMOVE")
def reconcile_missing_network(self, network, backend):
self.log.info("D: Missing Ganeti network %s in backend %s",
network, backend)
if self.fix:
backend_mod.create_network(network, backend)
self.log.info("F: Issued OP_NETWORK_CONNECT")
def reconcile_hanging_groups(self, network, backend, hanging_groups):
self.log.info('D: Network %s in backend %s is not connected to '
'the following groups:', network, backend)
self.log.info('- ' + '\n- '.join(hanging_groups))
if self.fix:
for group in hanging_groups:
self.log.info('F: Connecting network %s to nodegroup %s',
network, group)
backend_mod.connect_network(network, backend, depends=[],
group=group)
def reconcile_unsynced_network(self, network, backend, backend_network):
self.log.info("D: Unsynced network %s in backend %s", network, backend)
if self.fix:
self.log.info("F: Issuing OP_NETWORK_CONNECT")
backend_mod.process_network_status(
backend_network, self.event_time, 0,
"OP_NETWORK_CONNECT",
"success",
"Reconciliation simulated eventd")
def reconcile_ip_pools(self, network, available_maps, reserved_maps):
available_map = reduce(lambda x, y: x & y, available_maps)
reserved_map = reduce(lambda x, y: x & y, reserved_maps)
pool = network.get_pool()
# Temporary release unused floating IPs
temp_pool = network.get_pool()
used_ips = network.nics.values_list("ipv4", flat=True)
unused_static_ips = network.floating_ips.exclude(ipv4__in=used_ips)
map(lambda ip: temp_pool.put(ip.ipv4), unused_static_ips)
if temp_pool.available != available_map:
self.log.info("D: Unsynced available map of network %s:\n"
"\tDB: %r\n\tGB: %r", network,
temp_pool.available.to01(),
available_map.to01())
if self.fix:
pool.available = available_map
# Release unsued floating IPs, as they are not included in the
# available map
map(lambda ip: pool.reserve(ip.ipv4), unused_static_ips)
pool.save()
if pool.reserved != reserved_map:
self.log.info("D: Unsynced reserved map of network %s:\n"
"\tDB: %r\n\tGB: %r", network, pool.reserved.to01(),
reserved_map.to01())
if self.fix:
pool.reserved = reserved_map
pool.save()
def detect_conflicting_ips(self, network):
"""Detect NIC's that have the same IP in the same network."""
machine_ips = network.nics.all().values_list('ipv4', 'machine')
ips = map(lambda x: x[0], machine_ips)
distinct_ips = set(ips)
if len(distinct_ips) < len(ips):
for i in distinct_ips:
ips.remove(i)
for i in ips:
machines = [utils.id_to_instance_name(x[1])
for x in machine_ips if x[0] == i]
self.log.info('D: Conflicting IP:%s Machines: %s',
i, ', '.join(machines))
def reconcile_orphan_networks(self, db_networks, ganeti_networks):
# Detect Orphan Networks in Ganeti
db_network_ids = set([net.id for net in db_networks])
for back_end, ganeti_networks in ganeti_networks.items():
ganeti_network_ids = set(ganeti_networks.keys())
orphans = ganeti_network_ids - db_network_ids
if len(orphans) > 0:
self.log.info('D: Orphan Networks in backend %s:',
back_end.clustername)
self.log.info('- ' + '\n- '.join([str(o) for o in orphans]))
if self.fix:
for net_id in orphans:
self.log.info('Disconnecting and deleting network %d',
net_id)
try:
network = Network.objects.get(id=net_id)
backend_mod.delete_network(network,
backend=back_end)
except Network.DoesNotExist:
self.log.info("Not entry for network %s in DB !!",
net_id)
def get_backend_network(network, backend):
try:
return BackendNetwork.objects.get(network=network, backend=backend)
except BackendNetwork.DoesNotExist:
return None
def get_network_pool(gnet):
"""Return available and reserved IP maps.
Extract the available and reserved IP map from the info return from Ganeti
for a network.
"""
converter = IPPool(Foo(gnet['network']))
a_map = bitarray_from_map(gnet['map'])
a_map.invert()
reserved = gnet['external_reservations']
r_map = a_map.copy()
r_map.setall(True)
if reserved:
for address in reserved.split(','):
index = converter.value_to_index(address.strip())
a_map[index] = True
r_map[index] = False
return a_map, r_map
def bitarray_from_map(bitmap):
return bitarray.bitarray(bitmap.replace("X", "1").replace(".", "0"))
class Foo():
def __init__(self, subnet):
self.available_map = ''
self.reserved_map = ''
self.size = 0
self.network = Foo.Foo1(subnet)
class Foo1():
def __init__(self, subnet):
self.subnet = subnet
self.gateway = None
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment