Commit ec8d1e3c authored by Christos Stavrakakis's avatar Christos Stavrakakis
Browse files

Merge branch 'feature-ipv6-networks' into develop

parents 798393b8 f58416a5
......@@ -34,14 +34,12 @@
from optparse import make_option
from django.core.management.base import BaseCommand, CommandError
from synnefo.management.common import validate_network_info, get_backend
from synnefo.webproject.management.utils import pprint_table, parse_bool
from synnefo.management.common import get_backend, convert_api_faults
from synnefo.webproject.management.utils import parse_bool
from synnefo import quotas
from synnefo.db.models import Network, Backend
from synnefo.db.utils import validate_mac, InvalidMacAddress
from synnefo.logic import networks
from synnefo.logic.backend import create_network
from synnefo.api.util import values_from_flavor
NETWORK_FLAVORS = Network.FLAVORS.keys()
......@@ -53,12 +51,6 @@ class Command(BaseCommand):
help = "Create a new network"
option_list = BaseCommand.option_list + (
make_option(
"-n",
"--dry-run",
dest="dry_run",
default=False,
action="store_true"),
make_option(
'--name',
dest='name',
......@@ -91,8 +83,9 @@ class Command(BaseCommand):
make_option(
'--dhcp',
dest='dhcp',
action='store_true',
default=False,
default="False",
choices=["True", "False"],
metavar="True|False",
help='Automatically assign IPs'),
make_option(
'--public',
......@@ -142,13 +135,16 @@ class Command(BaseCommand):
' public networks'),
)
@convert_api_faults
def handle(self, *args, **options):
if args:
raise CommandError("Command doesn't accept any arguments")
dry_run = options["dry_run"]
name = options['name']
subnet = options['subnet']
gateway = options['gateway']
subnet6 = options['subnet6']
gateway6 = options['gateway6']
backend_id = options['backend_id']
public = options['public']
flavor = options['flavor']
......@@ -158,66 +154,38 @@ class Command(BaseCommand):
tags = options['tags']
userid = options["owner"]
floating_ip_pool = parse_bool(options["floating_ip_pool"])
dhcp = parse_bool(options["dhcp"])
if not name:
raise CommandError("Name is required")
if not subnet:
raise CommandError("Subnet is required")
raise CommandError("name is required")
if not flavor:
raise CommandError("Flavor is required")
raise CommandError("flavor is required")
if (subnet is None) and (subnet6 is None):
raise CommandError("subnet or subnet6 is required")
if subnet is None and gateway is not None:
raise CommandError("Can not use gateway without subnet")
if subnet6 is None and gateway6 is not None:
raise CommandError("Can not use gateway6 without subnet6")
if public and not (backend_id or floating_ip_pool):
raise CommandError("backend-id is required")
if not userid and not public:
raise CommandError("'owner' is required for private networks")
if mac_prefix and flavor == "MAC_FILTERED":
raise CommandError("Can not override MAC_FILTERED mac-prefix")
if link and flavor == "PHYSICAL_VLAN":
raise CommandError("Can not override PHYSICAL_VLAN link")
if backend_id:
if backend_id is not None:
try:
backend_id = int(backend_id)
except ValueError:
raise CommandError("Invalid backend-id: %s", backend_id)
backend = get_backend(backend_id)
fmode, flink, fmac_prefix, ftags = values_from_flavor(flavor)
mode = mode or fmode
link = link or flink
mac_prefix = mac_prefix or fmac_prefix
tags = tags or ftags
try:
validate_mac(mac_prefix + "0:00:00:00")
except InvalidMacAddress:
raise CommandError("Invalid MAC prefix '%s'" % mac_prefix)
subnet, gateway, subnet6, gateway6 = validate_network_info(options)
if not link or not mode:
raise CommandError("Can not create network."
" No connectivity link or mode")
netinfo = {
"name": name,
"userid": options["owner"],
"subnet": subnet,
"gateway": gateway,
"gateway6": gateway6,
"subnet6": subnet6,
"dhcp": options["dhcp"],
"flavor": flavor,
"public": public,
"mode": mode,
"link": link,
"mac_prefix": mac_prefix,
"tags": tags,
"floating_ip_pool": floating_ip_pool,
"state": "ACTIVE"}
if dry_run:
self.stdout.write("Creating network:\n")
pprint_table(self.stdout, tuple(netinfo.items()))
return
network = Network.objects.create(**netinfo)
if userid:
quotas.issue_and_accept_commission(network)
network = networks.create(user_id=userid, name=name, flavor=flavor,
subnet=subnet, gateway=gateway,
subnet6=subnet6, gateway6=gateway6,
dhcp=dhcp, public=public, mode=mode,
link=link, mac_prefix=mac_prefix, tags=tags,
floating_ip_pool=floating_ip_pool)
# Create network in Backend if needed
if floating_ip_pool:
......
......@@ -70,6 +70,7 @@ class Command(BaseCommand):
help="Password for the new server")
)
@common.convert_api_faults
def handle(self, *args, **options):
if args:
raise CommandError("Command doesn't accept any arguments")
......
......@@ -108,8 +108,8 @@ def nic_to_dict(nic):
d = {'id': util.construct_nic_id(nic),
'network_id': str(nic.network.id),
'mac_address': nic.mac,
'ipv4': nic.ipv4 if nic.ipv4 else None,
'ipv6': nic.ipv6 if nic.ipv6 else None,
'ipv4': nic.ipv4,
'ipv6': nic.ipv6,
'OS-EXT-IPS:type': ip_type}
if nic.firewall_profile:
......
......@@ -231,27 +231,30 @@ def get_floating_ip(user_id, ipv4, for_update=False):
raise faults.ItemNotFound("Floating IP does not exist.")
def validate_network_params(subnet, gateway=None, subnet6=None, gateway6=None):
try:
# Use strict option to not all subnets with host bits set
network = ipaddr.IPv4Network(subnet, strict=True)
except ValueError:
raise faults.BadRequest("Invalid network IPv4 subnet")
# Check that network size is allowed!
if not validate_network_size(network.prefixlen):
raise faults.OverLimit(message="Unsupported network size",
details="Network mask must be in range"
" (%s, 29]" % MAX_CIDR_BLOCK)
def validate_network_params(subnet=None, gateway=None, subnet6=None,
gateway6=None):
if (subnet is None) and (subnet6 is None):
raise faults.BadRequest("subnet or subnet6 is required")
# Check that gateway belongs to network
if gateway:
if subnet:
try:
gateway = ipaddr.IPv4Address(gateway)
# Use strict option to not all subnets with host bits set
network = ipaddr.IPv4Network(subnet, strict=True)
except ValueError:
raise faults.BadRequest("Invalid network IPv4 gateway")
if not gateway in network:
raise faults.BadRequest("Invalid network IPv4 gateway")
raise faults.BadRequest("Invalid network IPv4 subnet")
# Check that network size is allowed!
if not validate_network_size(network.prefixlen):
raise faults.OverLimit(message="Unsupported network size",
details="Network mask must be in range"
" (%s, 29]" % MAX_CIDR_BLOCK)
if gateway: # Check that gateway belongs to network
try:
gateway = ipaddr.IPv4Address(gateway)
except ValueError:
raise faults.BadRequest("Invalid network IPv4 gateway")
if not gateway in network:
raise faults.BadRequest("Invalid network IPv4 gateway")
if subnet6:
try:
......@@ -292,6 +295,7 @@ def backend_public_networks(backend):
"""
bnets = BackendNetwork.objects.filter(backend=backend,
network__public=True,
network__subnet__isnull=False,
network__deleted=False,
network__drained=False)
return [b.network for b in bnets]
......
......@@ -495,7 +495,9 @@ class Network(models.Model):
name = models.CharField('Network Name', max_length=128)
userid = models.CharField('User ID of the owner', max_length=128,
null=True, db_index=True)
subnet = models.CharField('Subnet', max_length=32, default='10.0.0.0/24')
# subnet will be null for IPv6 only networks
subnet = models.CharField('Subnet', max_length=32, null=True)
# subnet6 will be null for IPv4 only networks
subnet6 = models.CharField('IPv6 Subnet', max_length=64, null=True)
gateway = models.CharField('Gateway', max_length=32, null=True)
gateway6 = models.CharField('IPv6 Gateway', max_length=64, null=True)
......@@ -794,7 +796,8 @@ def pooled_rapi_client(obj):
if backend.offline:
log.warning("Trying to connect with offline backend: %s", backend)
raise faults.ServiceUnavailable
raise faults.ServiceUnavailable("Can not connect to offline"
" backend: %s" % backend)
b = backend
client = get_rapi_client(b.id, b.hash, b.clustername, b.port,
......
......@@ -141,6 +141,7 @@ class NetworkFactory(factory.DjangoModelFactory):
userid = factory.Sequence(user_seq())
subnet = factory.Sequence(lambda n: '192.168.{0}.0/24'.format(n))
gateway = factory.LazyAttribute(lambda a: a.subnet[:-4] + '1')
subnet6 = "2001:648:2ffc:1112::/64"
dhcp = False
flavor = factory.Sequence(round_seq(models.Network.FLAVORS.keys()))
mode = factory.LazyAttribute(lambda a:
......@@ -154,6 +155,11 @@ class NetworkFactory(factory.DjangoModelFactory):
state = factory.Sequence(round_seq_first(models.Network.OPER_STATES))
class IPv6NetworkFactory(NetworkFactory):
subnet = None
gateway = None
class DeletedNetwork(NetworkFactory):
deleted = True
......@@ -172,8 +178,8 @@ class NetworkInterfaceFactory(factory.DjangoModelFactory):
machine = factory.SubFactory(VirtualMachineFactory)
network = factory.SubFactory(NetworkFactory)
index = factory.Sequence(lambda x: x, type=int)
mac = factory.Sequence(lambda n:
'aa:{0}{0}:{0}{0}:aa:{0}{0}:{0}{0}'.format(hex(int(n) % 15)[2:3]))
mac = factory.Sequence(lambda n: 'aa:{0}{0}:{0}{0}:aa:{0}{0}:{0}{0}'
.format(hex(int(n) % 15)[2:3]))
ipv4 = factory.LazyAttributeSequence(lambda a, n: a.network.subnet[:-4] +
'{0}'.format(int(n) + 2))
state = "ACTIVE"
......
......@@ -231,7 +231,7 @@ def _process_net_status(vm, etime, nics):
vm.nics.all().delete()
for nic in ganeti_nics:
ipv4 = nic.get('ipv4', '')
ipv4 = nic["ipv4"]
net = nic['network']
if ipv4:
net.reserve_address(ipv4)
......@@ -257,15 +257,12 @@ def process_ganeti_nics(ganeti_nics):
net = Network.objects.get(pk=pk)
# Get the new nic info
mac = new_nic.get('mac', '')
ipv4 = new_nic.get('ip', '')
if net.subnet6:
ipv6 = mac2eui64(mac, net.subnet6)
else:
ipv6 = ''
firewall = new_nic.get('firewall', '')
firewall_profile = _reverse_tags.get(firewall, '')
mac = new_nic.get('mac')
ipv4 = new_nic.get('ip')
ipv6 = mac2eui64(mac, net.subnet6) if net.subnet6 is not None else None
firewall = new_nic.get('firewall')
firewall_profile = _reverse_tags.get(firewall)
if not firewall_profile and net.public:
firewall_profile = settings.DEFAULT_FIREWALL_PROFILE
......@@ -655,6 +652,14 @@ def _create_network(network, backend):
else:
conflicts_check = False
# Use a dummy network subnet for IPv6 only networks. Currently Ganeti does
# not support IPv6 only networks. To bypass this limitation, we create the
# network with a dummy network subnet, and make Cyclades connect instances
# to such networks, with address=None.
subnet = network.subnet
if subnet is None:
subnet = "10.0.0.0/24"
try:
bn = BackendNetwork.objects.get(network=network, backend=backend)
mac_prefix = bn.mac_prefix
......@@ -664,7 +669,7 @@ def _create_network(network, backend):
with pooled_rapi_client(backend) as client:
return client.CreateNetwork(network_name=network.backend_id,
network=network.subnet,
network=subnet,
network6=network.subnet6,
gateway=network.gateway,
gateway6=network.gateway6,
......
......@@ -32,7 +32,8 @@ import datetime
from django.utils import importlib
from synnefo.settings import (BACKEND_ALLOCATOR_MODULE, BACKEND_REFRESH_MIN,
BACKEND_PER_USER, ARCHIPELAGO_BACKENDS)
BACKEND_PER_USER, ARCHIPELAGO_BACKENDS,
DEFAULT_INSTANCE_NETWORKS)
from synnefo.db.models import Backend
from synnefo.logic.backend import update_resources
from synnefo.api.util import backend_public_networks
......@@ -101,7 +102,9 @@ def get_available_backends():
"""
backends = list(Backend.objects.select_for_update().filter(drained=False,
offline=False))
return filter(lambda x: has_free_ip(x), backends)
if "SNF:ANY_PUBLIC" in DEFAULT_INSTANCE_NETWORKS:
backends = filter(lambda x: has_free_ip(x), backends)
return backends
def filter_archipelagos_backends(available_backends, disk_template):
......
# Copyright 2012 GRNET S.A. All rights reserved.
# Copyright 2012-2013 GRNET S.A. All rights reserved.
#
# Redistribution and use in source and binary forms, with or
# without modification, are permitted provided that the following
......@@ -31,8 +31,9 @@
# interpreted as representing official policies, either expressed
# or implied, of GRNET S.A.
from synnefo.db.models import Backend, IPPoolTable
from synnefo.db.models import Backend
from synnefo.webproject.management.commands import ListCommand
from synnefo.api import util
class Command(ListCommand):
......@@ -51,17 +52,10 @@ class Command(ListCommand):
def get_ips(backend):
free_ips = 0
total_ips = 0
for bnet in backend.networks.filter(deleted=False,
network__drained=False,
network__public=True,
network__deleted=False):
network = bnet.network
try:
pool = IPPoolTable.objects.get(id=network.pool_id).pool
free_ips += pool.count_available()
total_ips += pool.pool_size
except IPPoolTable.DoesNotExist:
pass
for network in util.backend_public_networks(backend):
pool = network.get_pool(with_lock=False)
free_ips += pool.count_available()
total_ips += pool.pool_size
return "%s/%s" % (free_ips, total_ips)
FIELDS = {
......
# Copyright 2011-2012 GRNET S.A. All rights reserved.
# Copyright 2011-2013 GRNET S.A. All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
......@@ -34,22 +34,10 @@ the state of the Ganeti backend. See docstring on top of
logic/reconciliation.py for a description of reconciliation rules.
"""
import sys
import datetime
import bitarray
import logging
from optparse import make_option
from django.core.management.base import BaseCommand
from django.db import transaction
from synnefo.db.models import Backend, Network, BackendNetwork
from synnefo.db.pools import IPPool
from synnefo.logic import reconciliation, utils
from synnefo.logic import backend as backend_mod
fix = False
write = sys.stdout.write
from synnefo.logic import reconciliation
class Command(BaseCommand):
......@@ -66,8 +54,6 @@ Network reconciliation can detect and fix the following cases:
"""
can_import_settings = True
output_transaction = True # The management command runs inside
# an SQL transaction
option_list = BaseCommand.option_list + (
make_option('--fix-all', action='store_true',
dest='fix', default=False,
......@@ -78,250 +64,28 @@ Network reconciliation can detect and fix the following cases:
)
def handle(self, **options):
global fix, write
fix = options['fix']
write = self.stdout.write
self.verbosity = int(options['verbosity'])
conflicting_ips = options['conflicting_ips']
reconcile_networks(conflicting_ips)
def reconcile_networks(conflicting_ips=False):
# Get models from DB
backends = Backend.objects.exclude(offline=True)
networks = Network.objects.filter(deleted=False)
# Get info from all ganeti backends
ganeti_networks = {}
ganeti_hanging_networks = {}
for b in backends:
g_nets = reconciliation.get_networks_from_ganeti(b)
ganeti_networks[b] = g_nets
g_hanging_nets = reconciliation.hanging_networks(b, g_nets)
ganeti_hanging_networks[b] = g_hanging_nets
# Perform reconciliation for each network
for network in networks:
ip_available_maps = []
ip_reserved_maps = []
for bend in backends:
bnet = get_backend_network(network, bend)
gnet = ganeti_networks[bend].get(network.id)
if not bnet:
if network.floating_ip_pool:
# Network is a floating IP pool and does not exist in
# backend. We need to create it
bnet = reconcile_parted_network(network, bend)
elif not gnet:
# Network does not exist either in Ganeti nor in BD.
continue
else:
# Network exists in Ganeti and not in DB.
if network.action != "DESTROY" and not network.public:
bnet = reconcile_parted_network(network, bend)
if not gnet:
# Network does not exist in Ganeti. If the network action is
# DESTROY, we have to mark as deleted in DB, else we have to
# create it in Ganeti.
if network.action == "DESTROY":
if bnet.operstate != "DELETED":
reconcile_stale_network(bnet)
else:
reconcile_missing_network(network, bend)
# Skip rest reconciliation!
continue
try:
hanging_groups = ganeti_hanging_networks[bend][network.id]
except KeyError:
# Network is connected to all nodegroups
hanging_groups = []
if hanging_groups:
# CASE-3: Ganeti networks not connected to all nodegroups
reconcile_hanging_groups(network, bend, hanging_groups)
continue
if bnet.operstate != 'ACTIVE':
# CASE-4: Unsynced network state. At this point the network
# exists and is connected to all nodes so is must be active!
reconcile_unsynced_network(network, bend, bnet)
# Get ganeti IP Pools
available_map, reserved_map = get_network_pool(gnet)
ip_available_maps.append(available_map)
ip_reserved_maps.append(reserved_map)
if ip_available_maps or ip_reserved_maps:
# CASE-5: Unsynced IP Pools
reconcile_ip_pools(network, ip_available_maps, ip_reserved_maps)
if conflicting_ips:
detect_conflicting_ips()
# CASE-6: Orphan networks
reconcile_orphan_networks(networks, ganeti_networks)
def get_backend_network(network, backend):
try:
return BackendNetwork.objects.get(network=network, backend=backend)
except BackendNetwork.DoesNotExist:
return None
def reconcile_parted_network(network, backend):
write("D: Missing DB entry for network %s in backend %s\n" %
(network, backend))
if fix:
network.create_backend_network(backend)
write("F: Created DB entry\n")
bnet = get_backend_network(network, backend)
return bnet
def reconcile_stale_network(backend_network):
write("D: Stale DB entry for network %s in backend %s\n" %
(backend_network.network, backend_network.backend))
if fix:
etime = datetime.datetime.now()
backend_mod.process_network_status(backend_network, etime, 0,
"OP_NETWORK_REMOVE",
"success",
"Reconciliation simulated event")
write("F: Reconciled event: OP_NETWORK_REMOVE\n")
def reconcile_missing_network(network, backend):
write("D: Missing Ganeti network %s in backend %s\n" %
(network, backend))
if fix:
backend_mod.create_network(network, backend)
write("F: Issued OP_NETWORK_CONNECT\n")
def reconcile_hanging_groups(network, backend, hanging_groups):
write('D: Network %s in backend %s is not connected to '
'the following groups:\n' % (network, backend))
write('- ' + '\n- '.join(hanging_groups) + '\n')
if fix:
for group in hanging_groups:
write('F: Connecting network %s to nodegroup %s\n'
% (network, group))
backend_mod.connect_network(network, backend, depends=[],
group=group)
def reconcile_unsynced_network(network, backend, backend_network):
write("D: Unsynced network %s in backend %s\n" % (network, backend))
if fix:
write("F: Issuing OP_NETWORK_CONNECT\n")
etime = datetime.datetime.now()
backend_mod.process_network_status(backend_network, etime, 0,
"OP_NETWORK_CONNECT",
"success",
"Reconciliation simulated eventd")
@transaction.commit_on_success
def reconcile_ip_pools(network, available_maps, reserved_maps):
available_map = reduce(lambda x, y: x & y, available_maps)