Commit 9bbc33bd authored by Dimitris Aragiorgis's avatar Dimitris Aragiorgis Committed by Christos Stavrakakis

Introduce network flavors to replace network types

Add tags and mode to network db model. Replace existing type with
flavor. Introduce 4 different flavors. Each flavor has mode, link,
mac_prefix, tags. Supported flavors are CUSTOM, IP_LESS_ROUTED,
MAC_FILTERED, PHYSICAL_VLAN and are hardcoded in db models.

Introduce new function that returns flavor's mode, link, mac_prefix,
tags.

In settings only DEFAULT_ROUTING_TABLE, DEFAULT_BRIDGE and
DEFAULT_MAC_PREFIX, DEFAULT_PRIVATE_BRIDGE exist.

Introduce FIELD_POOL_MAP to map flavor fields to pool tables.

Make snf-manage network-create command as flexible as possible. It is
now able to create a network based on an existing flavor and overide
defaults by passing mode, link, mac_prefix, tags option. Resource pools
cannot be used by CUSTOM flavors. Currently MAC_FILTERED and
PHYSICAL_VLAN use MacPrefixPoolTable for mac_prefix and BridgePoolTable
for link accordingly and cannot be overriden.

API blocks creation of public networks. Introduce new setting
API_ENABLED_NETWORK_FLAVORS to indicate which flavors can the end-user
use to create private networks.
Signed-off-by: default avatarDimitris Aragiorgis <dimara@grnet.gr>
parent 9d1f6e82
......@@ -38,12 +38,9 @@ from synnefo.management.common import validate_network_info, get_backend
from synnefo.db.models import Network
from synnefo.logic.backend import create_network
from synnefo.api.util import values_from_flavor
from synnefo.api.util import net_resources
NETWORK_TYPES = ['PUBLIC_ROUTED', 'PRIVATE_MAC_FILTERED',
'PRIVATE_PHYSICAL_VLAN', 'CUSTOM_ROUTED',
'CUSTOM_BRIDGED']
NETWORK_FLAVORS = Network.FLAVORS.keys()
class Command(BaseCommand):
......@@ -68,6 +65,14 @@ class Command(BaseCommand):
dest='gateway',
default=None,
help='Gateway of the network'),
make_option('--subnet6',
dest='subnet6',
default=None,
help='IPv6 subnet of the network'),
make_option('--gateway6',
dest='gateway6',
default=None,
help='IPv6 gateway of the network'),
make_option('--dhcp',
dest='dhcp',
action='store_true',
......@@ -78,32 +83,32 @@ class Command(BaseCommand):
action='store_true',
default=False,
help='Network is public'),
make_option('--type',
dest='type',
default='PRIVATE_MAC_FILTERED',
choices=NETWORK_TYPES,
help='Type of network. Choices: ' + ', '.join(NETWORK_TYPES)),
make_option('--subnet6',
dest='subnet6',
default=None,
help='IPv6 subnet of the network'),
make_option('--gateway6',
dest='gateway6',
make_option('--flavor',
dest='flavor',
default=None,
help='IPv6 gateway of the network'),
make_option('--backend-id',
dest='backend_id',
choices=NETWORK_FLAVORS,
help='Network flavor. Choices: ' + ', '.join(NETWORK_FLAVORS)),
make_option('--mode',
dest='mode',
default=None,
help='ID of the backend that the network will be created. Only for'
' public networks'),
help="Overwrite flavor connectivity mode."),
make_option('--link',
dest='link',
default=None,
help="Connectivity link of the Network. None for default."),
help="Overwrite flavor connectivity link."),
make_option('--mac-prefix',
dest='mac_prefix',
default=None,
help="MAC prefix of the network. None for default")
help="Overwrite flavor connectivity MAC prefix"),
make_option('--tags',
dest='tags',
default=None,
help='The tags of the Network (comma separated strings)'),
make_option('--backend-id',
dest='backend_id',
default=None,
help='ID of the backend that the network will be created. Only for'
' public networks'),
)
def handle(self, *args, **options):
......@@ -112,53 +117,59 @@ class Command(BaseCommand):
name = options['name']
subnet = options['subnet']
net_type = options['type']
backend_id = options['backend_id']
public = options['public']
flavor = options['flavor']
mode = options['mode']
link = options['link']
mac_prefix = options['mac_prefix']
tags = options['tags']
if not name:
raise CommandError("Name is required")
if not subnet:
raise CommandError("Subnet is required")
if not flavor:
raise CommandError("Flavor is required")
if public and not backend_id:
raise CommandError("backend-id is required")
if backend_id and not public:
raise CommandError("Private networks must be created to"
" all backends")
if mac_prefix and net_type == "PRIVATE_MAC_FILTERED":
if mac_prefix and flavor == "MAC_FILTERED":
raise CommandError("Can not override MAC_FILTERED mac-prefix")
if link and net_type == "PRIVATE_PHYSICAL_VLAN":
if link and flavor == "PHYSICAL_VLAN":
raise CommandError("Can not override PHYSICAL_VLAN link")
if backend_id:
backend = get_backend(backend_id)
default_link, default_mac_prefix = net_resources(net_type)
if not link:
link = default_link
if not mac_prefix:
mac_prefix = default_mac_prefix
fmode, flink, fmac_prefix, ftags = values_from_flavor(flavor)
mode = mode or fmode
link = link or flink
mac_prefix = mac_prefix or fmac_prefix
tags = tags or ftags
subnet, gateway, subnet6, gateway6 = validate_network_info(options)
if not link:
raise CommandError("Can not create network. No connectivity link")
if not link or not mode:
raise CommandError("Can not create network. No connectivity link or mode")
network = Network.objects.create(
name=name,
userid=options['owner'],
subnet=subnet,
gateway=gateway,
gateway6=gateway6,
subnet6=subnet6,
dhcp=options['dhcp'],
type=net_type,
flavor=flavor,
public=public,
mode=mode,
link=link,
mac_prefix=mac_prefix,
gateway6=gateway6,
subnet6=subnet6,
tags=tags,
state='PENDING')
if public:
......
......@@ -90,7 +90,7 @@ class Command(BaseCommand):
if filter_by:
networks = filter_results(networks, filter_by)
headers = ['id', 'name', 'type', 'owner',
headers = ['id', 'name', 'flavor', 'owner',
'mac_prefix', 'dhcp', 'state', 'link', 'vms', 'public']
if options['ipv6']:
......@@ -102,7 +102,7 @@ class Command(BaseCommand):
for network in networks.order_by("id"):
fields = [str(network.id),
network.name,
network.type,
network.flavor,
network.userid or '',
network.mac_prefix or '',
str(network.dhcp),
......
......@@ -90,7 +90,7 @@ def network_to_dict(network, user_id, detail=True):
d['gateway'] = network.gateway
d['gateway6'] = network.gateway6
d['dhcp'] = network.dhcp
d['type'] = network.type
d['type'] = network.flavor
d['updated'] = util.isoformat(network.updated)
d['created'] = util.isoformat(network.created)
d['status'] = network.state
......@@ -169,7 +169,7 @@ def create_network(serials, request):
subnet6 = d.get('cidr6', None)
gateway = d.get('gateway', None)
gateway6 = d.get('gateway6', None)
net_type = d.get('type', 'PRIVATE_MAC_FILTERED')
flavor = d.get('type', 'MAC_FILTERED')
public = d.get('public', False)
dhcp = d.get('dhcp', True)
except (KeyError, ValueError):
......@@ -178,12 +178,11 @@ def create_network(serials, request):
if public:
raise Forbidden('Can not create a public network.')
if net_type not in ['PUBLIC_ROUTED', 'PRIVATE_MAC_FILTERED',
'PRIVATE_PHYSICAL_VLAN', 'CUSTOM_ROUTED',
'CUSTOM_BRIDGED']:
raise BadRequest("Invalid network type: %s", net_type)
if net_type not in settings.ENABLED_NETWORKS:
raise Forbidden("Can not create %s network" % net_type)
if flavor not in Network.FLAVORS.keys():
raise BadRequest("Invalid network flavors %s" % flavor)
if flavor not in settings.API_ENABLED_NETWORK_FLAVORS:
raise Forbidden("Can not create %s network" % flavor)
cidr_block = int(subnet.split('/')[1])
if not util.validate_network_size(cidr_block):
......@@ -198,10 +197,7 @@ def create_network(serials, request):
serial.save()
try:
link, mac_prefix = util.net_resources(net_type)
if not link:
raise Exception("Can not create network. No connectivity link.")
mode, link, mac_prefix, tags = util.values_from_flavor(flavor)
network = Network.objects.create(
name=name,
userid=user_id,
......@@ -210,15 +206,17 @@ def create_network(serials, request):
gateway=gateway,
gateway6=gateway6,
dhcp=dhcp,
type=net_type,
flavor=flavor,
mode=mode,
link=link,
mac_prefix=mac_prefix,
tags=tags,
action='CREATE',
state='PENDING',
serial=serial)
except EmptyPool:
log.error("Failed to allocate resources for network of type: %s",
net_type)
flavor)
raise ServiceUnavailable("Failed to allocate resources for network")
# Create BackendNetwork entries for each Backend
......
......@@ -462,29 +462,6 @@ def construct_nic_id(nic):
return "-".join(["nic", unicode(nic.machine.id), unicode(nic.index)])
def net_resources(net_type):
mac_prefix = settings.MAC_POOL_BASE
if net_type == 'PRIVATE_MAC_FILTERED':
link = settings.PRIVATE_MAC_FILTERED_BRIDGE
mac_pool = MacPrefixPoolTable.get_pool()
mac_prefix = mac_pool.get()
mac_pool.save()
elif net_type == 'PRIVATE_PHYSICAL_VLAN':
pool = BridgePoolTable.get_pool()
link = pool.get()
pool.save()
elif net_type == 'CUSTOM_ROUTED':
link = settings.CUSTOM_ROUTED_ROUTING_TABLE
elif net_type == 'CUSTOM_BRIDGED':
link = settings.CUSTOM_BRIDGED_BRIDGE
elif net_type == 'PUBLIC_ROUTED':
link = settings.PUBLIC_ROUTED_ROUTING_TABLE
else:
raise BadRequest('Unknown network type')
return link, mac_prefix
def verify_personality(personality):
"""Verify that a a list of personalities is well formed"""
if len(personality) > settings.MAX_PERSONALITY:
......@@ -520,3 +497,53 @@ def get_flavor_provider(flavor):
if disk_template.startswith("ext"):
disk_template, provider = disk_template.split("_", 1)
return disk_template, provider
def values_from_flavor(flavor):
"""Get Ganeti connectivity info from flavor type.
If link or mac_prefix equals to "pool", then the resources
are allocated from the corresponding Pools.
"""
try:
flavor = Network.FLAVORS[flavor]
except KeyError:
raise BadRequest("Unknown network flavor")
mode = flavor.get("mode")
link = flavor.get("link")
if link == "pool":
link = allocate_resource("bridge")
mac_prefix = flavor.get("mac_prefix")
if mac_prefix == "pool":
mac_prefix = allocate_resource("mac_prefix")
tags = flavor.get("tags")
return mode, link, mac_prefix, tags
def allocate_resource(res_type):
table = get_pool_table(res_type)
pool = table.get_pool()
value = pool.get()
pool.save()
return value
def release_resource(res_type, value):
table = get_pool_table(res_type)
pool = table.get_pool()
pool.put(value)
pool.save()
def get_pool_table(res_type):
if res_type == "bridge":
return BridgePoolTable
elif res_type == "mac_prefix":
return MacPrefixPoolTable
else:
raise Exception("Unknown resource type")
......@@ -24,24 +24,9 @@ POLL_LIMIT = 3600
# Maximum allowed network size for private networks.
MAX_CIDR_BLOCK = 22
# The first mac prefix to use
MAC_POOL_BASE = 'aa:00:0'
MAC_POOL_LIMIT = 65536
ENABLED_NETWORKS = ['PUBLIC_ROUTED',
'PRIVATE_MAC_FILTERED',
'PRIVATE_PHYSICAL_VLAN']
# CUSTOM_ROUTED,
# CUSTOM_BRIDGED,
# Settings for PUBLIC_ROUTED network:
# -----------------------------------
# In this case VMCs act as routers that forward the traffic to/from VMs, based
# on the defined routing table($PUBLIC_ROUTED_ROUTING_TABLE) and ip rules, that
# exist in every node, implenting an IP-less routed and proxy-arp setup.
# (This value is also hardcoded in fixture db/fixtures/initial_data.json)
PUBLIC_ROUTED_ROUTING_TABLE = 'snf_public'
PUBLIC_ROUTED_TAGS = ['ip-less-routed']
# Default settings used by network flavors
DEFAULT_MAC_PREFIX = 'aa:00:0'
DEFAULT_BRIDGE = 'br0'
# Boolean value indicating whether synnefo would hold a Pool and allocate IP
# addresses. If this setting is set to False, IP pool management will be
......@@ -49,37 +34,23 @@ PUBLIC_ROUTED_TAGS = ['ip-less-routed']
# you must run network reconciliation after turning it to True.
PUBLIC_USE_POOL = True
# Settings for PRIVATE_MAC_FILTERED network:
# Network flavors that users are allowed to create through API requests
API_ENABLED_NETWORK_FLAVORS = ['MAC_FILTERED']
# Settings for IP_LESS_ROUTED network:
# -----------------------------------
# In this case VMCs act as routers that forward the traffic to/from VMs, based
# on the defined routing table($DEFAULT_ROUTING_TABLE) and ip rules, that
# exist in every node, implenting an IP-less routed and proxy-arp setup.
DEFAULT_ROUTING_TABLE = 'snf_public'
# Settings for MAC_FILTERED network:
# ------------------------------------------
# All networks of this type are bridged to the same bridge. Isolation between
# networks is achieved by assigning a unique MAC-prefix to each network and
# filtering packets via ebtables.
PRIVATE_MAC_FILTERED_BRIDGE = 'br0'
PRIVATE_MAC_FILTERED_TAGS = ['private-filtered']
DEFAULT_MAC_FILTERED_BRIDGE = 'prv0'
# Settings for PRIVATE_PHSICAL_VLAN network:
# ------------------------------------------
# Each network of this type is mapped to an isolated physical VLAN, which must
# be preconfigured in the backend. Each vlan corresponds to a bridge named
# $PRIVATE_PHYSICAL_VLAN_PREFIX{1..$PRIVATE_PHYSICAL_VLAN_MAX_NUMBER} (e.g. prv5)
# VirtualMachine's taps are eventually bridged to the corresponding bridge.
PRIVATE_PHYSICAL_VLAN_BRIDGE_PREFIX = 'prv'
# The max limit of physical vlan pool
PRIVATE_PHYSICAL_VLAN_MAX_NUMBER = 100
PRIVATE_PHYSICAL_VLAN_TAGS = ['physical-vlan']
# Settings for CUSTOM_ROUTED:
# ---------------------------
# Same as PUBLIC_ROUTED but with custom values
CUSTOM_ROUTED_ROUTING_TABLE = 'custom_routing_table'
CUSTOM_ROUTED_TAGS = []
# Settings for CUSTOM_BRIDGED:
# ---------------------------
# Same as PRIVATE_BRIDGED but with custom values
CUSTOM_BRIDGED_BRIDGE = 'custom_bridge'
CUSTOM_BRIDGED_TAGS = []
# Firewalling
GANETI_FIREWALL_ENABLED_TAG = 'synnefo:network:0:protected'
......
......@@ -164,7 +164,7 @@ UI_OS_DEFAULT_USER_MAP = {
# Available network types for use to choose when creating a private network
# If only one set, no select options will be displayed
UI_NETWORK_AVAILABLE_NETWORK_TYPES = {'PRIVATE_MAC_FILTERED': 'mac-filtering'}
UI_NETWORK_AVAILABLE_NETWORK_TYPES = {'MAC_FILTERED': 'mac-filtering'}
# Suggested private networks to let the user choose from when creating a private
# network with dhcp enabled
......
......@@ -426,13 +426,40 @@ class Network(models.Model):
'ERROR': 'ERROR'
}
NETWORK_TYPES = (
('PUBLIC_ROUTED', 'Public routed network'),
('PRIVATE_PHYSICAL_VLAN', 'Private vlan network'),
('PRIVATE_MAC_FILTERED', 'Private network with mac-filtering'),
('CUSTOM_ROUTED', 'Custom routed network'),
('CUSTOM_BRIDGED', 'Custom bridged network')
)
FLAVORS = {
'CUSTOM': {
'mode': 'bridged',
'link': settings.DEFAULT_BRIDGE,
'mac_prefix': settings.DEFAULT_MAC_PREFIX,
'tags': None,
'desc': "Basic flavor used for a bridged network",
},
'IP_LESS_ROUTED': {
'mode': 'routed',
'link': settings.DEFAULT_ROUTING_TABLE,
'mac_prefix': settings.DEFAULT_MAC_PREFIX,
'tags': 'ip-less-routed',
'desc': "Flavor used for an IP-less routed network using"
" Proxy ARP",
},
'MAC_FILTERED': {
'mode': 'bridged',
'link': settings.DEFAULT_MAC_FILTERED_BRIDGE,
'mac_prefix': 'pool',
'tags': 'private-filtered',
'desc': "Flavor used for bridged networks that offer isolation"
" via filtering packets based on their src "
" MAC (ebtables)",
},
'PHYSICAL_VLAN': {
'mode': 'bridged',
'link': 'pool',
'mac_prefix': settings.DEFAULT_MAC_PREFIX,
'tags': 'physical-vlan',
'desc': "Flavor used for bridged network that offer isolation"
" via dedicated physical vlan",
},
}
name = models.CharField('Network Name', max_length=128)
userid = models.CharField('User ID of the owner', max_length=128,
......@@ -442,10 +469,11 @@ class Network(models.Model):
gateway = models.CharField('Gateway', max_length=32, null=True)
gateway6 = models.CharField('IPv6 Gateway', max_length=64, null=True)
dhcp = models.BooleanField('DHCP', default=True)
type = models.CharField(choices=NETWORK_TYPES, max_length=50,
default='PRIVATE_PHYSICAL_VLAN')
link = models.CharField('Network Link', max_length=128, null=True)
flavor = models.CharField('Flavor', max_length=32, null=False)
mode = models.CharField('Network Mode', max_length=16, null=True)
link = models.CharField('Network Link', max_length=32, null=True)
mac_prefix = models.CharField('MAC Prefix', max_length=32, null=False)
tags = models.CharField('Network Tags', max_length=128, null=True)
public = models.BooleanField(default=False, db_index=True)
created = models.DateTimeField(auto_now_add=True)
updated = models.DateTimeField(auto_now=True)
......@@ -482,7 +510,10 @@ class Network(models.Model):
"""Return the network tag to be used in backend
"""
return getattr(snf_settings, self.type + '_TAGS')
if self.tags:
return self.tags.split(',')
else:
return []
def create_backend_network(self, backend=None):
"""Create corresponding BackendNetwork entries."""
......
......@@ -43,6 +43,7 @@ from synnefo.db.models import (Backend, VirtualMachine, Network,
MacPrefixPoolTable, VirtualMachineDiagnostic)
from synnefo.logic import utils
from synnefo import quotas
from synnefo.api.util import release_resource
from logging import getLogger
log = getLogger(__name__)
......@@ -255,15 +256,13 @@ def update_network_state(serials, network):
log.info("Network %r deleted. Releasing link %r mac_prefix %r",
network.id, network.mac_prefix, network.link)
network.deleted = True
if network.mac_prefix and network.type == 'PRIVATE_MAC_FILTERED':
mac_pool = MacPrefixPoolTable.get_pool()
mac_pool.put(network.mac_prefix)
mac_pool.save()
if network.link and network.type == 'PRIVATE_VLAN':
bridge_pool = BridgePoolTable.get_pool()
bridge_pool.put(network.link)
bridge_pool.save()
if network.mac_prefix:
if network.FLAVORS[network.flavor]["mac_prefix"] == "pool":
release_resource(res_type="mac_prefix",
value=network.mac_prefix)
if network.link:
if network.FLAVORS[network.flavor]["link"] == "pool":
release_resource(res_type="bridge", value=network.link)
# Issue commission
serial = quotas.issue_network_commission(network.userid, delete=True)
......@@ -518,8 +517,6 @@ def connect_network(network, backend, depend_job=None, group=None):
"""Connect a network to nodegroups."""
log.debug("Connecting network %s to backend %s", network, backend)
mode = "routed" if "ROUTED" in network.type else "bridged"
if network.public:
conflicts_check = True
else:
......@@ -528,11 +525,11 @@ def connect_network(network, backend, depend_job=None, group=None):
depend_jobs = [depend_job] if depend_job else []
with pooled_rapi_client(backend) as client:
if group:
client.ConnectNetwork(network.backend_id, group, mode,
client.ConnectNetwork(network.backend_id, group, network.mode,
network.link, conflicts_check, depend_jobs)
else:
for group in client.GetGroups():
client.ConnectNetwork(network.backend_id, group, mode,
client.ConnectNetwork(network.backend_id, group, network.mode,
network.link, conflicts_check,
depend_jobs)
......@@ -723,14 +720,10 @@ def _create_network_synced(network, backend):
def connect_network_synced(network, backend):
if network.type in ('PUBLIC_ROUTED', 'CUSTOM_ROUTED'):
mode = 'routed'
else:
mode = 'bridged'
with pooled_rapi_client(backend) as client:
for group in client.GetGroups():
job = client.ConnectNetwork(network.backend_id, group, mode,
network.link)
job = client.ConnectNetwork(network.backend_id, group,
network.mode, network.link)
result = wait_for_job(client, job)
if result[0] != 'success':
return result
......
......@@ -32,7 +32,7 @@ from django.core.management.base import BaseCommand
from synnefo.db.models import (Network, BackendNetwork,
BridgePoolTable, MacPrefixPoolTable)
from synnefo.db.pools import EmptyPool
from synnefo.settings import MAC_POOL_BASE
class Command(BaseCommand):
......@@ -63,7 +63,7 @@ class Command(BaseCommand):
write("Used bridges from Pool: %d\n" % len(bridges))
network_bridges = Network.objects.filter(type='PRIVATE_PHYSICAL_VLAN',
network_bridges = Network.objects.filter(flavor='PHYSICAL_VLAN',
deleted=False)\
.values_list('link', flat=True)
......@@ -97,18 +97,16 @@ class Command(BaseCommand):
return
macs = []
for i in xrange(0, macp_pool.size()):
for i in xrange(1, macp_pool.size()):
if not macp_pool.is_available(i, index=True) and \
not macp_pool.is_reserved(i, index=True):
value = macp_pool.index_to_value(i)
if value != MAC_POOL_BASE:
macs.append(macp_pool.index_to_value(i))
macs.append(value)
write("Used MAC prefixes from Pool: %d\n" % len(macs))
network_mac_prefixes = \
Network.objects.filter(deleted=False)\
.exclude(mac_prefix=MAC_POOL_BASE) \
Network.objects.filter(deleted=False, flavor='MAC_FILTERED')\
.values_list('mac_prefix', flat=True)
write("Used MAC prefixes from Networks: %d\n" %