Commit 0adc25f1 authored by Ilias Tsitsimpis's avatar Ilias Tsitsimpis
Browse files

Merge pull request #65 from cstavr/feature-volume-types-metadata

This patch-set extends Cyclades with support for:

  * Volumes types: A type for block storage volumes, which replaces 
    the 'disk_template' attribute of volumes and flavors.
  * Volume metadata: User-defined metadata for volumes.
  * Snapshot metadata: User-defined metadata for snapshots.

Also, this patch-set contains various fixes relative with handling of volumes
and snapshots.
parents 86ca098d 95ee8e90
......@@ -130,7 +130,7 @@ extract_tests () {
export SYNNEFO_SETTINGS_DIR=/tmp/snf-test-settings
astakos_all_tests="im quotaholder_app oa2"
cyclades_all_tests="api db logic plankton quotas vmapi helpdesk userdata"
cyclades_all_tests="api db logic plankton quotas vmapi helpdesk userdata volume"
pithos_all_tests="api"
astakosclient_all_tests="astakosclient"
ALL_COMPONENTS="astakos cyclades pithos astakosclient"
......
......@@ -682,8 +682,8 @@ to Cyclades.
Working with Cyclades
---------------------
Flavors
~~~~~~~
Flavors and Volume Types
~~~~~~~~~~~~~~~~~~~~~~~~
When creating a VM, the user must specify the `flavor` of the virtual server.
Flavors are the virtual hardware templates, and provide a description about
......@@ -695,8 +695,9 @@ Flavors are created by the administrator and the user can select one of the
available flavors. After VM creation, the user can resize his VM, by
adding/removing CPU and RAM.
Cyclades support different storage backends that are described by the disk
template of the flavor, which is mapped to Ganeti's instance `disk template`.
Cyclades support different storage backends that are described by the `volume
type` of the flavor. Each volume type contains a `disk template` attribute
which is mapped to Ganeti's instance `disk template`.
Currently the available disk templates are the following:
* `file`: regulars file
......@@ -709,16 +710,24 @@ Currently the available disk templates are the following:
- `ext_archipelago`: External shared storage provided by
`Archipelago <http://www.synnefo.org/docs/archipelago/latest/index.html>`_.
Volume types are created by the administrator using the `snf-manage
volume-type-create` command and providing the `disk template` and a
human-friendly name:
.. code-block:: console
$ snf-manage volume-type-create --disk-template=drbd --name=DRBD
Flavors are created by the administrator using `snf-manage flavor-create`
command. The command takes as argument number of CPUs, amount of RAM, the size
of the disks and the disk templates and create the flavors that belong to the
of the disks and the volume type IDs and creates the flavors that belong to the
cartesian product of the specified arguments. For example, the following
command will create two flavors of `40G` disk size with `drbd` disk template,
command will create two flavors of `40G` disk size of volume type with ID `1`,
`4G` RAM and `2` or `4` CPUs.
.. code-block:: console
$ snf-manage flavor-create 2,4 4096 40 drbd
$ snf-manage flavor-create 2,4 4096 40 1
To see the available flavors, run `snf-manage flavor-list` command. The
administrator can delete a flavor by using `flavor-modify` command:
......@@ -1767,6 +1776,10 @@ network-remove Delete a network
flavor-create Create a new flavor
flavor-list List flavors
flavor-modify Modify a flavor
volume-type-create Create a new volume type
volume-type-list List volume types
volume-type-show Show volume type details
volume-type-modify Modify a volume type
image-list List images
image-show Show image details
pool-create Create a bridge or mac-prefix pool
......
......@@ -81,7 +81,10 @@ def get_resources_stats(backend=None):
for res in ["cpu", "ram", "disk", "disk_template"]:
server_count[res] = {}
allocated[res] = 0
val = "flavor__%s" % res
if res == "disk_template":
val = "flavor__volume_type__%s" % res
else:
val = "flavor__%s" % res
results = active_servers.values(val).annotate(count=Count(val))
for result in results:
server_count[res][result[val]] = result["count"]
......
......@@ -43,7 +43,8 @@ def flavor_to_dict(flavor, detail=True):
d['ram'] = flavor.ram
d['disk'] = flavor.disk
d['vcpus'] = flavor.cpu
d['SNF:disk_template'] = flavor.disk_template
d['SNF:disk_template'] = flavor.volume_type.disk_template
d['SNF:volume_type'] = flavor.volume_type_id
d['SNF:allow_create'] = flavor.allow_create
return d
......@@ -58,7 +59,8 @@ def list_flavors(request, detail=False):
# overLimit (413)
log.debug('list_flavors detail=%s', detail)
active_flavors = Flavor.objects.exclude(deleted=True)
active_flavors = Flavor.objects.select_related("volume_type")\
.exclude(deleted=True)
flavors = [flavor_to_dict(flavor, detail)
for flavor in active_flavors.order_by('id')]
......
......@@ -68,7 +68,7 @@ class FlavorAPITest(BaseAPITest):
self.assertEqual(api_flavor['name'], db_flavor.name)
self.assertEqual(api_flavor['ram'], db_flavor.ram)
self.assertEqual(api_flavor['SNF:disk_template'],
db_flavor.disk_template)
db_flavor.volume_type.disk_template)
def test_flavor_details(self):
"""Test if the expected flavor is returned."""
......@@ -85,7 +85,7 @@ class FlavorAPITest(BaseAPITest):
self.assertEqual(api_flavor['name'], db_flavor.name)
self.assertEqual(api_flavor['ram'], db_flavor.ram)
self.assertEqual(api_flavor['SNF:disk_template'],
db_flavor.disk_template)
db_flavor.volume_type.disk_template)
def test_deleted_flavor_details(self):
"""Test that API returns details for deleted flavors"""
......
......@@ -591,7 +591,7 @@ class ServerCreateAPITest(ComputeAPITest):
self.assertEqual(response.status_code, 202, msg=response.content)
vm_id = json.loads(response.content)["server"]["id"]
volume = Volume.objects.get(machine_id=vm_id)
self.assertEqual(volume.disk_template, self.flavor.disk_template)
self.assertEqual(volume.volume_type, self.flavor.volume_type)
self.assertEqual(volume.size, self.flavor.disk)
self.assertEqual(volume.source, "image:%s" % fixed_image()["id"])
self.assertEqual(volume.delete_on_termination, True)
......@@ -610,7 +610,7 @@ class ServerCreateAPITest(ComputeAPITest):
self.assertEqual(response.status_code, 202, msg=response.content)
vm_id = json.loads(response.content)["server"]["id"]
volume = Volume.objects.get(machine_id=vm_id)
self.assertEqual(volume.disk_template, self.flavor.disk_template)
self.assertEqual(volume.volume_type, self.flavor.volume_type)
self.assertEqual(volume.size, 10)
self.assertEqual(volume.source, "image:%s" % fixed_image()["id"])
self.assertEqual(volume.delete_on_termination, False)
......@@ -630,7 +630,7 @@ class ServerCreateAPITest(ComputeAPITest):
self.assertEqual(response.status_code, 202, msg=response.content)
vm_id = json.loads(response.content)["server"]["id"]
volume = Volume.objects.get(machine_id=vm_id)
self.assertEqual(volume.disk_template, self.flavor.disk_template)
self.assertEqual(volume.volume_type, self.flavor.volume_type)
self.assertEqual(volume.size, 10)
self.assertEqual(volume.source, "snapshot:%s" % fixed_image()["id"])
self.assertEqual(volume.origin, fixed_image()["mapfile"])
......@@ -855,8 +855,10 @@ class ServerActionAPITest(ComputeAPITest):
response = self.mypost('servers/%d/action' % vm.id,
vm.userid, json.dumps(request), 'json')
self.assertBadRequest(response)
flavor2 = mfactory.FlavorFactory(disk_template="foo")
flavor3 = mfactory.FlavorFactory(disk_template="baz")
# Check flavor with different volume type
flavor2 = mfactory.FlavorFactory(volume_type__disk_template="foo")
flavor3 = mfactory.FlavorFactory(volume_type__disk_template="baz")
vm = self.get_vm(flavor=flavor2, operstate="STOPPED")
request = {'resize': {'flavorRef': flavor3.id}}
response = self.mypost('servers/%d/action' % vm.id,
......@@ -864,7 +866,7 @@ class ServerActionAPITest(ComputeAPITest):
self.assertBadRequest(response)
# Check success
vm = self.get_vm(flavor=flavor, operstate="STOPPED")
flavor4 = mfactory.FlavorFactory(disk_template=flavor.disk_template,
flavor4 = mfactory.FlavorFactory(volume_type=vm.flavor.volume_type,
disk=flavor.disk,
cpu=4, ram=2048)
request = {'resize': {'flavorRef': flavor4.id}}
......@@ -981,7 +983,7 @@ class ServerAttachments(ComputeAPITest):
def test_attach_detach_volume(self, mrapi):
vol = mfactory.VolumeFactory(status="AVAILABLE")
vm = vol.machine
disk_template = vm.flavor.disk_template
volume_type = vm.flavor.volume_type
# Test that we cannot detach the root volume
response = self.mydelete("servers/%d/os-volume_attachments/%d" %
(vm.id, vol.id), vm.userid)
......@@ -989,7 +991,7 @@ class ServerAttachments(ComputeAPITest):
# Test that we cannot attach a used volume
vol1 = mfactory.VolumeFactory(status="IN_USE",
disk_template=disk_template,
volume_type=volume_type,
userid=vm.userid)
request = json.dumps({"volumeAttachment": {"volumeId": vol1.id}})
response = self.mypost("servers/%d/os-volume_attachments" %
......@@ -997,16 +999,17 @@ class ServerAttachments(ComputeAPITest):
request, "json")
self.assertBadRequest(response)
# We cannot attach a volume of different disk template
vol1.status = "AVAILABLE"
vol1.disk_template = "lalalal"
# We cannot attach a volume of different disk template
volume_type_2 = mfactory.VolumeTypeFactory(disk_template="lalalal")
vol1.volume_type = volume_type_2
vol1.save()
response = self.mypost("servers/%d/os-volume_attachments/" %
vm.id, vm.userid,
request, "json")
self.assertBadRequest(response)
vol1.disk_template = disk_template
vol1.volume_type = volume_type
vol1.save()
mrapi().ModifyInstance.return_value = 43
response = self.mypost("servers/%d/os-volume_attachments" %
......
......@@ -187,10 +187,10 @@ def get_flavor(flavor_id, include_deleted=False):
try:
flavor_id = int(flavor_id)
if include_deleted:
return Flavor.objects.get(id=flavor_id)
else:
return Flavor.objects.get(id=flavor_id, deleted=include_deleted)
flavors = Flavor.objects.select_related("volume_type")
if not include_deleted:
flavors = flavors.filter(deleted=False)
return flavors.get(id=flavor_id)
except (ValueError, TypeError):
raise faults.BadRequest("Invalid flavor ID '%s'" % flavor_id)
except Flavor.DoesNotExist:
......
[
{
"model": "db.VolumeType",
"pk": 1,
"fields": {
"name": "drbd",
"disk_template": "drbd"
}
},
{
"model": "db.Flavor",
"pk": 1,
......@@ -6,7 +15,7 @@
"cpu": 1,
"ram": 1024,
"disk": 20,
"disk_template": "drbd"
"volume_type": 1
}
},
......@@ -17,7 +26,7 @@
"cpu": 1,
"ram": 1024,
"disk": 30,
"disk_template": "drbd"
"volume_type": 1
}
},
......@@ -28,7 +37,7 @@
"cpu": 1,
"ram": 1024,
"disk": 40,
"disk_template": "drbd"
"volume_type": 1
}
},
......@@ -39,7 +48,7 @@
"cpu": 1,
"ram": 2048,
"disk": 20,
"disk_template": "drbd"
"volume_type": 1
}
},
......@@ -50,7 +59,7 @@
"cpu": 1,
"ram": 2048,
"disk": 30,
"disk_template": "drbd"
"volume_type": 1
}
},
......@@ -61,7 +70,7 @@
"cpu": 1,
"ram": 2048,
"disk": 40,
"disk_template": "drbd"
"volume_type": 1
}
},
......@@ -72,7 +81,7 @@
"cpu": 1,
"ram": 4096,
"disk": 20,
"disk_template": "drbd"
"volume_type": 1
}
},
......@@ -83,7 +92,7 @@
"cpu": 1,
"ram": 4096,
"disk": 30,
"disk_template": "drbd"
"volume_type": 1
}
},
......@@ -94,7 +103,7 @@
"cpu": 1,
"ram": 4096,
"disk": 40,
"disk_template": "drbd"
"volume_type": 1
}
},
......@@ -105,7 +114,7 @@
"cpu": 2,
"ram": 1024,
"disk": 20,
"disk_template": "drbd"
"volume_type": 1
}
},
......@@ -116,7 +125,7 @@
"cpu": 2,
"ram": 1024,
"disk": 30,
"disk_template": "drbd"
"volume_type": 1
}
},
......@@ -127,7 +136,7 @@
"cpu": 2,
"ram": 1024,
"disk": 40,
"disk_template": "drbd"
"volume_type": 1
}
},
......@@ -138,7 +147,7 @@
"cpu": 2,
"ram": 2048,
"disk": 20,
"disk_template": "drbd"
"volume_type": 1
}
},
......@@ -149,7 +158,7 @@
"cpu": 2,
"ram": 2048,
"disk": 30,
"disk_template": "drbd"
"volume_type": 1
}
},
......@@ -160,7 +169,7 @@
"cpu": 2,
"ram": 2048,
"disk": 40,
"disk_template": "drbd"
"volume_type": 1
}
},
......@@ -171,7 +180,7 @@
"cpu": 2,
"ram": 4096,
"disk": 20,
"disk_template": "drbd"
"volume_type": 1
}
},
......@@ -182,7 +191,7 @@
"cpu": 2,
"ram": 4096,
"disk": 30,
"disk_template": "drbd"
"volume_type": 1
}
},
......@@ -193,7 +202,7 @@
"cpu": 2,
"ram": 4096,
"disk": 40,
"disk_template": "drbd"
"volume_type": 1
}
},
......@@ -204,7 +213,7 @@
"cpu": 4,
"ram": 1024,
"disk": 20,
"disk_template": "drbd"
"volume_type": 1
}
},
......@@ -215,7 +224,7 @@
"cpu": 4,
"ram": 1024,
"disk": 30,
"disk_template": "drbd"
"volume_type": 1
}
},
......@@ -226,7 +235,7 @@
"cpu": 4,
"ram": 1024,
"disk": 40,
"disk_template": "drbd"
"volume_type": 1
}
},
......@@ -237,7 +246,7 @@
"cpu": 4,
"ram": 2048,
"disk": 20,
"disk_template": "drbd"
"volume_type": 1
}
},
......@@ -248,7 +257,7 @@
"cpu": 4,
"ram": 2048,
"disk": 30,
"disk_template": "drbd"
"volume_type": 1
}
},
......@@ -259,7 +268,7 @@
"cpu": 4,
"ram": 2048,
"disk": 40,
"disk_template": "drbd"
"volume_type": 1
}
},
......@@ -270,7 +279,7 @@
"cpu": 4,
"ram": 4096,
"disk": 20,
"disk_template": "drbd"
"volume_type": 1
}
},
......@@ -281,7 +290,7 @@
"cpu": 4,
"ram": 4096,
"disk": 30,
"disk_template": "drbd"
"volume_type": 1
}
},
......@@ -292,7 +301,7 @@
"cpu": 4,
"ram": 4096,
"disk": 40,
"disk_template": "drbd"
"volume_type": 1
}
}
]
......@@ -35,24 +35,50 @@ import logging
log = logging.getLogger(__name__)
class VolumeType(models.Model):
NAME_LENGTH = 255
name = models.CharField("Name", max_length=NAME_LENGTH)
disk_template = models.CharField('Disk Template', max_length=32)
deleted = models.BooleanField('Deleted', default=False)
def __str__(self):
return self.__unicode__()
def __unicode__(self):
return u"<VolumeType %s(disk_template:%s)>" % \
(self.name, self.disk_template)
@property
def template(self):
return self.disk_template.split("_")[0]
@property
def provider(self):
if "_" in self.disk_template:
return self.disk_template.split("_", 1)[1]
else:
return None
class Flavor(models.Model):
cpu = models.IntegerField('Number of CPUs', default=0)
ram = models.IntegerField('RAM size in MiB', default=0)
disk = models.IntegerField('Disk size in GiB', default=0)
disk_template = models.CharField('Disk template', max_length=32)
volume_type = models.ForeignKey(VolumeType, related_name="flavors",
on_delete=models.PROTECT, null=False)
deleted = models.BooleanField('Deleted', default=False)
# Whether the flavor can be used to create new servers
allow_create = models.BooleanField(default=True, null=False)
class Meta:
verbose_name = u'Virtual machine flavor'
unique_together = ('cpu', 'ram', 'disk', 'disk_template')
unique_together = ('cpu', 'ram', 'disk', 'volume_type')
@property
def name(self):
"""Returns flavor name (generated)"""
return u'C%sR%sD%s%s' % (self.cpu, self.ram, self.disk,
self.disk_template)
self.volume_type.disk_template)
def __str__(self):
return self.__unicode__()
......@@ -846,8 +872,8 @@ class NetworkInterface(models.Model):
return self.__unicode__()
def __unicode__(self):
return u"<NIC %s:vm:%s network:%s>" % (self.id, self.machine_id,
self.network_id)
return u"<NIC %s:vm:%s network:%s>" % \
(self.id, self.machine_id, self.network_id)
@property
def backend_uuid(self):
......@@ -1046,8 +1072,8 @@ class Volume(models.Model):
userid = models.CharField("Owner's UUID", max_length=100, null=False,
db_index=True)
size = models.IntegerField("Volume size in GB", null=False)
disk_template = models.CharField('Disk template', max_length=32,
null=False)
volume_type = models.ForeignKey(VolumeType, related_name="volumes",
on_delete=models.PROTECT, null=False)
delete_on_termination = models.BooleanField("Delete on Server Termination",
default=True, null=False)
......@@ -1055,8 +1081,6 @@ class Volume(models.Model):
source = models.CharField(max_length=128, null=True)
origin = models.CharField(max_length=128, null=True)
# TODO: volume_type should be foreign key to VolumeType model
volume_type = None
deleted = models.BooleanField("Deleted", default=False, null=False)
# Datetime fields
created = models.DateTimeField(auto_now_add=True)
......@@ -1105,17 +1129,6 @@ class Volume(models.Model):
else:
return None
@property
def template(self):
return self.disk_template.split("_")[0]
@property
def provider(self):
if "_" in self.disk_template:
return self.disk_template.split("_", 1)[1]
else:
return None
@staticmethod
def prefix_source(source_id, source_type):
if source_type == "volume":
......
......@@ -14,6 +14,7 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import factory
from factory.fuzzy import FuzzyChoice
from synnefo.db import models
from random import choice
from string import letters, digits
......@@ -42,13 +43,23 @@ def random_string(x):
return ''.join([choice(digits + letters) for i in range(x)])
class VolumeTypeFactory(factory.DjangoModelFactory):
FACTORY_FOR = models.VolumeType
FACTORY_DJANGO_GET_OR_CREATE = ("disk_template",)
name = factory.Sequence(prefix_seq("vtype"))
disk_template = FuzzyChoice(
choices=["file", "plain", "drbd", "ext_archipelago"]
)
deleted = False
class FlavorFactory(factory.DjangoModelFactory):
FACTORY_FOR = models.Flavor