cyclades: Implement volumes and snapshots

First implementation of volumes and snapshots.

snf-cyclades-app/synnefo/logic/backend.py

@@ -1002,6 +1002,48 @@ def set_firewall_profile(vm, profile, nic):
     return None
 
 
+def attach_volume(vm, volume, depends=[]):
+    log.debug("Attaching volume %s to vm %s", vm, volume)
+
+    disk = {"size": volume.size,
+            "name": volume.backend_volume_uuid,
+            "volume_name": volume.backend_volume_uuid}
+    if volume.source_volume_id is not None:
+        disk["origin"] = volume.source_volume.backend_volume_uuid
+    elif volume.source_snapshot is not None:
+        disk["origin"] = volume.source_snapshot["checksum"]
+    elif volume.source_image is not None:
+        disk["origin"] = volume.source_image["checksum"]
+
+    kwargs = {
+        "instance": vm.backend_vm_id,
+        "disks": [("add", "-1", disk)],
+        "depends": depends,
+    }
+    if vm.backend.use_hotplug():
+        kwargs["hotplug"] = True
+    if settings.TEST:
+        kwargs["dry_run"] = True
+
+    with pooled_rapi_client(vm) as client:
+        return client.ModifyInstance(**kwargs)
+
+
+def detach_volume(vm, volume):
+    log.debug("Removing volume %s from vm %s", volume, vm)
+    kwargs = {
+        "instance": vm.backend_vm_id,
+        "disks": [("remove", volume.backend_volume_uuid, {})],
+    }
+    if vm.backend.use_hotplug():
+        kwargs["hotplug"] = True
+    if settings.TEST:
+        kwargs["dry_run"] = True
+
+    with pooled_rapi_client(vm) as client:
+        return client.ModifyInstance(**kwargs)
+
+
 def get_instances(backend, bulk=True):
     with pooled_rapi_client(backend) as c:
         return c.GetInstances(bulk=bulk)

snf-cyclades-app/synnefo/volume/snapshots.py

+import datetime
+from django.utils import simplejson as json
+from django.db import transaction
+from snf_django.lib.api import faults
+from snf_django.lib.api.utils import isoformat
+from synnefo.plankton.utils import image_backend
+from synnefo.logic import backend
+from synnefo.volume import util
+
+
+SNAPSHOTS_CONTAINER = "snapshots"
+SNAPSHOTS_DOMAIN = "plankton"
+SNAPSHOTS_PREFIX = "plankton:"
+SNAPSHOTS_TYPE = "application/octet-stream"
+SNAPSHOTS_MAPFILE_PREFIX = "archip:"
+
+
+@transaction.commit_on_success
+def create(user_id, volume, name, description, metadata, force=False):
+
+    if volume.machine is None:
+        raise faults.BadRequest("Can not snapshot detached volume!")
+
+    volume.snapshot_counter += 1
+    volume.save()
+
+    snapshot_metadata = {}
+    snapshot_metadata[SNAPSHOTS_PREFIX + "name"] = description
+    snapshot_metadata[SNAPSHOTS_PREFIX + "description"] = description
+    snapshot_metadata[SNAPSHOTS_PREFIX + "metadata"] = json.dumps(metadata)
+    snapshot_metadata[SNAPSHOTS_PREFIX + "volume_id"] = volume.id
+    snapshot_metadata[SNAPSHOTS_PREFIX + "status"] = "CREATING"
+    #XXX: just to work
+    snapshot_metadata[SNAPSHOTS_PREFIX + "is_snapshot"] = True
+
+    snapshot_name = generate_snapshot_name(volume)
+    mapfile = SNAPSHOTS_MAPFILE_PREFIX + snapshot_name
+
+    with image_backend(user_id) as pithos_backend:
+        # move this to plankton backend
+        snapshot_uuid = pithos_backend.backend.register_object_map(
+            user=user_id,
+            account=user_id,
+            container=SNAPSHOTS_CONTAINER,
+            name=name,
+            size=volume.size,
+            type=SNAPSHOTS_TYPE,
+            mapfile=mapfile,
+            meta=snapshot_metadata,
+            replace_meta=False,
+            permissions=None)
+            #checksum=None,
+
+    backend.snapshot_instance(volume.machine, snapshot_name=snapshot_uuid)
+
+    snapshot = util.get_snapshot(user_id, snapshot_uuid)
+
+    return snapshot
+
+
+def generate_snapshot_name(volume):
+    time = isoformat(datetime.datetime.now())
+    return "snf-snapshot-of-volume-%s-%s-%s" % (volume.id,
+                                                volume.snapshot_counter, time)
+
+
+@transaction.commit_on_success
+def delete(snapshot):
+    user_id = snapshot["owner"]
+    with image_backend(user_id) as pithos_backend:
+        pithos_backend.delete_snapshot(snapshot["uuid"])
+    return snapshot

snf-cyclades-app/synnefo/volume/util.py

+from synnefo.db import models
+from snf_django.lib.api import faults
+from synnefo.api.util import get_image_dict, get_vm
+
+
+def get_volume(user_id, volume_id, for_update=False,
+               exception=faults.ItemNotFound):
+    volumes = models.Volume.objects
+    if for_update:
+        volumes = volumes.select_for_update()
+    try:
+        return volumes.get(id=volume_id, userid=user_id)
+    except models.Volume.DoesNotExist:
+        raise exception("Volume %s not found" % volume_id)
+
+
+def get_snapshot(user_id, snapshot_id, exception=faults.ItemNotFound):
+    try:
+        return get_image_dict(snapshot_id, user_id)
+    except faults.ItemNotFound:
+        raise exception("Snapshot %s not found" % snapshot_id)
+
+
+def get_image(user_id, image_id, exception=faults.ItemNotFound):
+    try:
+        return get_image_dict(image_id, user_id)
+    except faults.ItemNotFound:
+        raise exception("Image %s not found" % image_id)
+
+
+def get_server(user_id, server_id, for_update=False,
+               exception=faults.ItemNotFound):
+    try:
+        return get_vm(server_id, user_id, for_update=for_update,
+                      non_deleted=True, non_suspended=True)
+    except faults.ItemNotFound:
+        raise exception("Server %s not found" % server_id)

snf-cyclades-app/synnefo/volume/views.py

+# Copyright 2013 GRNET S.A. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or
+# without modification, are permitted provided that the following
+# conditions are met:
+#
+#   1. Redistributions of source code must retain the above
+#      copyright notice, this list of conditions and the following
+#      disclaimer.
+#
+#   2. Redistributions in binary form must reproduce the above
+#      copyright notice, this list of conditions and the following
+#      disclaimer in the documentation and/or other materials
+#      provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
+# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
+# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+#
+# The views and conclusions contained in the software and
+# documentation are those of the authors and should not be
+# interpreted as representing official policies, either expressed
+# or implied, of GRNET S.A.
+
+from itertools import ifilter
+from logging import getLogger
+from django.http import HttpResponse
+from django.utils import simplejson as json
+
+import datetime
+from dateutil.parser import parse as date_parse
+
+from snf_django.lib import api
+from snf_django.lib.api import faults, utils
+
+from synnefo.volume import volumes, snapshots, util
+from synnefo.db.models import Volume
+from synnefo.plankton.utils import image_backend
+log = getLogger('synnefo.volume')
+
+
+def display_null_field(field):
+    if field is None:
+        return None
+    else:
+        str(field)
+
+
+def volume_to_dict(volume, detail=True):
+    data = {
+        "id": str(volume.id),
+        "name": display_null_field(volume.name),
+        # TODO: Links!
+        "links": "",
+    }
+    if detail:
+        details = {
+            "status": volume.status.lower(),
+            "size": volume.size,
+            "description": volume.description,
+            "created_at": utils.isoformat(volume.created),
+            "metadata": dict((m.key, m.value) for m in volume.metadata.all()),
+            "snapshot_id": display_null_field(volume.source_snapshot_id),
+            "source_volid": display_null_field(volume.source_volume_id),
+            "image_id": display_null_field(volume.source_image_id),
+            "attachments": get_volume_attachments(volume),
+            # TODO:
+            "volume_type": None,
+            #"availabilit_zone": None,
+            #"bootable": None,
+            #"os-vol-tenant-attr:tenant_id": None,
+            #"os-vol-host-attr:host": None,
+            #"os-vol-mig-status-attr:name_id": None,
+            #"os-vol-mig-status-attr:migstat": None,
+        }
+        data.update(details)
+    return data
+
+
+def get_volume_attachments(volume):
+    if volume.machine_id is None:
+        return []
+    else:
+        return [{"server_id": volume.machine_id,
+                 "volume_id": volume.id,
+                 "device_index": volume.index}]
+
+
+@api.api_method(http_method="POST", user_required=True, logger=log)
+def create_volume(request):
+    """Create a new Volume."""
+
+    req = utils.get_request_dict(request)
+    log.debug("create_volume %s", req)
+
+    user_id = request.user_uniq
+    # Get and validate 'name' parameter
+    # TODO: auto generate name
+    name = req.get("name", None)
+    if name is None:
+        raise faults.BadRequest("Volume 'name' is needed.")
+    # Get and validate 'size' parameter
+    size = req.get("size")
+    if size is None:
+        raise faults.BadRequest("Volume 'size' is needed.")
+    try:
+        size = int(size)
+        if size <= 0:
+            raise ValueError
+    except ValueError:
+        raise faults.BadRequest("Volume 'size' needs to be a positive integer"
+                                " value. '%s' cannot be accepted." % size)
+
+    # TODO: Fix volume type, validate, etc..
+    volume_type = req.get("volume_type", None)
+
+    # Optional parameters
+    description = req.get("description", "")
+    metadata = req.get("metadata", {})
+    if not isinstance(metadata, dict):
+        msg = "Volume 'metadata' needs to be a dictionary of key-value pairs."\
+              " '%s' can not be accepted." % metadata
+        raise faults.BadRequest(msg)
+
+    # Id of the volume to clone from
+    source_volume_id = req.get("source_volid")
+    # Id of the snapshot to create the volume from
+    source_snapshot_id = req.get("snapshot_id")
+    # Reference to an Image stored in Glance
+    source_image_id = req.get("imageRef")
+    # TODO: Check that not all of them are used
+
+    server_id = req.get("server_id")
+    if server_id is None:
+        raise faults.BadRequest("Attribute 'server_id' is mandatory")
+
+    # Create the volume
+    volume = volumes.create(user_id=user_id, size=size, name=name,
+                            source_volume_id=source_volume_id,
+                            source_snapshot_id=source_snapshot_id,
+                            source_image_id=source_image_id,
+                            volume_type=volume_type, description=description,
+                            metadata=metadata, server_id=server_id)
+
+    # Render response
+    data = json.dumps(dict(volume=volume_to_dict(volume, detail=False)))
+    return HttpResponse(data, status=200)
+
+
+@api.api_method(http_method="GET", user_required=True, logger=log)
+def list_volumes(request, detail=False):
+    log.debug('list_volumes detail=%s', detail)
+    volumes = Volume.objects.filter(userid=request.user_uniq)
+
+    since = utils.isoparse(request.GET.get('changes-since'))
+    if since:
+        volumes = volumes.filter(updated__gte=since)
+        if not volumes:
+            return HttpResponse(status=304)
+    else:
+        volumes = volumes.filter(deleted=False)
+
+    volumes = [volume_to_dict(v, detail) for v in volumes.order_by("id")]
+
+    data = json.dumps({'volumes': volumes})
+    return HttpResponse(data, content_type="application/json", status=200)
+
+
+@api.api_method(http_method="DELETE", user_required=True, logger=log)
+def delete_volume(request, volume_id):
+    log.debug("delete_volume volume_id: %s", volume_id)
+
+    volume = util.get.volume(request.user_uniq, volume_id, for_update=True)
+    volumes.delete(volume)
+
+    return HttpResponse(status=202)
+
+
+@api.api_method(http_method="GET", user_required=True, logger=log)
+def get_volume(request, volume_id):
+    log.debug('get_volume volume_id: %s', volume_id)
+
+    volume = util.get.volume(request.user_uniq, volume_id)
+
+    data = json.dumps({'volume': volume_to_dict(volume, detail=True)})
+    return HttpResponse(data, content_type="application/json", status=200)
+
+
+@api.api_method(http_method="PUT", user_required=True, logger=log)
+def update_volume(request, volume_id):
+    req = utils.get_request_dict(request)
+    log.debug('update_volume volume_id: %s, request: %s', volume_id, req)
+
+    volume = util.get.volume(request.user_uniq, volume_id, for_update=True)
+
+    new_name = req.get("name")
+    description = req.get("description")
+
+    if new_name is None and description is None:
+        raise faults.BadRequest("Nothing to update.")
+
+    if new_name is not None:
+        volume = volumes.rename(volume, new_name)
+    if description is not None:
+        volume = volumes.update_description(volume, description)
+    data = json.dumps({'volume': volume_to_dict(volume, detail=True)})
+    return HttpResponse(data, content_type="application/json", status=200)
+
+
+def snapshot_to_dict(snapshot, detail=True):
+    owner = snapshot["owner"]
+    status = snapshot["status"]
+    progress = snapshot["progress"]
+    data = {
+        "id": snapshot["uuid"],
+        "size": int(snapshot["size"]) >> 30,  # gigabytes
+        "name": snapshot["name"],
+        "description": snapshot["description"],
+        "status": status,
+        "user_id": owner,
+        "tenant_id": owner,
+        "os-extended-snapshot-attribute:progress": progress,
+        #"os-extended-snapshot-attribute:project_id": project,
+        "created_at": utils.isoformat(date_parse(snapshot["created_at"])),
+        "metadata": snapshot.get("metadata", {}),
+        "volume_id": snapshot.get("volume_id"),
+        "links": "",  # TODO fix links
+    }
+    return data
+
+
+@api.api_method(http_method="POST", user_required=True, logger=log)
+def create_snapshot(request):
+    """Create a new Snapshot."""
+
+    req = utils.get_request_dict(request)
+    log.debug("create_snapshot %s", req)
+
+    user_id = request.user_uniq
+    # Get and validate 'name' parameter
+    # TODO: auto generate name
+    metadata = req.get("metadata", {})
+    if not isinstance(metadata, dict):
+        msg = "Snapshot 'metadata' needs to be a dictionary of key-value"\
+              " pairs. '%s' can not be accepted." % metadata
+        raise faults.BadRequest(msg)
+
+    volume_id = req.get("volume_id", None)
+    if volume_id is None:
+        raise faults.BadRequest("'volume_id' attribute is missing.")
+    volume = util.get_volume(user_id, volume_id, for_update=True,
+                             exception=faults.BadRequest)
+
+    name = req.get("name", None)
+    if name is None:
+        name = "snapshot_volume_%s_%s" %\
+            (volume.id, str(datetime.datetime.now()))
+    description = req.get("description", "")
+
+    # TODO: What to do with force ?
+    force = req.get("force", False)
+    if not isinstance(force, bool):
+        raise faults.BadRequest("Invalid value for 'force' attribute.")
+
+    snapshot = snapshots.create(user_id=user_id, volume=volume, name=name,
+                                description=description, metadata=metadata,
+                                force=force)
+
+    # Render response
+    data = json.dumps(dict(snapshot=snapshot_to_dict(snapshot, detail=False)))
+    return HttpResponse(data, status=200)  # TOO: Maybe 202 ?
+
+
+@api.api_method(http_method="GET", user_required=True, logger=log)
+def list_snapshots(request, detail=False):
+    log.debug('list_snapshots detail=%s', detail)
+    since = utils.isoparse(request.GET.get('changes-since'))
+    with image_backend(request.user_uniq) as backend:
+        snapshots = backend.list_snapshots()
+        if since:
+            updated_since = lambda snap:\
+                date_parse(snap["updated_at"]) >= since
+            snapshots = ifilter(updated_since, snapshots)
+            if not snapshots:
+                return HttpResponse(status=304)
+
+    snapshots = sorted(snapshots, key=lambda x: x['id'])
+    snapshots_dict = [snapshot_to_dict(snapshot, detail)
+                      for snapshot in snapshots]
+
+    data = json.dumps(dict(snapshots=snapshots_dict))
+
+    return HttpResponse(data, status=200)
+
+
+@api.api_method(http_method="DELETE", user_required=True, logger=log)
+def delete_snapshot(request, snapshot_id):
+    log.debug("delete_snapshot snapshot_id: %s", snapshot_id)
+
+    snapshot = util.get_snapshot(request.user_uniq, snapshot_id)
+    snapshots.delete(snapshot)
+
+    return HttpResponse(status=202)
+
+
+@api.api_method(http_method="GET", user_required=True, logger=log)
+def get_snapshot(request, snapshot_id):
+    log.debug('get_snapshot snapshot_id: %s', snapshot_id)
+
+    snapshot = util.get_snapshot(request.user_uniq, snapshot_id)
+    data = json.dumps({'snapshot': snapshot_to_dict(snapshot, detail=True)})
+    return HttpResponse(data, content_type="application/json", status=200)
+
+
+@api.api_method(http_method="PUT", user_required=True, logger=log)
+def update_snapshot(request, snapshot_id):
+    req = utils.get_request_dict(request)
+    log.debug('update_snapshot snapshot_id: %s, request: %s', snapshot_id, req)
+    snapshot = util.get_snapshot(request.user_uniq, snapshot_id)
+    # TODO
+    #snapshot.name = req.get("name", snapshot.name)
+    #snapshot.description = req.get("description", snapshot.description)
+    #snapshot.save()
+    data = json.dumps({'snapshot': snapshot_to_dict(snapshot, detail=True)})
+    return HttpResponse(data, content_type="application/json", status=200)

snf-cyclades-app/synnefo/volume/volumes.py

+import logging
+
+from django.db import transaction
+from synnefo.db.models import Volume
+from snf_django.lib.api import faults
+from synnefo.volume import util
+from synnefo.logic import backend
+
+log = logging.getLogger(__name__)
+
+
+@transaction.commit_on_success
+def create(user_id, size, server_id, name=None, description=None,
+           source_volume_id=None, source_snapshot_id=None,
+           source_image_id=None, metadata=None):
+
+    if server_id is None:
+        raise faults.BadRequest("Volume must be attached to server")
+    server = util.get_server(user_id, server_id, for_update=True,
+                             exception=faults.BadRequest)
+
+    # Assert that not more than one source are used
+    sources = filter(lambda x: x is not None,
+                     [source_volume_id, source_snapshot_id, source_image_id])
+    if len(sources) > 1:
+        raise faults.BadRequest("Volume can not have more than one source!")
+
+    source_volume = None
+    if source_volume_id is not None:
+        source_volume = util.get_volume(user_id, source_volume_id,
+                                        for_update=True,
+                                        exception=faults.BadRequest)
+    source_snapshot = None
+    if source_snapshot_id is not None:
+        source_snapshot = util.get_snapshot(user_id, source_snapshot_id,
+                                            exception=faults.BadRequest)
+    source_image = None
+    if source_image_id is not None:
+        source_image = util.get_image(user_id, source_image_id,
+                                      exception=faults.BadRequest)
+
+    volume = Volume.objects.create(userid=user_id,
+                                   size=size,
+                                   name=name,
+                                   machine=server,
+                                   description=description,
+                                   source_volume=source_volume,
+                                   source_image_id=source_image_id,
+                                   source_snapshot_id=source_snapshot_id,
+                                   #volume_type=volume_type,
+                                   status="CREATING")
+
+    if metadata is not None:
+        for meta_key, meta_val in metadata.items():
+            volume.metadata.create(key=meta_key, value=meta_val)
+
+    # Annote volume with snapshot/image information
+    volume.source_snapshot = source_snapshot
+    volume.source_image = source_image
+
+    # Create the disk in the backend
+    volume.backendjobid = backend.attach_volume(server, volume)
+    volume.save()
+
+    return volume
+
+
+@transaction.commit_on_success
+def delete(volume):
+    if volume.machine_id is not None:
+        raise faults.BadRequest("Volume %s is still in use by server %s"
+                                % (volume.id, volume.machine_id))
+    volume.deleted = True
+    volume.save()
+
+    log.info("Deleted volume %s", volume)
+
+    return volume
+
+
+@transaction.commit_on_success
+def rename(volume, new_name):
+    volume.name = new_name
+    volume.save()
+    return volume
+
+
+@transaction.commit_on_success
+def update_description(volume, new_description):
+    volume.description = new_description
+    volume.save()
+    return volume