Merge branch 'hotfix-0.14.10' into develop

Conflicts:
	Changelog
	snf-cyclades-app/synnefo/api/servers.py
	snf-cyclades-app/synnefo/logic/backend.py
	snf-cyclades-app/synnefo/logic/callbacks.py
	snf-cyclades-app/synnefo/logic/management/commands/reconcile-networks.py
	snf-cyclades-app/synnefo/logic/reconciliation.py
	snf-cyclades-gtools/synnefo/ganeti/eventd.py
	version

Changelog

@@ -108,6 +108,23 @@ objects by domain attribute. This is used by Plankton for listing VM images.
 
 * Enforce container-level atomicity in (most) Pithos API calls.
 
+.. _Changelog-0.14.10:
+
+v0.14.10
+=======
+
+Released: UNRELEASED
+
+Cyclades
+-------
+
+* This is the first release to support Ganeti 2.8. Support for older versions
+  of Ganeti is dropped.
+* Use Ganeti opportunistic locking to achive parallelized instance creations
+  in the same backend. Add setting 'GANETI_USE_OPPORTUNISTIC_LOCKING' to
+  enable the use of this feature.
+* Fix warning message while getting object permissions to appear only when
+  path is None and the object has permissions
 
 .. _Changelog-0.14.9:
 

snf-cyclades-app/conf/20-snf-cyclades-app-backend.conf

@@ -33,6 +33,10 @@
 ## a network. This requires qemu-kvm=1.0.
 #GANETI_USE_HOTPLUG = False
 #
+## If True, Ganeti will try to allocate new instances only on nodes that are
+## not already locked. This might result in slightly unbalanced clusters.
+#GANETI_USE_OPPORTUNISTIC_LOCKING = True
+#
 ## This module implements the strategy for allocating a vm to a backend
 #BACKEND_ALLOCATOR_MODULE = "synnefo.logic.allocators.default_allocator"
 ## Refresh backend statistics timeout, in minutes, used in backend allocation

snf-cyclades-app/synnefo/app_settings/default/backend.py

@@ -33,6 +33,10 @@ GANETI_CREATEINSTANCE_KWARGS = {
 # a network. This requires qemu-kvm=1.0.
 GANETI_USE_HOTPLUG = True
 
+# If True, Ganeti will try to allocate new instances only on nodes that are
+# not already locked. This might result in slightly unbalanced clusters.
+GANETI_USE_OPPORTUNISTIC_LOCKING = True
+
 # This module implements the strategy for allocating a vm to a backend
 BACKEND_ALLOCATOR_MODULE = "synnefo.logic.allocators.default_allocator"
 # Refresh backend statistics timeout, in minutes, used in backend allocation

snf-cyclades-app/synnefo/logic/backend.py

@@ -665,7 +665,7 @@ def create_instance(vm, nics, flavor, image):
         'img_format': image['format']}
 
     # Use opportunistic locking
-    kw['opportunistic_locking'] = True
+    kw['opportunistic_locking'] = settings.GANETI_USE_OPPORTUNISTIC_LOCKING
 
     # Defined in settings.GANETI_CREATEINSTANCE_KWARGS
     # kw['hvparams'] = dict(serial_console=False)

snf-cyclades-app/synnefo/logic/callbacks.py

@@ -35,7 +35,8 @@ import json
 from functools import wraps
 
 from django.db import transaction
-from synnefo.db.models import Backend, VirtualMachine, Network, BackendNetwork
+from synnefo.db.models import (Backend, VirtualMachine, Network,
+                               BackendNetwork, pooled_rapi_client)
 from synnefo.logic import utils, backend as backend_mod
 
 from synnefo.lib.utils import merge_time
@@ -90,8 +91,8 @@ def instance_from_msg(func):
         except VirtualMachine.DoesNotExist:
             log.error("VM for instance %s with id %d not found in DB.",
                       msg['instance'], vm_id)
-        except Network.InvalidBackendIdError, Network.DoesNotExist:
-            log.error("Invalid message, cannot find network. msg: %s", msg)
+        except (Network.InvalidBackendIdError, Network.DoesNotExist) as e:
+            log.error("Invalid message, can not find network. msg: %s", msg)
     return wrapper
 
 
@@ -171,11 +172,40 @@ def update_db(vm, msg, event_time):
         log.error("Message is of unknown type %s.", msg['type'])
         return
 
+    operation = msg["operation"]
+    status = msg["status"]
+    jobID = msg["jobId"]
+    logmsg = msg["logmsg"]
     nics = msg.get("instance_nics", None)
     job_fields = msg.get("job_fields", {})
-    backend_mod.process_op_status(vm, event_time, msg['jobId'],
-                                  msg['operation'], msg['status'],
-                                  msg['logmsg'], nics=nics,
+
+    # Special case: OP_INSTANCE_CREATE with opportunistic locking may fail
+    # if all Ganeti nodes are already locked. Retry the job without
+    # opportunistic locking..
+    if (operation == "OP_INSTANCE_CREATE" and status == "error" and
+       job_fields.get("opportunistic_locking", False)):
+        if vm.backendjobid != jobID:  # The job has already been retried!
+            return
+        # Remove extra fields
+        [job_fields.pop(f) for f in ("OP_ID", "reason")]
+        name = job_fields.pop("name", job_fields.pop("instance_name"))
+        # Turn off opportunistic locking before retrying the job
+        job_fields["opportunistic_locking"] = False
+        with pooled_rapi_client(vm) as c:
+            jobID = c.CreateInstance(name=name, **job_fields)
+        # Update the VM fields
+        vm.backendjobid = jobID
+        # Update the task_job_id for commissions
+        vm.task_job_id = jobID
+        vm.backendjobstatus = None
+        vm.save()
+        log.info("Retrying failed creation of instance '%s' without"
+                 " opportunistic locking. New job ID: '%s'", name, jobID)
+        return
+
+    backend_mod.process_op_status(vm, event_time, jobID,
+                                  operation, status,
+                                  logmsg, nics=nics,
                                   job_fields=job_fields)
 
     log.debug("Done processing ganeti-op-status msg for vm %s.",

snf-cyclades-app/synnefo/plankton/backend.py

@@ -219,9 +219,12 @@ class ImageBackend(object):
             self.backend.get_object_permissions(self.user, account, container,
                                                 name)
 
-        if path is None:
-            logger.warning("Image '%s' got permissions from None path",
-                           image_url)
+        if path is None and permissions != {}:
+            logger.warning("Image '%s' got permissions '%s' from 'None' path.",
+                           image_url, permissions)
+            raise Exception("Database Inconsistency Error:"
+                            " Image '%s' got permissions from 'None' path." %
+                            image_url)
 
         return permissions
 

snf-cyclades-gtools/synnefo/ganeti/eventd.py

@@ -241,7 +241,13 @@ class JobFileHandler(pyinotify.ProcessEvent):
             if op.status == "success":
                 msg["result"] = op.result
 
+            if op_id == "OP_INSTANCE_CREATE" and op.status == "error":
+                # In case an instance creation fails send the job input
+                # so that the job can be retried if needed.
+                msg["job_fields"] = op.Serialize()["input"]
+
             msg = json.dumps(msg)
+
             self.logger.debug("Delivering msg: %s (key=%s)", msg, routekey)
 
             # Send the message to RabbitMQ

version

 # This is a comment!
-0.14.9next
+0.14.10next