Reconciliation command documentation + config options

README.deploy

@@ -142,6 +142,35 @@ to Synnefo v0.3.
       VNCAUTHPROXY_CTRL_SOCKET.
 
 
+-Administration
+    * Reconciliation process: On certain occasions, such as a Ganeti or
+      RabbitMQ failure, the VM state in the system's database may differ from
+      that in the Ganeti installation. The reconciliation process is designed
+      to bring the system's database in sync with what Ganeti knows about
+      each VM.
+
+      The reconciliation process can be triggered for all VMs using the command
+
+          ./manage.py reconcile --all
+
+      It is advised, though not strictly necessary, to run the reconciliation
+      process periodically, through cron. To avoid overloading the Ganeti
+      master, the periodic reconciliation process takes a staggered approach
+      to updating the VMs, which is configured through the following
+      parameters:
+
+        * The settings.py parameter: RECONCILIATION_MIN, which specifies the
+        maximum time a VM can remain ``non-reconciled''. (default: 30 mins)
+
+        * The --interval option to the reconcile command, which declares the
+         interval time between reconciliation attempts (default: 1 min)
+
+      On each invocation of the reconcile command, the system will trigger a
+      reconciliation for ((num_all_vms/RECONCILIATION_MIN) * interval)
+      machines. Obviously the less the interval value and the more the
+      RECONCILIATION_MIN setting, the less load is going to be put on Ganeti.
+
+
 - OS Specific instructions
 
     * Debian Squeeze

conf/ci/pip-1.2.conf

@@ -9,4 +9,3 @@ south==0.7.1
 amqplib==0.6.1
 daemon==1.0
 pycrypto==2.1.0
-

logic/management/commands/reconcile.py

@@ -29,11 +29,11 @@
 #
 # Reconcile VM state - Management Script
 
-
-from django.core.management.base import NoArgsCommand
 from synnefo.db.models import VirtualMachine
 from django.conf import settings
 from datetime import datetime, timedelta
+from optparse import make_option
+from django.core.management.base import BaseCommand
 
 from synnefo.logic import amqp_connection
 from synnefo.logic.amqp_connection import AMQPError
@@ -41,20 +41,32 @@ from synnefo.logic.amqp_connection import AMQPError
 import json
 import sys
 
-class Command(NoArgsCommand):
+class Command(BaseCommand):
     help = 'Reconcile VM status with the backend'
 
-    def handle_noargs(self, **options):
+    option_list = BaseCommand.option_list +  (
+         make_option('--all', action='store_true', dest='all_vms', default=False,
+                     help='Run the reconciliation function for all VMs, now'),
+         make_option('--interval', action='store', dest='interval', default=1,
+                     help='Interval in minutes between reconciliations'),
+    )
+
+    def handle(self, all_vms = False, interval = 1, **options):
+        all =  VirtualMachine.objects.all()
 
-        now = datetime.now()
-        last_update = timedelta(minutes = settings.RECONCILIATION_MIN)
-        not_updated = VirtualMachine.objects \
+        if not all_vms:
+            now = datetime.now()
+            last_update = timedelta(minutes = settings.RECONCILIATION_MIN)
+            not_updated = VirtualMachine.objects \
                                     .filter(deleted = False) \
                                     .filter(suspended = False) \
                                     .filter(updated__lte = (now - last_update))
-        all =  VirtualMachine.objects.all()
 
-        to_update = all.count() / settings.RECONCILIATION_MIN
+            to_update = ((all.count() / settings.RECONCILIATION_MIN) * interval)
+        else:
+            to_update = all.count()
+            not_updated = all
+
         vm_ids = map(lambda x: x.id, not_updated[:to_update])
 
         for vmid in vm_ids :