Commit 9b94fc30 authored by Giorgos Korfiatis's avatar Giorgos Korfiatis

cyclades: Do not save stale objects after a commit

Call to issue_and_accept_commission saves the resource object (eg a vm)
and commits the transaction. This has the side-effect that the resource
object becomes stale: its lock is released and another thread may acquire
it and update the db record. It is thus not safe to further update and then
save the stale object, since this would override others' changes. Instead,
lock and fetch the object again, where needed.
parent 8101def7
......@@ -393,6 +393,7 @@ def do_create_server(userid, name, password, flavor, image, metadata={},
transaction.commit()
try:
vm = VirtualMachine.objects.select_for_update().get(id=vm.id)
jobID = create_instance(vm, nic, flavor, image)
# At this point the job is enqueued in the Ganeti backend
vm.backendjobid = jobID
......
......@@ -74,6 +74,13 @@ def process_op_status(vm, etime, jobid, opcode, status, logmsg, nics=None):
vm.backendopcode = opcode
vm.backendlogmsg = logmsg
# Update backendtime only for jobs that have been successfully completed,
# since only these jobs update the state of the VM. Else a "race condition"
# may occur when a successful job (e.g. OP_INSTANCE_REMOVE) completes
# before an error job and messages arrive in reversed order.
if status == 'success':
vm.backendtime = etime
# Notifications of success change the operating state
state_for_success = VirtualMachine.OPER_STATE_FROM_OPCODE.get(opcode, None)
if status == 'success' and state_for_success is not None:
......@@ -100,13 +107,10 @@ def process_op_status(vm, etime, jobid, opcode, status, logmsg, nics=None):
# Issue and accept commission to Quotaholder
if not already_deleted:
quotas.issue_and_accept_commission(vm, delete=True)
# Update backendtime only for jobs that have been successfully completed,
# since only these jobs update the state of the VM. Else a "race condition"
# may occur when a successful job (e.g. OP_INSTANCE_REMOVE) completes
# before an error job and messages arrive in reversed order.
if status == 'success':
vm.backendtime = etime
# the above has already saved the object and committed;
# a second save would override others' changes, since the
# object is now unlocked
return
vm.save()
......@@ -296,6 +300,10 @@ def update_network_state(network):
# Issue commission
if network.userid:
quotas.issue_and_accept_commission(network, delete=True)
# the above has already saved the object and committed;
# a second save would override others' changes, since the
# object is now unlocked
return
elif not network.public:
log.warning("Network %s does not have an owner!", network.id)
network.save()
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment