From d2cd6944029153fec2888b65740b8989c65dd16b Mon Sep 17 00:00:00 2001
From: Iustin Pop <iustin@google.com>
Date: Tue, 4 Jan 2011 10:59:05 +0100
Subject: [PATCH] RPC: mark jobqueue functions as URGENT
Recently, we've seen more and more cases of a specific breakage
pattern in Ganeti: master candidates which are semi-alive (as in, they
respond to ping, they can complete a TCP/SSL handshake, but otherwise
the root filesystem is broken) cause lots of confusion within masterd.
My analysis shows that waiting up to 5 minutes for a reply from such a
broken master candidate is too long, and this long wait breaks other
timeouts (e.g. the Luxi timeout), making standard recovery from this
situation very hard. It's much easier to kill the master daemon, edit
manually the config file and mark the node as regular, then restart
the master daemon.
The proposal is therefore to reduce the timeout for the job queue
functions to TMO_URGENT (1 minute), which should be more balanced
between a working but overloaded node and a broken node.
Signed-off-by: Iustin Pop <iustin@google.com>
Reviewed-by: Michael Hanselmann <hansmi@google.com>
---
lib/rpc.py | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/lib/rpc.py b/lib/rpc.py
index a4298be33..609549ed7 100644
--- a/lib/rpc.py
+++ b/lib/rpc.py
@@ -1,7 +1,7 @@
#
#
-# Copyright (C) 2006, 2007, 2008, 2009, 2010 Google Inc.
+# Copyright (C) 2006, 2007, 2008, 2009, 2010, 2011 Google Inc.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@@ -1401,7 +1401,7 @@ class RpcRunner(object):
[old_file_storage_dir, new_file_storage_dir])
@classmethod
- @_RpcTimeout(_TMO_FAST)
+ @_RpcTimeout(_TMO_URGENT)
def call_jobqueue_update(cls, node_list, address_list, file_name, content):
"""Update job queue.
@@ -1423,7 +1423,7 @@ class RpcRunner(object):
return cls._StaticSingleNodeCall(node, "jobqueue_purge", [])
@classmethod
- @_RpcTimeout(_TMO_FAST)
+ @_RpcTimeout(_TMO_URGENT)
def call_jobqueue_rename(cls, node_list, address_list, rename):
"""Rename a job queue file.
--
GitLab