Commit d385a174 authored by Iustin Pop's avatar Iustin Pop
Browse files

Increase the lock timeouts before we block-acquire

This has been observed to cause problems on real clusters via the
following mechanism:

- a long job (e.g. a replace-disks) is keeping an exclusive lock on an
  instance
- the watcher starts and submits its query instances opcode which
  wants shared locks for all instances
- after about an hour, the watcher job falls back to blocking acquire,
  after having acquired all other locks
- any instance opcode that wants an exclusive lock for an instance
  cannot start until the watcher has finished, even though there's no
  actual operation on that instance

In order to alleviate this problem, we simply increase the max timeout
until lock acquires are sent back to either blocking acquire or
priority increase. The timeout is computed such that we wait ~10 hours
(instead of one) for this to happen, which should be within the
maximum lifetime of a reasonable opcode on a healthy cluster. The
timeout also means that priority increases will happen every half hour.

...
parent fe295df3
......@@ -416,10 +416,18 @@ EXPORT_MODES = frozenset([
EXPORT_MODE_REMOTE,
])
# lock recalculate mode
# Lock recalculate mode
LOCKS_REPLACE = 'replace'
LOCKS_APPEND = 'append'
# Lock timeout (sum) before we should go into blocking acquire (still
# can be reset by priority change); computed as max time (10 hours)
# before we should actually go into blocking acquire given that we
# start from default priority level; in seconds
LOCK_ATTEMPTS_TIMEOUT = 10 * 3600 / 20.0
LOCK_ATTEMPTS_MAXWAIT = 15.0
LOCK_ATTEMPTS_MINWAIT = 1.0
# instance creation modes
INSTANCE_CREATE = "create"
INSTANCE_IMPORT = "import"
......
......@@ -55,23 +55,23 @@ def _CalculateLockAttemptTimeouts():
"""Calculate timeouts for lock attempts.
"""
result = [1.0]
result = [constants.LOCK_ATTEMPTS_MINWAIT]
running_sum = result[0]
# Wait for a total of at least 150s before doing a blocking acquire
while sum(result) < 150.0:
# Wait for a total of at least LOCK_ATTEMPTS_TIMEOUT before doing a
# blocking acquire
while running_sum < constants.LOCK_ATTEMPTS_TIMEOUT:
timeout = (result[-1] * 1.05) ** 1.25
# Cap timeout at 10 seconds. This gives other jobs a chance to run
# even if we're still trying to get our locks, before finally moving
# to a blocking acquire.
if timeout > 10.0:
timeout = 10.0
elif timeout < 0.1:
# Lower boundary for safety
timeout = 0.1
# Cap max timeout. This gives other jobs a chance to run even if
# we're still trying to get our locks, before finally moving to a
# blocking acquire.
timeout = min(timeout, constants.LOCK_ATTEMPTS_MAXWAIT)
# And also cap the lower boundary for safety
timeout = max(timeout, constants.LOCK_ATTEMPTS_MINWAIT)
result.append(timeout)
running_sum += timeout
return result
......
......@@ -26,6 +26,10 @@ import unittest
from ganeti import mcpu
from ganeti import opcodes
from ganeti.constants import \
LOCK_ATTEMPTS_TIMEOUT, \
LOCK_ATTEMPTS_MAXWAIT, \
LOCK_ATTEMPTS_MINWAIT
import testutils
......@@ -33,8 +37,8 @@ import testutils
class TestLockAttemptTimeoutStrategy(unittest.TestCase):
def testConstants(self):
tpa = mcpu.LockAttemptTimeoutStrategy._TIMEOUT_PER_ATTEMPT
self.assert_(len(tpa) > 10)
self.assert_(sum(tpa) >= 150.0)
self.assert_(len(tpa) > LOCK_ATTEMPTS_TIMEOUT / LOCK_ATTEMPTS_MAXWAIT)
self.assert_(sum(tpa) >= LOCK_ATTEMPTS_TIMEOUT)
def testSimple(self):
strat = mcpu.LockAttemptTimeoutStrategy(_random_fn=lambda: 0.5,
......@@ -45,8 +49,8 @@ class TestLockAttemptTimeoutStrategy(unittest.TestCase):
timeout = strat.NextAttempt()
self.assert_(timeout is not None)
self.assert_(timeout <= 10.0)
self.assert_(timeout >= 0.0)
self.assert_(timeout <= LOCK_ATTEMPTS_MAXWAIT)
self.assert_(timeout >= LOCK_ATTEMPTS_MINWAIT)
self.assert_(prev is None or timeout >= prev)
prev = timeout
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment