Commit f87b405e authored by Michael Hanselmann's avatar Michael Hanselmann
Browse files

Add job queue size limit

A job queue with too many jobs can increase memory usage and/or make
the master daemon slow. The current limit is just an arbitrary number.
A "soft" limit for automatic job archival is prepared.

Reviewed-by: iustinp
parent 7167159a
......@@ -687,6 +687,9 @@ def FormatError(err):
elif isinstance(err, errors.JobQueueDrainError):
obuf.write("Failure: the job queue is marked for drain and doesn't"
" accept new requests\n")
elif isinstance(err, errors.JobQueueFull):
obuf.write("Failure: the job queue is full and doesn't accept new"
" job submissions until old jobs are archived\n")
elif isinstance(err, errors.GenericError):
obuf.write("Unhandled Ganeti error: %s" % msg)
elif isinstance(err, luxi.NoMasterError):
......
......@@ -366,6 +366,8 @@ JOB_QUEUE_VERSION_FILE = QUEUE_DIR + "/version"
JOB_QUEUE_SERIAL_FILE = QUEUE_DIR + "/serial"
JOB_QUEUE_ARCHIVE_DIR = QUEUE_DIR + "/archive"
JOB_QUEUE_DRAIN_FILE = QUEUE_DIR + "/drain"
JOB_QUEUE_SIZE_HARD_LIMIT = 5000
JOB_QUEUE_SIZE_SOFT_LIMIT = JOB_QUEUE_SIZE_HARD_LIMIT * 0.8
JOB_ID_TEMPLATE = r"\d+"
......
......@@ -253,6 +253,14 @@ class JobQueueDrainError(JobQueueError):
"""
class JobQueueFull(JobQueueError):
"""Job queue full error.
Raised when job queue size reached its hard limit.
"""
# errors should be added above
......
......@@ -941,6 +941,18 @@ class JobQueue(object):
"""
if self._IsQueueMarkedDrain():
raise errors.JobQueueDrainError()
# Check job queue size
size = len(self._ListJobFiles())
if size >= constants.JOB_QUEUE_SIZE_SOFT_LIMIT:
# TODO: Autoarchive jobs. Make sure it's not done on every job
# submission, though.
#size = ...
pass
if size >= constants.JOB_QUEUE_SIZE_HARD_LIMIT:
raise errors.JobQueueFull()
# Get job identifier
job_id = self._NewSerialUnlocked()
job = _QueuedJob(self, job_id, ops)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment