From 58b22b6e77a87146ef7e6048b69112df3b62e9e6 Mon Sep 17 00:00:00 2001
From: Michael Hanselmann <hansmi@google.com>
Date: Thu, 18 Dec 2008 16:23:26 +0000
Subject: [PATCH] Use subdirectories for job queue archive
As it turned out, having many files in a single directory can be
very painful. With this patch, only 10'000 files are stored in a
directory for the job queue archive. With 10'000 directries, this
allows for up to 100 million jobs be archived without having large
numbers of files in a single directories. Not that it is realistic,
anyway.
Reviewed-by: ultrotter
---
lib/backend.py | 2 +-
lib/jqueue.py | 22 ++++++++++++++++++----
2 files changed, 19 insertions(+), 5 deletions(-)
diff --git a/lib/backend.py b/lib/backend.py
index 2d7f63a48..55aae5ddb 100644
--- a/lib/backend.py
+++ b/lib/backend.py
@@ -1990,7 +1990,7 @@ def JobQueueRename(old, new):
if not (_IsJobQueueFile(old) and _IsJobQueueFile(new)):
return False
- os.rename(old, new)
+ utils.RenameFile(old, new, mkdir=True)
return True
diff --git a/lib/jqueue.py b/lib/jqueue.py
index 373447607..084368eb3 100644
--- a/lib/jqueue.py
+++ b/lib/jqueue.py
@@ -49,6 +49,7 @@ from ganeti import rpc
JOBQUEUE_THREADS = 25
+JOBS_PER_ARCHIVE_DIRECTORY = 10000
class CancelJob(Exception):
@@ -714,7 +715,7 @@ class JobQueue(object):
@param new: the new name of the file
"""
- os.rename(old, new)
+ utils.RenameFile(old, new, mkdir=True)
names, addrs = self._GetNodeIp()
result = rpc.RpcRunner.call_jobqueue_rename(names, addrs, old, new)
@@ -741,6 +742,18 @@ class JobQueue(object):
return str(job_id)
+ @classmethod
+ def _GetArchiveDirectory(cls, job_id):
+ """Returns the archive directory for a job.
+
+ @type job_id: str
+ @param job_id: Job identifier
+ @rtype: str
+ @return: Directory name
+
+ """
+ return str(int(job_id) / JOBS_PER_ARCHIVE_DIRECTORY)
+
def _NewSerialUnlocked(self):
"""Generates a new job identifier.
@@ -774,8 +787,8 @@ class JobQueue(object):
"""
return os.path.join(constants.QUEUE_DIR, "job-%s" % job_id)
- @staticmethod
- def _GetArchivedJobPath(job_id):
+ @classmethod
+ def _GetArchivedJobPath(cls, job_id):
"""Returns the archived job file for a give job id.
@type job_id: str
@@ -784,7 +797,8 @@ class JobQueue(object):
@return: the path to the archived job file
"""
- return os.path.join(constants.JOB_QUEUE_ARCHIVE_DIR, "job-%s" % job_id)
+ path = "%s/job-%s" % (cls._GetArchiveDirectory(job_id), job_id)
+ return os.path.join(constants.JOB_QUEUE_ARCHIVE_DIR, path)
@classmethod
def _ExtractJobID(cls, name):
--
GitLab