Commit 3865ca48 authored by Michael Hanselmann's avatar Michael Hanselmann

utils: Move I/O-related code into separate file

Signed-off-by: default avatarMichael Hanselmann <>
Reviewed-by: default avatarIustin Pop <>
parent 9d1b963f
......@@ -216,6 +216,7 @@ utils_PYTHON = \
lib/utils/ \
lib/utils/ \
lib/utils/ \
lib/utils/ \
lib/utils/ \
lib/utils/ \
lib/utils/ \
......@@ -489,6 +490,7 @@ python_tests = \
test/ \
test/ \
test/ \
test/ \
test/ \
test/ \
test/ \
......@@ -2512,7 +2512,7 @@ def _EnsureJobQueueFile(file_name):
def JobQueueUpdate(file_name, content):
"""Updates a file in the queue directory.
This is just a wrapper over L{utils.WriteFile}, with proper
This is just a wrapper over L{}, with proper
@type file_name: str
......@@ -3399,7 +3399,7 @@ class DevCacheManager(object):
def RemoveCache(cls, dev_path):
"""Remove data for a dev_path.
This is just a wrapper over L{utils.RemoveFile} with a converted
This is just a wrapper over L{} with a converted
path name and logging.
@type dev_path: str
......@@ -61,6 +61,7 @@ from ganeti.utils.log import * # pylint: disable-msg=W0401
from ganeti.utils.hash import * # pylint: disable-msg=W0401
from ganeti.utils.wrapper import * # pylint: disable-msg=W0401
from ganeti.utils.filelock import * # pylint: disable-msg=W0401
from import * # pylint: disable-msg=W0401
#: when set to True, L{RunCmd} is disabled
......@@ -701,85 +702,6 @@ def RunParts(dir_name, env=None, reset_env=False):
return rr
def RemoveFile(filename):
"""Remove a file ignoring some errors.
Remove a file, ignoring non-existing ones or directories. Other
errors are passed.
@type filename: str
@param filename: the file to be removed
except OSError, err:
if err.errno not in (errno.ENOENT, errno.EISDIR):
def RemoveDir(dirname):
"""Remove an empty directory.
Remove a directory, ignoring non-existing ones.
Other errors are passed. This includes the case,
where the directory is not empty, so it can't be removed.
@type dirname: str
@param dirname: the empty directory to be removed
except OSError, err:
if err.errno != errno.ENOENT:
def RenameFile(old, new, mkdir=False, mkdir_mode=0750):
"""Renames a file.
@type old: string
@param old: Original path
@type new: string
@param new: New path
@type mkdir: bool
@param mkdir: Whether to create target directory if it doesn't exist
@type mkdir_mode: int
@param mkdir_mode: Mode for newly created directories
return os.rename(old, new)
except OSError, err:
# In at least one use case of this function, the job queue, directory
# creation is very rare. Checking for the directory before renaming is not
# as efficient.
if mkdir and err.errno == errno.ENOENT:
# Create directory and try again
Makedirs(os.path.dirname(new), mode=mkdir_mode)
return os.rename(old, new)
def Makedirs(path, mode=0750):
"""Super-mkdir; create a leaf directory and all intermediate ones.
This is a wrapper around C{os.makedirs} adding error handling not implemented
before Python 2.5.
os.makedirs(path, mode)
except OSError, err:
# Ignore EEXIST. This is only handled in os.makedirs as included in
# Python 2.5 and above.
if err.errno != errno.EEXIST or not os.path.exists(path):
def ResetTempfileModule():
"""Resets the random name generator of the tempfile module.
......@@ -1002,63 +924,6 @@ def IsProcessHandlingSignal(pid, signum, status_path=None):
return signum in _ParseSigsetT(sigcgt)
def ReadPidFile(pidfile):
"""Read a pid from a file.
@type pidfile: string
@param pidfile: path to the file containing the pid
@rtype: int
@return: The process id, if the file exists and contains a valid PID,
otherwise 0
raw_data = ReadOneLineFile(pidfile)
except EnvironmentError, err:
if err.errno != errno.ENOENT:
logging.exception("Can't read pid file")
return 0
pid = int(raw_data)
except (TypeError, ValueError), err:"Can't parse pid file contents", exc_info=True)
return 0
return pid
def ReadLockedPidFile(path):
"""Reads a locked PID file.
This can be used together with L{StartDaemon}.
@type path: string
@param path: Path to PID file
@return: PID as integer or, if file was unlocked or couldn't be opened, None
fd =, os.O_RDONLY)
except EnvironmentError, err:
if err.errno == errno.ENOENT:
# PID file doesn't exist
return None
# Try to acquire lock
except errors.LockError:
# Couldn't lock, daemon is running
return int(, 100))
return None
def ValidateServiceName(name):
"""Validate the given service name.
......@@ -1225,72 +1090,6 @@ def ParseCpuMask(cpu_mask):
return cpu_list
def AddAuthorizedKey(file_obj, key):
"""Adds an SSH public key to an authorized_keys file.
@type file_obj: str or file handle
@param file_obj: path to authorized_keys file
@type key: str
@param key: string containing key
key_fields = key.split()
if isinstance(file_obj, basestring):
f = open(file_obj, 'a+')
f = file_obj
nl = True
for line in f:
# Ignore whitespace changes
if line.split() == key_fields:
nl = line.endswith('\n')
if not nl:
def RemoveAuthorizedKey(file_name, key):
"""Removes an SSH public key from an authorized_keys file.
@type file_name: str
@param file_name: path to authorized_keys file
@type key: str
@param key: string containing key
key_fields = key.split()
fd, tmpname = tempfile.mkstemp(dir=os.path.dirname(file_name))
out = os.fdopen(fd, 'w')
f = open(file_name, 'r')
for line in f:
# Ignore whitespace changes while comparing lines
if line.split() != key_fields:
os.rename(tmpname, file_name)
def SetEtcHostsEntry(file_name, ip, hostname, aliases):
"""Sets the name of an IP address and hostname in /etc/hosts.
......@@ -1391,66 +1190,6 @@ def RemoveHostFromEtcHosts(hostname):
RemoveEtcHostsEntry(constants.ETC_HOSTS, hostname.split(".")[0])
def TimestampForFilename():
"""Returns the current time formatted for filenames.
The format doesn't contain colons as some shells and applications treat them
as separators. Uses the local timezone.
return time.strftime("%Y-%m-%d_%H_%M_%S")
def CreateBackup(file_name):
"""Creates a backup of a file.
@type file_name: str
@param file_name: file to be backed up
@rtype: str
@return: the path to the newly created backup
@raise errors.ProgrammerError: for invalid file names
if not os.path.isfile(file_name):
raise errors.ProgrammerError("Can't make a backup of a non-file '%s'" %
prefix = ("%s.backup-%s." %
(os.path.basename(file_name), TimestampForFilename()))
dir_name = os.path.dirname(file_name)
fsrc = open(file_name, 'rb')
(fd, backup_name) = tempfile.mkstemp(prefix=prefix, dir=dir_name)
fdst = os.fdopen(fd, 'wb')
logging.debug("Backing up %s at %s", file_name, backup_name)
shutil.copyfileobj(fsrc, fdst)
return backup_name
def ListVisibleFiles(path):
"""Returns a list of visible files in a directory.
@type path: str
@param path: the directory to enumerate
@rtype: list
@return: the list of all files not starting with a dot
@raise ProgrammerError: if L{path} is not an absolue and normalized path
if not IsNormAbsPath(path):
raise errors.ProgrammerError("Path passed to ListVisibleFiles is not"
" absolute/normalized: '%s'" % path)
files = [i for i in os.listdir(path) if not i.startswith(".")]
return files
def GetHomeDir(user, default=None):
"""Try to get the homedir of the given user.
......@@ -1483,227 +1222,6 @@ def NewUUID():
return ReadFile(_RANDOM_UUID_FILE, size=128).rstrip("\n")
def EnsureDirs(dirs):
"""Make required directories, if they don't exist.
@param dirs: list of tuples (dir_name, dir_mode)
@type dirs: list of (string, integer)
for dir_name, dir_mode in dirs:
os.mkdir(dir_name, dir_mode)
except EnvironmentError, err:
if err.errno != errno.EEXIST:
raise errors.GenericError("Cannot create needed directory"
" '%s': %s" % (dir_name, err))
os.chmod(dir_name, dir_mode)
except EnvironmentError, err:
raise errors.GenericError("Cannot change directory permissions on"
" '%s': %s" % (dir_name, err))
if not os.path.isdir(dir_name):
raise errors.GenericError("%s is not a directory" % dir_name)
def ReadFile(file_name, size=-1):
"""Reads a file.
@type size: int
@param size: Read at most size bytes (if negative, entire file)
@rtype: str
@return: the (possibly partial) content of the file
f = open(file_name, "r")
def WriteFile(file_name, fn=None, data=None,
mode=None, uid=-1, gid=-1,
atime=None, mtime=None, close=True,
dry_run=False, backup=False,
prewrite=None, postwrite=None):
"""(Over)write a file atomically.
The file_name and either fn (a function taking one argument, the
file descriptor, and which should write the data to it) or data (the
contents of the file) must be passed. The other arguments are
optional and allow setting the file mode, owner and group, and the
mtime/atime of the file.
If the function doesn't raise an exception, it has succeeded and the
target file has the new contents. If the function has raised an
exception, an existing target file should be unmodified and the
temporary file should be removed.
@type file_name: str
@param file_name: the target filename
@type fn: callable
@param fn: content writing function, called with
file descriptor as parameter
@type data: str
@param data: contents of the file
@type mode: int
@param mode: file mode
@type uid: int
@param uid: the owner of the file
@type gid: int
@param gid: the group of the file
@type atime: int
@param atime: a custom access time to be set on the file
@type mtime: int
@param mtime: a custom modification time to be set on the file
@type close: boolean
@param close: whether to close file after writing it
@type prewrite: callable
@param prewrite: function to be called before writing content
@type postwrite: callable
@param postwrite: function to be called after writing content
@rtype: None or int
@return: None if the 'close' parameter evaluates to True,
otherwise the file descriptor
@raise errors.ProgrammerError: if any of the arguments are not valid
if not os.path.isabs(file_name):
raise errors.ProgrammerError("Path passed to WriteFile is not"
" absolute: '%s'" % file_name)
if [fn, data].count(None) != 1:
raise errors.ProgrammerError("fn or data required")
if [atime, mtime].count(None) == 1:
raise errors.ProgrammerError("Both atime and mtime must be either"
" set or None")
if backup and not dry_run and os.path.isfile(file_name):
dir_name, base_name = os.path.split(file_name)
fd, new_name = tempfile.mkstemp('.new', base_name, dir_name)
do_remove = True
# here we need to make sure we remove the temp file, if any error
# leaves it in place
if uid != -1 or gid != -1:
os.chown(new_name, uid, gid)
if mode:
os.chmod(new_name, mode)
if callable(prewrite):
if data is not None:
os.write(fd, data)
if callable(postwrite):
if atime is not None and mtime is not None:
os.utime(new_name, (atime, mtime))
if not dry_run:
os.rename(new_name, file_name)
do_remove = False
if close:
result = None
result = fd
if do_remove:
return result
def GetFileID(path=None, fd=None):
"""Returns the file 'id', i.e. the dev/inode and mtime information.
Either the path to the file or the fd must be given.
@param path: the file path
@param fd: a file descriptor
@return: a tuple of (device number, inode number, mtime)
if [path, fd].count(None) != 1:
raise errors.ProgrammerError("One and only one of fd/path must be given")
if fd is None:
st = os.stat(path)
st = os.fstat(fd)
return (st.st_dev, st.st_ino, st.st_mtime)
def VerifyFileID(fi_disk, fi_ours):
"""Verifies that two file IDs are matching.
Differences in the inode/device are not accepted, but and older
timestamp for fi_disk is accepted.
@param fi_disk: tuple (dev, inode, mtime) representing the actual
file data
@param fi_ours: tuple (dev, inode, mtime) representing the last
written file data
@rtype: boolean
(d1, i1, m1) = fi_disk
(d2, i2, m2) = fi_ours
return (d1, i1) == (d2, i2) and m1 <= m2
def SafeWriteFile(file_name, file_id, **kwargs):
"""Wraper over L{WriteFile} that locks the target file.
By keeping the target file locked during WriteFile, we ensure that
cooperating writers will safely serialise access to the file.
@type file_name: str
@param file_name: the target filename
@type file_id: tuple
@param file_id: a result from L{GetFileID}
fd =, os.O_RDONLY | os.O_CREAT)
if file_id is not None:
disk_id = GetFileID(fd=fd)
if not VerifyFileID(disk_id, file_id):
raise errors.LockError("Cannot overwrite file %s, it has been modified"
" since last written" % file_name)
return WriteFile(file_name, **kwargs)
def ReadOneLineFile(file_name, strict=False):
"""Return the first non-empty line from a file.
@type strict: boolean
@param strict: if True, abort if the file has more than one
non-empty line
file_lines = ReadFile(file_name).splitlines()
full_lines = filter(bool, file_lines)
if not file_lines or not full_lines:
raise errors.GenericError("No data in one-liner file %s" % file_name)
elif strict and len(full_lines) > 1:
raise errors.GenericError("Too many lines in one-liner file %s" %
return full_lines[0]
def FirstFree(seq, base=0):
"""Returns the first non-existing integer from seq.
......@@ -1904,19 +1422,6 @@ def Daemonize(logfile):
return wpipe
def DaemonPidFileName(name):
"""Compute a ganeti pid file absolute path
@type name: str
@param name: the daemon name
@rtype: str
@return: the full path to the pidfile corresponding to the given
daemon name
return PathJoin(constants.RUN_GANETI_DIR, "" % name)
def EnsureDaemon(name):
"""Check for and start daemon if not alive.
......@@ -1943,49 +1448,6 @@ def StopDaemon(name):
return True
def WritePidFile(pidfile):
"""Write the current process pidfile.
@type pidfile: string
@param pidfile: the path to the file to be written
@raise errors.LockError: if the pid file already exists and
points to a live process
@rtype: int
@return: the file descriptor of the lock file; do not close this unless
you want to unlock the pid file
# We don't rename nor truncate the file to not drop locks under
# existing processes
fd_pidfile =, os.O_WRONLY | os.O_CREAT, 0600)
# Lock the PID file (and fail if not possible to do so). Any code
# wanting to send a signal to the daemon should try to lock the PID
# file before reading it. If acquiring the lock succeeds, the daemon is
# no longer running and the signal should not be sent.
os.write(fd_pidfile, "%d\n" % os.getpid())
return fd_pidfile
def RemovePidFile(pidfile):
"""Remove the current process pidfile.
Any errors are ignored.
@type pidfile: string
@param pidfile: Path to the file to be removed
# TODO: we could check here that the file contains our pid
except Exception: # pylint: disable-msg=W0703
def KillProcess(pid, signal_=signal.SIGTERM, timeout=30,
"""Kill a process given by its pid.
......@@ -2049,40 +1511,6 @@ def KillProcess(pid, signal_=signal.SIGTERM, timeout=30,
_helper(pid, signal.SIGKILL, waitpid)
def FindFile(name, search_path, test=os.path.exists):
"""Look for a filesystem object in a given path.
This is an abstract method to search for filesystem object (files,
dirs) under a given search path.
@type name: str
@param name: the name to look for
@type search_path: str
@param search_path: location to start at
@type test: callable
@param test: a function taking one argument that should return True
if the a given object is valid; the default value is
os.path.exists, causing only existing files to be returned
@rtype: str or None
@return: full path to the object if found, None otherwise
# validate the filename mask
if constants.EXT_PLUGIN_MASK.match(name) is None:
logging.critical("Invalid value passed for external script name: '%s'",
return None