Commit 2958c56e authored by Michael Hanselmann's avatar Michael Hanselmann

ganeti-cleaner: Separate queue cleaning code

This code does not need to run as root, therefore it's better to split
it out. It is now run with the same permissions as the master daemon.
Signed-off-by: default avatarMichael Hanselmann <hansmi@google.com>
Reviewed-by: default avatarIustin Pop <iustin@google.com>
parent 052783ff
...@@ -40,6 +40,7 @@ ...@@ -40,6 +40,7 @@
# daemons # daemons
/daemons/daemon-util /daemons/daemon-util
/daemons/ganeti-cleaner /daemons/ganeti-cleaner
/daemons/ganeti-master-cleaner
/daemons/ganeti-confd /daemons/ganeti-confd
/daemons/ganeti-masterd /daemons/ganeti-masterd
/daemons/ganeti-noded /daemons/ganeti-noded
......
...@@ -154,6 +154,7 @@ CLEANFILES = \ ...@@ -154,6 +154,7 @@ CLEANFILES = \
$(SHELL_ENV_INIT) \ $(SHELL_ENV_INIT) \
daemons/daemon-util \ daemons/daemon-util \
daemons/ganeti-cleaner \ daemons/ganeti-cleaner \
daemons/ganeti-master-cleaner \
devel/upload \ devel/upload \
$(BUILT_EXAMPLES) \ $(BUILT_EXAMPLES) \
doc/examples/bash_completion \ doc/examples/bash_completion \
...@@ -652,7 +653,8 @@ dist_sbin_SCRIPTS = \ ...@@ -652,7 +653,8 @@ dist_sbin_SCRIPTS = \
nodist_sbin_SCRIPTS = \ nodist_sbin_SCRIPTS = \
$(PYTHON_BOOTSTRAP_SBIN) \ $(PYTHON_BOOTSTRAP_SBIN) \
daemons/ganeti-cleaner daemons/ganeti-cleaner \
daemons/ganeti-master-cleaner
if HS_CONFD if HS_CONFD
nodist_sbin_SCRIPTS += htools/hconfd nodist_sbin_SCRIPTS += htools/hconfd
...@@ -719,6 +721,7 @@ EXTRA_DIST = \ ...@@ -719,6 +721,7 @@ EXTRA_DIST = \
$(RUN_IN_TEMPDIR) \ $(RUN_IN_TEMPDIR) \
daemons/daemon-util.in \ daemons/daemon-util.in \
daemons/ganeti-cleaner.in \ daemons/ganeti-cleaner.in \
daemons/ganeti-master-cleaner.in \
$(pkglib_python_scripts) \ $(pkglib_python_scripts) \
devel/upload.in \ devel/upload.in \
tools/kvm-ifup.in \ tools/kvm-ifup.in \
...@@ -754,6 +757,7 @@ EXTRA_DIST = \ ...@@ -754,6 +757,7 @@ EXTRA_DIST = \
man_MANS = \ man_MANS = \
man/ganeti.7 \ man/ganeti.7 \
man/ganeti-cleaner.8 \ man/ganeti-cleaner.8 \
man/ganeti-master-cleaner.8 \
man/ganeti-confd.8 \ man/ganeti-confd.8 \
man/ganeti-listrunner.8 \ man/ganeti-listrunner.8 \
man/ganeti-masterd.8 \ man/ganeti-masterd.8 \
...@@ -1047,7 +1051,8 @@ pep8_python_code = \ ...@@ -1047,7 +1051,8 @@ pep8_python_code = \
test/daemon-util_unittest.bash: daemons/daemon-util test/daemon-util_unittest.bash: daemons/daemon-util
test/ganeti-cleaner_unittest.bash: daemons/ganeti-cleaner test/ganeti-cleaner_unittest.bash: daemons/ganeti-cleaner \
daemons/ganeti-master-cleaner
test/bash_completion.bash: doc/examples/bash_completion-debug test/bash_completion.bash: doc/examples/bash_completion-debug
......
...@@ -26,25 +26,12 @@ set -e -u ...@@ -26,25 +26,12 @@ set -e -u
: ${CHECK_CERT_EXPIRED:=$PKGLIBDIR/check-cert-expired} : ${CHECK_CERT_EXPIRED:=$PKGLIBDIR/check-cert-expired}
readonly CLEANER_LOG_DIR=$LOG_DIR/cleaner readonly CLEANER_LOG_DIR=$LOG_DIR/cleaner
readonly QUEUE_ARCHIVE_DIR=$DATA_DIR/queue/archive
readonly CRYPTO_DIR=$RUN_DIR/crypto readonly CRYPTO_DIR=$RUN_DIR/crypto
in_cluster() { in_cluster() {
[[ -e $DATA_DIR/ssconf_master_node ]] [[ -e $DATA_DIR/ssconf_master_node ]]
} }
cleanup_master() {
# Return if machine is not part of a cluster
in_cluster || return 0
# Return if queue archive directory doesn't exist
[[ -d $QUEUE_ARCHIVE_DIR ]] || return 0
# Remove old jobs
find $QUEUE_ARCHIVE_DIR -mindepth 2 -type f -mtime +$REMOVE_AFTER -print0 | \
xargs -r0 rm -vf
}
cleanup_node() { cleanup_node() {
# Return if directory for crypto keys doesn't exist # Return if directory for crypto keys doesn't exist
[[ -d $CRYPTO_DIR ]] || return 0 [[ -d $CRYPTO_DIR ]] || return 0
...@@ -90,7 +77,6 @@ echo "Cleaner started at $(date)" ...@@ -90,7 +77,6 @@ echo "Cleaner started at $(date)"
find $CLEANER_LOG_DIR -maxdepth 1 -type f | sort | head -n -$KEEP_LOGS | \ find $CLEANER_LOG_DIR -maxdepth 1 -type f | sort | head -n -$KEEP_LOGS | \
xargs -r rm -vf xargs -r rm -vf
cleanup_master
cleanup_node cleanup_node
cleanup_watcher cleanup_watcher
......
#!/bin/bash
#
# Copyright (C) 2009, 2010, 2011, 2012 Google Inc.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
# 02110-1301, USA.
set -e -u
@SHELL_ENV_INIT@
readonly CLEANER_LOG_DIR=$LOG_DIR/master-cleaner
readonly QUEUE_ARCHIVE_DIR=$DATA_DIR/queue/archive
in_cluster() {
[[ -e $DATA_DIR/ssconf_master_node ]]
}
cleanup_master() {
# Return if machine is not part of a cluster
in_cluster || return 0
# Return if queue archive directory doesn't exist
[[ -d $QUEUE_ARCHIVE_DIR ]] || return 0
# Remove old jobs
find $QUEUE_ARCHIVE_DIR -mindepth 2 -type f -mtime +$REMOVE_AFTER -print0 | \
xargs -r0 rm -vf
}
# Define how many days archived jobs should be left alone
REMOVE_AFTER=21
# Define how many log files to keep around (usually one per day)
KEEP_LOGS=50
# Log file for this run
LOG_FILE=$CLEANER_LOG_DIR/cleaner-$(date +'%Y-%m-%dT%H_%M').$$.log
# Create log directory
mkdir -p $CLEANER_LOG_DIR
# Redirect all output to log file
exec >>$LOG_FILE 2>&1
echo "Cleaner started at $(date)"
# Remove old cleaner log files
find $CLEANER_LOG_DIR -maxdepth 1 -type f | sort | head -n -$KEEP_LOGS | \
xargs -r rm -vf
cleanup_master
exit 0
...@@ -4,4 +4,7 @@ PATH=/sbin:/bin:/usr/sbin:/usr/bin:/usr/local/sbin:/usr/local/bin ...@@ -4,4 +4,7 @@ PATH=/sbin:/bin:/usr/sbin:/usr/bin:/usr/local/sbin:/usr/local/bin
*/5 * * * * root [ -x @SBINDIR@/ganeti-watcher ] && @SBINDIR@/ganeti-watcher */5 * * * * root [ -x @SBINDIR@/ganeti-watcher ] && @SBINDIR@/ganeti-watcher
# Clean job archive (at 01:45 AM) # Clean job archive (at 01:45 AM)
45 1 * * * root [ -x @SBINDIR@/ganeti-cleaner ] && @SBINDIR@/ganeti-cleaner 45 1 * * * @GNTMASTERUSER@ [ -x @SBINDIR@/ganeti-master-cleaner ] && @SBINDIR@/ganeti-master-cleaner
# Clean job archive (at 02:45 AM)
45 2 * * * @GNTNODEDUSER@ [ -x @SBINDIR@/ganeti-cleaner ] && @SBINDIR@/ganeti-cleaner
...@@ -124,6 +124,8 @@ def GetPaths(): ...@@ -124,6 +124,8 @@ def GetPaths():
rapi_log = constants.DAEMONS_LOGFILES[constants.RAPI] rapi_log = constants.DAEMONS_LOGFILES[constants.RAPI]
rapi_dir = os.path.join(pathutils.DATA_DIR, "rapi") rapi_dir = os.path.join(pathutils.DATA_DIR, "rapi")
cleaner_log_dir = os.path.join(pathutils.LOG_DIR, "cleaner")
master_cleaner_log_dir = os.path.join(pathutils.LOG_DIR, "master-cleaner")
paths = [ paths = [
(pathutils.DATA_DIR, DIR, 0755, getent.masterd_uid, (pathutils.DATA_DIR, DIR, 0755, getent.masterd_uid,
...@@ -192,6 +194,8 @@ def GetPaths(): ...@@ -192,6 +194,8 @@ def GetPaths():
(rapi_log, FILE, 0600, getent.rapi_uid, getent.masterd_gid, False), (rapi_log, FILE, 0600, getent.rapi_uid, getent.masterd_gid, False),
(pathutils.LOG_OS_DIR, DIR, 0750, getent.masterd_uid, (pathutils.LOG_OS_DIR, DIR, 0750, getent.masterd_uid,
getent.daemons_gid), getent.daemons_gid),
(cleaner_log_dir, DIR, 0750, getent.noded_uid, getent.noded_gid),
(master_cleaner_log_dir, DIR, 0750, getent.masterd_uid, getent.masterd_gid),
]) ])
return paths return paths
......
...@@ -14,13 +14,12 @@ Synopsis ...@@ -14,13 +14,12 @@ Synopsis
DESCRIPTION DESCRIPTION
----------- -----------
The **ganeti-cleaner** is a periodically run script to clean old job The **ganeti-cleaner** is a periodically run script to remove expired
files from the job queue archive and to remove expired X509 X509 certificates and keys, as well as outdated **ganeti-watcher**
certificates and keys. information.
**ganeti-cleaner** automatically removes all files older than 21 days **ganeti-cleaner** automatically removes all expired certificates and
from ``@LOCALSTATEDIR@/lib/ganeti/queue/archive`` and all expired keys from ``@LOCALSTATEDIR@/run/ganeti/crypto``.
certificates and keys from ``@LOCALSTATEDIR@/run/ganeti/crypto``.
.. vim: set textwidth=72 : .. vim: set textwidth=72 :
.. Local Variables: .. Local Variables:
......
ganeti-master-cleaner(8) Ganeti | Version @GANETI_VERSION@
==========================================================
Name
----
ganeti-master-cleaner - Ganeti job queue cleaner
Synopsis
--------
**ganeti-master-cleaner**
DESCRIPTION
-----------
The **ganeti-master-cleaner** is a periodically run script to clean old
job files from the job queue archive.
**ganeti-master-cleaner** automatically removes all files older than 21
days from ``@LOCALSTATEDIR@/lib/ganeti/queue/archive``.
.. vim: set textwidth=72 :
.. Local Variables:
.. mode: rst
.. fill-column: 72
.. End:
...@@ -24,6 +24,7 @@ set -o pipefail ...@@ -24,6 +24,7 @@ set -o pipefail
export PYTHON=${PYTHON:=python} export PYTHON=${PYTHON:=python}
GNTC=daemons/ganeti-cleaner GNTC=daemons/ganeti-cleaner
GNTMC=daemons/ganeti-master-cleaner
CCE=tools/check-cert-expired CCE=tools/check-cert-expired
err() { err() {
...@@ -44,8 +45,13 @@ gencert() { ...@@ -44,8 +45,13 @@ gencert() {
} }
check_logfiles() { check_logfiles() {
local n=$1 local n=$1 p=$2 path
[[ "$(find $tmpls/log/ganeti/cleaner -mindepth 1 | wc -l)" -le "$n" ]] || \ if [[ "$p2" = master ]]; then
path=$tmpls/log/ganeti/mastercleaner
else
path=$tmpls/log/ganeti/cleaner
fi
[[ "$(find $path -mindepth 1 | wc -l)" -le "$n" ]] || \
err "Found more than $n logfiles" err "Found more than $n logfiles"
} }
...@@ -77,7 +83,15 @@ count_and_check_certs() { ...@@ -77,7 +83,15 @@ count_and_check_certs() {
} }
run_cleaner() { run_cleaner() {
CHECK_CERT_EXPIRED=$CCE LOCALSTATEDIR=$tmpls $GNTC local cmd
if [[ "$1" = master ]]; then
cmd=$GNTMC
else
cmd=$GNTC
fi
CHECK_CERT_EXPIRED=$CCE LOCALSTATEDIR=$tmpls $cmd
} }
create_archived_jobs() { create_archived_jobs() {
...@@ -162,10 +176,25 @@ run_cleaner ...@@ -162,10 +176,25 @@ run_cleaner
test -d $tmpls/log/ganeti/cleaner || err 'log/ganeti/cleaner should exist' test -d $tmpls/log/ganeti/cleaner || err 'log/ganeti/cleaner should exist'
check_logfiles 1 check_logfiles 1
upto 'Checking number of retained log files' test -d $tmpls/log/ganeti/master-cleaner && \
err 'log/ganeti/master-cleaner should not exist yet'
run_cleaner master
test -d $tmpls/log/ganeti/master-cleaner || \
err 'log/ganeti/master-cleaner should exist'
check_logfiles 1 master
upto 'Checking number of retained log files (master)'
for (( i=0; i < (maxlog + 10); ++i )); do
run_cleaner master
check_logfiles 1
check_logfiles $(( (i + 2) > $maxlog?$maxlog:(i + 2) )) master
done
upto 'Checking number of retained log files (node)'
for (( i=0; i < (maxlog + 10); ++i )); do for (( i=0; i < (maxlog + 10); ++i )); do
run_cleaner run_cleaner
check_logfiles $(( (i + 2) > $maxlog?$maxlog:(i + 2) )) check_logfiles $(( (i + 2) > $maxlog?$maxlog:(i + 2) ))
check_logfiles $maxlog master
done done
upto 'Removal of archived jobs (non-master)' upto 'Removal of archived jobs (non-master)'
...@@ -175,12 +204,16 @@ test -f $tmpls/lib/ganeti/ssconf_master_node && \ ...@@ -175,12 +204,16 @@ test -f $tmpls/lib/ganeti/ssconf_master_node && \
err 'ssconf_master_node should not exist' err 'ssconf_master_node should not exist'
run_cleaner run_cleaner
count_jobs 55 count_jobs 55
run_cleaner master
count_jobs 55
upto 'Removal of archived jobs (master node)' upto 'Removal of archived jobs (master node)'
create_archived_jobs create_archived_jobs
count_jobs 55 count_jobs 55
echo $HOSTNAME > $tmpls/lib/ganeti/ssconf_master_node echo $HOSTNAME > $tmpls/lib/ganeti/ssconf_master_node
run_cleaner run_cleaner
count_jobs 55
run_cleaner master
count_jobs 31 count_jobs 31
upto 'Certificate expiration' upto 'Certificate expiration'
...@@ -191,16 +224,22 @@ create_certdirs $tmpdir/validcert foo{a,b,c}123 trvRMH4Wvt OfDlh6Pc2n ...@@ -191,16 +224,22 @@ create_certdirs $tmpdir/validcert foo{a,b,c}123 trvRMH4Wvt OfDlh6Pc2n
create_certdirs $tmpdir/expcert bar{x,y,z}999 fx0ljoImWr em3RBC0U8c create_certdirs $tmpdir/expcert bar{x,y,z}999 fx0ljoImWr em3RBC0U8c
create_certdirs '' empty{1,2,3} gd2HCvRc iFG55Z0a PP28v5kg create_certdirs '' empty{1,2,3} gd2HCvRc iFG55Z0a PP28v5kg
count_and_check_certs 10 count_and_check_certs 10
run_cleaner master
count_and_check_certs 10
run_cleaner run_cleaner
count_and_check_certs 5 count_and_check_certs 5
check_logfiles $maxlog check_logfiles $maxlog
check_logfiles $maxlog master
count_jobs 31 count_jobs 31
upto 'Watcher status files' upto 'Watcher status files'
create_watcher_state create_watcher_state
count_watcher data 10 count_watcher data 10
count_watcher instance-status 10 count_watcher instance-status 10
run_cleaner master
count_watcher data 10
count_watcher instance-status 10
run_cleaner run_cleaner
count_watcher data 5 count_watcher data 5
count_watcher instance-status 5 count_watcher instance-status 5
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment