Commit b5fd9757 authored by Ilias Tsitsimpis's avatar Ilias Tsitsimpis Committed by Christos Stavrakakis

burnin: Add new bash script to run burnin

parent c222778e
#!/bin/bash
#Example script for an snf-burnin cronjob.
#Starts an a snf-burnin test, deletes stale instances and archives old logs.
#It aborts if snf-burnin runs for longer than expected.
#Usage: ./snf-burnin-run.sh
API=""
PLANKTON=""
PLANKTON_USER=""
TOKEN=""
IMAGEID=""
OUTPUT=""
# Delete previously stalled objects.
snf-burnin --api=$API --token=$TOKEN --delete-stale
# Run burnin for 25 minutes. Fail after that.
timeout --foreground 25m \
snf-burnin --api=$API \
--plankton=$PLANKTON \
--plankton-user=$PLANKTON_USER \
--token=$TOKEN \
--image-id=$IMAGEID \
--action-timeout 120 \
--log-folder=$OUTPUT \
--nofailfast
# Delete possibly stalled objects.
snf-burnin --api=$API --token=$TOKEN --delete-stale
#Delete old folders
old=$(date -d "1 week ago" +%Y%m%d%H%M%S)
for dir in $OUTPUT/* ; do
d=`basename $dir`
(($d<$old)) && rm -r "$dir"
done
# --------------------------------------------------------------------
# Configure script parameters
# ----------------------------------------
# Here we define the tokens for each user burnin will
# test along with an alias for each token.
# For each user define an ALIAS, his TOKEN, an IMAGEID and a FLAVOR.
USERS=(\
"burnin1" "token to be used" \
"image id to be used" "flavor id to be used" \
"burnin2" "token to be used" \
"image id to be used" "flavor id to be used" \
"burnin3" "token to be used" \
"image id to be used" "flavor id to be used" \
)
# ----------------------------------------
# Here we define the email parameters
# Email Tag
TAG="[synnefo.org-burnin]"
# Email Recipients
RECIPIENTS="burnin@synnefo.org"
# Subject for a successful burnin run
# (will be "$TAG ($ALIAS) $SUCCESS_SUBJECT" for each burnin instance)
SUCCESS_SUBJECT="Burnin Succeeded"
# Subject for a failed burnin run
# (will be "$TAG ($ALIAS) $FAILURE_SUBJECT" for each burnin instance)
FAILURE_SUBJECT="Burnin Failed"
# ----------------------------------------
# Some burnin parameters
AUTH_URL="https://accounts.synnefo.org/identity/v2.0/"
TIMEOUT=240
# ----------------------------------------
# Burnin executable and log files
Burnin="snf-burnin"
# Log Folder will be $LOGFOLDER/$ALIAS for each burnin instance
LOGFOLDER="/var/log/burnin_results/"
# Output file will be $OUTPUTFOLDER/burnin-$ALIAS.out for each burnin instance
OUTPUTFOLDER="/tmp"
# Lock file (we don't want two instances of this script)
LOCKFILE="/tmp/burnin.lockfile"
# --------------------------------------------------------------------
# Script functions
run_burnin() {
local alias="$1"
local token="$2"
local image="$3"
local flavor="$4"
local success_subject="$TAG ($alias) $SUCCESS_SUBJECT"
local failure_subject="$TAG ($alias) $FAILURE_SUBJECT"
local logfolder="$LOGFOLDER/$alias"
local outputfile="$OUTPUTFOLDER/burnin-$alias.out"
local failed=false
local error_summary
local stale_subject
# Save date-stamp to output
date > $outputfile
echo -e \
"\n\n===== Burnin Output ========================================" \
>> $outputfile
# Check for stale servers/networks
$Burnin --token="$token" --auth-url="$AUTH_URL" --show-stale 2>&1 | \
grep "test" >> $outputfile 2>&1
if [ $? -ne 0 ]; then
# No stale servers/networks found. Run burnin
$Burnin --token="$token" \
--action-timeout="$TIMEOUT" \
--image-id="$image" \
--log-folder="$logfolder" \
--auth-url="$AUTH_URL" \
--force-flavor="$flavor" \
--nofailfast \
&>> $outputfile
echo -e \
"\n\n===== Burnin Logs ==========================================" \
>> $outputfile
# Search log files for errors
for file in `ls -1d $logfolder/* | tail -1`/*/detail* ; do
if egrep "(ERROR)|(FAILED)" $file > /dev/null; then
failed=true
echo "FILENAME: $file" >> $outputfile
echo "ERROR: " >> $outputfile
cat "$file" >> $outputfile
fi
done
# Clean output file from escape characters
sed -ri "s/\x1B\[([0-9]{1,2}(;[0-9]{1,2})?)?[m|K]//g" $outputfile
# Send emails
if $failed; then
error_summary="`cat $outputfile | \
egrep "ERROR: test_|FAIL: test_" | \
awk '{ print $2 }' | grep -v "^$" | \
sed -e 's/_/ /g' | cut -d" " -f3- | \
tr '\n' ',' | sed -e 's/,$//g' | \
sed -e 's/,/, /g'`"
cat $outputfile | /usr/bin/mailx -E \
-s "$failure_subject: $error_summary" $RECIPIENTS
# else
# cat $outputfile | /usr/bin/mailx -E \
# -s "$success_subject" $RECIPIENTS
fi
else
# Burnin found stale servers/networks. Try to clean them
$Burnin --token="$token" --auth-url="$AUTH_URL" --delete-stale \
>> $outputfile 2>&1
if [ $? -ne 0 ]; then
stale_subject="$failure_subject: Couldn't delete stale servers/networks"
else
stale_subject="$success_subject: Stale servers/networks deleted"
fi
# Clean output file from escape characters
sed -ri "s/\x1B\[([0-9]{1,2}(;[0-9]{1,2})?)?[m|K]//g" $outputfile
# Send mail
cat $outputfile | /usr/bin/mailx -E \
-s "$stale_subject" $RECIPIENTS
fi
}
# --------------------------------------------------------------------
# For each user run burnin function
(
flock -xn 200 || exit 1
set ${USERS[@]}
while [ -n "$1" ]; do
run_burnin "$1" "$2" "$3" "$4" &
shift 4
done
wait
) 200>$LOCKFILE
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment