Commit 53d4cdf1 authored by Iustin Pop's avatar Iustin Pop
Browse files

Convert manual shell tests to shelltestrunner



This is more of a RFC… Basically most of the shell-based tests are
converted from exec+grep to shelltestrunner.

Things are not all fine and nice though:

- we have dependencies between tests, as some generate some data files
  needed later; this is not nice, and we depend on serial execution in
  testrunner
- we can still fail with no so nice messages in the offline-test
  script (when we generate most of the data)

But overall, I think the tests are much nicer to
define/read/debug:

- each test is standalone, with the only dependency being an optional
  input data file; this is much better than a single monolithic shell
  script
- in case of failures, the failure is clearly shown by shell test,
  both for exit code and stdout/stderr
- shelltest can run in --debug mode, where the exact details are shown
  much better than the alternative of "set -x" for the shell script

Comments welcome!
Signed-off-by: default avatarIustin Pop <iustin@google.com>
Reviewed-by: default avatarGuido Trotter <ultrotter@google.com>
parent b714ff89
......@@ -759,7 +759,19 @@ TEST_FILES = \
test/data/vgreduce-removemissing-2.02.66-ok.txt \
test/data/vgs-missing-pvs-2.02.02.txt \
test/data/vgs-missing-pvs-2.02.66.txt \
test/import-export_unittest-helper
test/import-export_unittest-helper \
test/htools-balancing.test \
test/htools-basic.test \
test/htools-dynutil.test \
test/htools-excl.test \
test/htools-hail.test \
test/htools-hspace.test \
test/htools-invalid.test \
test/htools-multi-group.test \
test/htools-no-backend.test \
test/htools-rapi.test \
test/htools-single-group.test \
test/htools-text-backend.test
python_tests = \
test/ganeti.asyncnotifier_unittest.py \
......
......@@ -25,198 +25,67 @@ set -o pipefail
. $(dirname $0)/cli-tests-defs.sh
T=`mktemp -d`
echo Running offline htools tests
export T=`mktemp -d`
trap 'rm -rf $T' EXIT
trap 'echo FAIL' ERR
trap 'echo FAIL to build test files' ERR
echo Using $T as temporary dir
echo Checking command line basic options
for prog in $ALL_ROLES; do
$prog --version >/dev/null
$prog --help >/dev/null
! $prog --no-such-option 2>/dev/null
done
echo OK
echo Checking missing backend failure
for prog in hspace hinfo hbal; do
! $prog 2>/dev/null
done
echo OK
echo Checking hail missing input file
! hail 2>/dev/null
echo OK
echo Checking extra arguments
for prog in hspace hbal hinfo; do
(! $prog unexpected-argument 2>&1 ) | \
grep -q "Error: this program doesn't take any arguments"
done
echo OK
echo Checking failure on multiple backends
(! hbal -t /dev/null -m localhost 2>&1 ) | \
grep -q "Error: Only one of the rapi, luxi, and data files options should be given."
echo OK
echo Checking text file loading
hbal -t $TESTDATA_DIR/missing-resources.data 2>&1 | \
grep "node node2 is missing .* ram and .* disk" >/dev/null
hbal -t $TESTDATA_DIR/common-suffix.data -v 2>&1 | \
grep "Stripping common suffix of '.example.com' from names" >/dev/null
(! hbal -t $TESTDATA_DIR/invalid-node.data 2>&1 ) | \
grep "Unknown node '.*' for instance new-0" >/dev/null
echo OK
echo Checking hspace machine-readable mode
hspace --simu p,4,8T,64g,16 --machine-readable \
--disk-template drbd -l 8 >$T/capacity
( . $T/capacity && test "$HTS_OK" = "1" )
echo OK
echo Checking hspace simulation to hinfo to hbal
echo -n Generating hspace simulation data for hinfo and hbal...
# this cluster spec should be fine
hspace --simu p,4,8T,64g,16 -S $T/simu-onegroup \
./test/hspace --simu p,4,8T,64g,16 -S $T/simu-onegroup \
--disk-template drbd -l 8 -v -v -v >/dev/null 2>&1
# results in .tiered and .standard
for suffix in standard tiered; do
BACKEND="-t$T/simu-onegroup.$suffix"
hinfo -v -v -p --print-instances $BACKEND >/dev/null 2>&1
hbal -v -v -p --print-instances $BACKEND >/dev/null 2>&1
# hbal should not be able to balance
hbal $BACKEND | grep -qE "(Nothing to do, exiting|No solution found)"
done
echo OK
echo Checking hinfo and hbal on multi-nodegroup
hspace --simu p,4,8T,64g,16 --simu p,4,8T,64g,16 \
echo -n Generating hinfo and hbal test files for multi-group...
./test/hspace --simu p,4,8T,64g,16 --simu p,4,8T,64g,16 \
-S $T/simu-twogroups --disk-template drbd -l 8 >/dev/null 2>&1
# results in .tiered and .standard
for suffix in standard tiered; do
BACKEND="-t$T/simu-twogroups.$suffix"
hinfo -v -v -p --print-instances $BACKEND >/dev/null 2>&1
! hbal $BACKEND >/dev/null 2>&1
# hbal should not be able to balance
! hbal $BACKEND 2>&1 | grep -q "Found multiple node groups"
# but hbal should be able to balance one node group
hbal $BACKEND -G group-01 >/dev/null
# and it should not find an invalid group
! hbal $BACKEND -G no-such-group >/dev/null 2>&1
done
echo OK
echo Checking rebalancing
echo -n Generating test files for rebalancing...
# we generate a cluster with two node groups, one with unallocable
# policy, then we change all nodes from this group to the allocable
# one, and we check for rebalancing
FROOT="$T/simu-rebal-orig"
hspace --simu u,4,8T,64g,16 --simu p,4,8T,64g,16 \
./test/hspace --simu u,4,8T,64g,16 --simu p,4,8T,64g,16 \
-S $FROOT --disk-template drbd -l 8 >/dev/null 2>&1
for suffix in standard tiered; do
RELOC="$T/simu-rebal-merged.$suffix"
# this relocates the nodes
sed -re 's/^(node-.*|fake-uuid-)-02(|.*)/\1-01\2/' \
< $FROOT.$suffix > $RELOC
BACKEND="-t$RELOC"
hinfo -v -v -p --print-instances $BACKEND >/dev/null 2>&1
hbal -v -v -v -p --print-instances $BACKEND -G group-01 2>/dev/null | \
grep -qE -v "(Nothing to do, exiting|No solution found)"
hbal $BACKEND -G group-01 -C$T/rebal-cmds.$suffix \
-S $T/simu-rebal.$suffix >/dev/null 2>&1
grep -qE "gnt-instance (failover|migrate|replace-disks)" \
$T/rebal-cmds.$suffix
hbal $BACKEND -G group-01 -C \
-S $T/simu-rebal.$suffix 2>/dev/null | \
grep -qE "gnt-instance (failover|migrate|replace-disks)"
# state saved by hbal should be original
cmp $RELOC $T/simu-rebal.$suffix.original
# and we can't double rebalance
hbal -t $T/simu-rebal.$suffix.balanced \
-G group-01 | \
grep -qE "(Nothing to do, exiting|No solution found)"
done
export BACKEND_BAL_STD="-t$T/simu-rebal-merged.standard"
export BACKEND_BAL_TIER="-t$T/simu-rebal-merged.tiered"
echo OK
echo Checking utilisation-based code
BACKEND="-t $T/simu-onegroup.standard"
echo a > $T/dynu
(! hbal -U <(echo a) $BACKEND 2>&1 ) | grep -q "Cannot parse line"
(! hbal -U <(echo a b c d e f g h) $BACKEND 2>&1 ) | \
grep -q "Cannot parse line"
(! hbal -U <(echo inst cpu mem dsk net) $BACKEND 2>&1 ) | \
grep -Eq "cannot parse string '(cpu|mem|dsk|net)'"
# unknown instances are currently just ignored
hbal -U <(echo no-such-inst 2 2 2 2) $BACKEND >/dev/null 2>&1
# new-0 is the name of the first instance allocated by hspace
hbal -U <(echo new-0 2 2 2 2) $BACKEND >/dev/null 2>&1
echo OK
echo Checking selected/excluded instances
(! hbal $BACKEND --exclude-instances no-such-instance 2>&1 ) | \
grep -q "Unknown instance"
(! hbal $BACKEND --select-instances no-such-instances 2>&1 ) | \
grep -q "Unknown instance"
hbal $BACKEND --exclude-instances new-0 --select-instances new-1 >/dev/null
echo OK
echo IAllocator checks
# test that on invalid files it can't parse the request
(! hail /dev/null 2>&1 ) | grep -q "Invalid JSON"
! hail <(echo '[]') >/dev/null 2>&1
(! hail <(echo '{}') 2>&1 ) | grep -q "key 'request' not found"
(! hail <(echo '{"request": 0}') 2>&1 ) | grep -q "key 'request'"
! hail $TESTDATA_DIR/hail-invalid-reloc.json >/dev/null 2>&1
# just test that it can read the file, print the cluster and generate
# pre and post allocation files
hail -v -v -v -p $TESTDATA_DIR/hail-alloc-drbd.json \
-S $T/hail-alloc >/dev/null 2>&1
! cmp -s $T/hail-alloc.pre-ialloc $T/hail-alloc.post-ialloc
hail -v -v -v -p $TESTDATA_DIR/hail-reloc-drbd.json \
-S $T/hail-reloc >/dev/null 2>&1
! cmp -s $T/hail-reloc.pre-ialloc $T/hail-reloc.post-ialloc
# and now start the real tests
hail $TESTDATA_DIR/hail-alloc-drbd.json | \
grep -q '"success":true,.*,"result":\["node2","node1"\]'
hail $TESTDATA_DIR/hail-reloc-drbd.json | \
grep -q '"success":true,.*,"result":\["node1"\]'
hail $TESTDATA_DIR/hail-node-evac.json | \
grep -Fq '"success":true,"info":"Request successful: 0 instances failed to move and 1 were moved successfully"'
hail $TESTDATA_DIR/hail-change-group.json | \
grep -Fq '"success":true,"info":"Request successful: 0 instances failed to move and 1 were moved successfully"'
# For various tests
export BACKEND_DYNU="-t $T/simu-onegroup.standard"
export BACKEND_EXCL="-t $T/simu-onegroup.standard"
echo -n Generating data files for IAllocator checks...
for evac_mode in primary-only secondary-only all; do
sed -e 's/"evac_mode": "all"/"evac_mode": "'${evac_mode}'"/' \
< $TESTDATA_DIR/hail-node-evac.json \
> $T/hail-node-evac.json.$evac_mode
hail $T/hail-node-evac.json.$evac_mode | \
grep -q '"success":true,"info":"Request successful: 0 instances failed to move and 1 were moved successfully"'
done
# check that hail can use the simu and text backends
hail --simu p,8,8T,16g,16 $TESTDATA_DIR/hail-alloc-drbd.json | \
grep -q '"success":true,'
hail -t $T/simu-rebal-merged.standard $TESTDATA_DIR/hail-alloc-drbd.json | \
grep -q '"success":true,'
echo OK
echo Checking file-based RAPI
echo -n Checking file-based RAPI...
mkdir -p $T/hscan
URL="file://$TESTDATA_DIR/rapi"
hinfo -v -v -p --print-instances -m $URL >/dev/null 2>&1
hbal -v -v -p --print-instances -m $URL >/dev/null 2>&1
hscan -d $T/hscan/ -p -v -v $URL >/dev/null 2>&1
export RAPI_URL="file://$TESTDATA_DIR/rapi"
./test/hscan -d $T/hscan/ -p -v -v $RAPI_URL >/dev/null 2>&1
# check that we file parsing is correct, i.e. hscan saves correct text
# files, and is idempotent (rapi+text == rapi)
HS="$(ls $T/hscan/*.data|head -n1)"
hinfo -p --print-instances -m $URL > $T/hscan/direct.hinfo 2>&1
hinfo -p --print-instances -t $HS > $T/hscan/fromtext.hinfo 2>&1
cmp -s $T/hscan/direct.hinfo $T/hscan/fromtext.hinfo
# files, and is idempotent (rapi+text == rapi); more is tested in
# shelltest later
RAPI_TXT="$(ls $T/hscan/*.data|head -n1)"
./test/hinfo -p --print-instances -m $RAPI_URL > $T/hscan/direct.hinfo 2>&1
./test/hinfo -p --print-instances -t $RAPI_TXT > $T/hscan/fromtext.hinfo 2>&1
echo OK
echo Running shelltest...
shelltest $SHELLTESTARGS test/ -- --hide-successes
echo All OK
### std tests
# test basic parsing
./test/hinfo -v -v -p --print-instances $BACKEND_BAL_STD
>>>= 0
./test/hbal -v -v -v -p --print-instances $BACKEND_BAL_STD -G group-01
>>> !/(Nothing to do, exiting|No solution found)/
>>>2 !/(Nothing to do, exiting|No solution found)/
>>>= 0
# test command output
./test/hbal $BACKEND_BAL_STD -G group-01 -C -S $T/simu-rebal.standard
>>> /gnt-instance (failover|migrate|replace-disks)/
>>>= 0
# test saving commands
./test/hbal $BACKEND_BAL_STD -G group-01 -C$T/rebal-cmds.standard
>>>= 0
# and now check the file (depends on previous test)
cat $T/rebal-cmds.standard
>>> /gnt-instance (failover|migrate|replace-disks)/
>>>= 0
# state saved before rebalancing should be identical; depends on the
# previous test
diff -u $T/simu-rebal-merged.standard $T/simu-rebal.standard.original
>>>
>>>= 0
# no double rebalance; depends on previous test
./test/hbal -t $T/simu-rebal.standard.balanced -G group-01
>>> /(Nothing to do, exiting|No solution found)/
>>>= 0
### now tiered tests
# test basic parsing
./test/hinfo -v -v -p --print-instances $BACKEND_BAL_TIER
>>>= 0
./test/hbal -v -v -v -p --print-instances $BACKEND_BAL_TIER -G group-01
>>> !/(Nothing to do, exiting|No solution found)/
>>>2 !/(Nothing to do, exiting|No solution found)/
>>>= 0
# test command output
./test/hbal $BACKEND_BAL_TIER -G group-01 -C -S $T/simu-rebal.tiered
>>> /gnt-instance (failover|migrate|replace-disks)/
>>>= 0
# test saving commands
./test/hbal $BACKEND_BAL_TIER -G group-01 -C$T/rebal-cmds.tiered
>>>= 0
# and now check the file (depends on previous test)
cat $T/rebal-cmds.tiered
>>> /gnt-instance (failover|migrate|replace-disks)/
>>>= 0
# state saved before rebalancing should be identical; depends on the
# previous test
diff -u $T/simu-rebal-merged.tiered $T/simu-rebal.tiered.original
>>>
>>>= 0
# no double rebalance; depends on previous test
./test/hbal -t $T/simu-rebal.tiered.balanced -G group-01
>>> /(Nothing to do, exiting|No solution found)/
>>>= 0
# help/version tests
./test/hail --version
>>>= 0
./test/hail --help
>>>= 0
./test/hbal --version
>>>= 0
./test/hbal --help
>>>= 0
./test/hspace --version
>>>= 0
./test/hspace --help
>>>= 0
./test/hscan --version
>>>= 0
./test/hscan --help
>>>= 0
./test/hinfo --version
>>>= 0
./test/hinfo --help
>>>= 0
echo a > $T/dynu; ./test/hbal -U $T/dynu $BACKEND_DYNU
>>>2 /Cannot parse line/
>>>= !0
echo a b c d e f g h > $T/dynu; ./test/hbal -U $T/dynu $BACKEND_DYNU
>>>2 /Cannot parse line/
>>>= !0
echo inst cpu mem dsk net >$T/dynu; ./test/hbal -U $T/dynu $BACKEND_DYNU
>>>2 /cannot parse string '(cpu|mem|dsk|net)'/
>>>= !0
# unknown instances are currently just ignored
echo no-such-inst 2 2 2 2 > $T/dynu; ./test/hbal -U $T/dynu $BACKEND_DYNU
>>>= 0
# new-0 is the name of the first instance allocated by hspace
echo new-0 2 2 2 2 > $T/dynu; ./test/hbal -U $T/dynu $BACKEND_DYNU
>>>= 0
./test/hbal $BACKEND_EXCL --exclude-instances no-such-instance
>>>2 /Unknown instance/
>>>= !0
./test/hbal $BACKEND_EXCL --select-instances no-such-instances
>>>2 /Unknown instance/
>>>= !0
./test/hbal $BACKEND_EXCL --exclude-instances new-0 --select-instances new-1
>>>= 0
# test that on invalid files it can't parse the request
./test/hail /dev/null
>>>2 /Invalid JSON/
>>>= !0
# another invalid example
echo '[]' | ./test/hail -
>>>2 /Unable to read JSObject/
>>>= !0
# empty dict
echo '{}' | ./test/hail -
>>>2 /key 'request' not found/
>>>= !0
echo '{"request": 0}' | ./test/hail -
>>>2 /key 'request'/
>>>= !0
./test/hail $TESTDATA_DIR/hail-invalid-reloc.json
>>>2 /key 'name': Unable to read String/
>>>= !0
# and now start the real tests
./test/hail $TESTDATA_DIR/hail-alloc-drbd.json
>>> /"success":true,.*,"result":\["node2","node1"\]/
>>>= 0
./test/hail $TESTDATA_DIR/hail-reloc-drbd.json
>>> /"success":true,.*,"result":\["node1"\]/
>>>= 0
./test/hail $TESTDATA_DIR/hail-node-evac.json
>>> /"success":true,"info":"Request successful: 0 instances failed to move and 1 were moved successfully"/
>>>= 0
./test/hail $TESTDATA_DIR/hail-change-group.json
>>> /"success":true,"info":"Request successful: 0 instances failed to move and 1 were moved successfully"/
>>>= 0
# check that hail can use the simu backend
./test/hail --simu p,8,8T,16g,16 $TESTDATA_DIR/hail-alloc-drbd.json
>>> /"success":true,/
>>>= 0
# check that hail can use the text backend
./test/hail -t $T/simu-rebal-merged.standard $TESTDATA_DIR/hail-alloc-drbd.json
>>> /"success":true,/
>>>= 0
# check that hail can use the simu backend
./test/hail -t $T/simu-rebal-merged.standard $TESTDATA_DIR/hail-alloc-drbd.json
>>> /"success":true,/
>>>= 0
# check that hail pre/post saved state differs after allocation
./test/hail -v -v -v -p $TESTDATA_DIR/hail-alloc-drbd.json -S $T/hail-alloc >/dev/null 2>&1 && ! diff -q $T/hail-alloc.pre-ialloc $T/hail-alloc.post-ialloc
>>> /Files .* and .* differ/
>>>= 0
# check that hail pre/post saved state differs after relocation
./test/hail -v -v -v -p $TESTDATA_DIR/hail-reloc-drbd.json -S $T/hail-reloc >/dev/null 2>&1 && ! diff -q $T/hail-reloc.pre-ialloc $T/hail-reloc.post-ialloc
>>> /Files .* and .* differ/
>>>= 0
# evac tests
./test/hail $T/hail-node-evac.json.primary-only
>>> /"success":true,"info":"Request successful: 0 instances failed to move and 1 were moved successfully"/
>>>= 0
./test/hail $T/hail-node-evac.json.secondary-only
>>> /"success":true,"info":"Request successful: 0 instances failed to move and 1 were moved successfully"/
>>>= 0
./test/hail $T/hail-node-evac.json.all
>>> /"success":true,"info":"Request successful: 0 instances failed to move and 1 were moved successfully"/
>>>= 0
# test that hspace machine readable output looks correct
./test/hspace --simu p,4,8T,64g,16 --machine-readable --disk-template drbd -l 8
>>> /^HTS_OK=1/
>>>= 0
# test again via a file and shell parsing
./test/hspace --simu p,4,8T,64g,16 --machine-readable --disk-template drbd -l 8 > $T/capacity && sh -c ". $T/capacity && test x\$HTS_OK = x1"
>>>= 0
# invalid option test
./test/hail --no-such-option
>>>= 2
# invalid option test
./test/hbal --no-such-option
>>>= 2
# invalid option test
./test/hspace --no-such-option
>>>= 2
# invalid option test
./test/hscan --no-such-option
>>>= 2
# invalid option test
./test/hinfo --no-such-option
>>>= 2
# extra arguments
./test/hspace unexpected-argument
>>>2
Error: this program doesn't take any arguments.
>>>=1
./test/hbal unexpected-argument
>>>2
Error: this program doesn't take any arguments.
>>>=1
./test/hinfo unexpected-argument
>>>2
Error: this program doesn't take any arguments.
>>>=1
# standard multi-group tests
./test/hinfo -v -v -p --print-instances -t$T/simu-twogroups.standard
>>>= 0
./test/hbal -t$T/simu-twogroups.standard
>>>= !0
# hbal should not be able to balance
./test/hbal -t$T/simu-twogroups.standard
>>>2 /Found multiple node groups/
>>>= !0
# but hbal should be able to balance one node group
./test/hbal -t$T/simu-twogroups.standard -G group-01
>>>= 0
# and it should not find an invalid group
./test/hbal -t$T/simu-twogroups.standard -G no-such-group
>>>= !0
# tiered allocs multi-group tests
./test/hinfo -v -v -p --print-instances -t$T/simu-twogroups.tiered
>>>= 0
./test/hbal -t$T/simu-twogroups.tiered
>>>= !0
# hbal should not be able to balance
./test/hbal -t$T/simu-twogroups.tiered
>>>2 /Found multiple node groups/
>>>= !0
# but hbal should be able to balance one node group
./test/hbal -t$T/simu-twogroups.tiered -G group-01
>>>= 0
# and it should not find an invalid group
./test/hbal -t$T/simu-twogroups.tiered -G no-such-group
>>>= !0
# hail no input file
./test/hail
>>>= 1
# hbal no backend
./test/hbal
>>>= 1
# hspace no backend
./test/hspace
>>>= 1
# hinfo no backend
./test/hinfo
>>>= 1
# hbal multiple backends
./test/hbal -t /dev/null -m localhost
>>>2
Error: Only one of the rapi, luxi, and data files options should be given.
>>>= 1
# test loading data via RAPI
./test/hinfo -v -v -p --print-instances -m $RAPI_URL
>>>= 0
./test/hbal -v -v -p --print-instances -m $RAPI_URL
>>>= 0
# this compares generated files from hscan
diff -u $T/hscan/direct.hinfo $T/hscan/fromtext.hinfo
>>>
>>>= 0
# standard single-group tests
./test/hinfo -v -v -p --print-instances -t$T/simu-onegroup.standard
>>>= 0
./test/hbal -v -v -p --print-instances -t$T/simu-onegroup.standard
>>>= 0
# hbal should not be able to balance
./test/hbal -t$T/simu-onegroup.standard
>>> /(Nothing to do, exiting|No solution found)/
>>>= 0
# tiered single-group tests
./test/hinfo -v -v -p --print-instances -t$T/simu-onegroup.tiered
>>>= 0
./test/hbal -v -v -p --print-instances -t$T/simu-onegroup.tiered
>>>= 0
# hbal should not be able to balance
./test/hbal -t$T/simu-onegroup.tiered
>>> /(Nothing to do, exiting|No solution found)/
>>>= 0
# missing resources test
./test/hbal -t $TESTDATA_DIR/missing-resources.data
>>>2 /node node2 is missing .* ram and .* disk/
>>>= 0
./test/hinfo -t $TESTDATA_DIR/missing-resources.data
>>>2 /node node2 is missing .* ram and .* disk/
>>>= 0
# common suffix test
./test/hbal -t $TESTDATA_DIR/common-suffix.data -v -v
>>>/Stripping common suffix of '\.example\.com' from names/
>>>= 0