bcherb2
d1e3f8bce2
Each file in .tests/mutations/ is one deliberate bug: what it breaks, why that matters, the assertion meant to catch it, and an OLD/NEW pair applied to a copy of the tree. A mutation is KILLED only when the suite fails AND the named assertion is among the failures — failing for an unrelated reason is reported as WRONG-TEST, because that is luck, and luck is lost the next time the unrelated test moves. A mutation whose OLD block no longer matches is STALE rather than quietly skipped. 19 reproduces the DU-C1 shape: a comment moved back inside the fzf line-continuation. It is killed by "dotup pick exits 0".
166 lines
6.5 KiB
Bash
Executable File
166 lines
6.5 KiB
Bash
Executable File
#!/bin/sh
|
|
# Mutation testing for dotup. Break the code on purpose; a suite that still
|
|
# passes has not been measuring what it claims to.
|
|
#
|
|
# sh .tests/mutate.sh every mutation in mutations/
|
|
# sh .tests/mutate.sh 02 05 13 just those
|
|
# sh .tests/mutate.sh --list what each one does and why it matters
|
|
# sh .tests/mutate.sh --keep leave the mutated tree behind on failure
|
|
#
|
|
# Each file in mutations/ is one deliberate bug: a header saying what it breaks
|
|
# and why that matters, the ASSERTION that is supposed to catch it, an OLD
|
|
# block that must appear EXACTLY ONCE in dot_local/bin/executable_dotup, and
|
|
# the NEW text to put in its place. A
|
|
# mutation whose OLD block no longer matches is reported as STALE rather than
|
|
# quietly skipped -- the code moved, and the mutation has to move with it.
|
|
#
|
|
# Nothing here touches the working tree. The whole repo is copied once, the
|
|
# copy's dotup is mutated, the copy's own tests run against it, and the copy's
|
|
# dotup is restored from a pristine byte-for-byte spare between runs.
|
|
#
|
|
# A mutation is KILLED when the suite it names fails AND the named assertion is
|
|
# one of the failures. A suite that fails for some unrelated reason is reported
|
|
# as WRONG-TEST: the bug was noticed by accident, which is not the same as being
|
|
# tested for, and the next edit to the unrelated test would lose it. SURVIVED
|
|
# means the bug is live and nothing noticed at all.
|
|
set -eu
|
|
cd "$(dirname "$0")"
|
|
HERE=$PWD
|
|
ROOT=$(CDPATH= cd -- .. && pwd)
|
|
MUTS=$HERE/mutations
|
|
|
|
KEEP=0; only=; LIST=0
|
|
while [ $# -gt 0 ]; do
|
|
case $1 in
|
|
--keep) KEEP=1 ;;
|
|
--list) LIST=1 ;;
|
|
-*) echo "unknown flag $1" >&2; exit 2 ;;
|
|
*) only="$only $1" ;;
|
|
esac
|
|
shift
|
|
done
|
|
|
|
G=$(printf '\033[32m'); R_=$(printf '\033[31m'); Y=$(printf '\033[33m')
|
|
B=$(printf '\033[1m'); Z=$(printf '\033[0m'); DIM=$(printf '\033[2m')
|
|
|
|
if [ "$LIST" = 1 ]; then
|
|
for f in "$MUTS"/*.mut; do
|
|
printf '%s%s%s\n' "$B" "$(basename "$f" .mut)" "$Z"
|
|
sed -n 's/^# what: / breaks /p; s/^# why: / matters /p
|
|
s/^# kills: / caught by /p; s/^# run: / suite /p' "$f"
|
|
done
|
|
exit 0
|
|
fi
|
|
|
|
WORK=${TMPDIR:-/tmp}/dotup-mutate.$$
|
|
cleanup() { [ "$KEEP" = 1 ] || rm -rf "$WORK"; }
|
|
trap cleanup EXIT INT TERM
|
|
|
|
echo "== copying the tree =="
|
|
mkdir -p "$WORK"
|
|
tar -C "$ROOT" --exclude=.git --exclude=tmp --exclude='.tests/state' -cf - . \
|
|
| tar -C "$WORK" -xf -
|
|
D=$WORK/dot_local/bin/executable_dotup
|
|
cp "$D" "$WORK/.dotup.pristine"
|
|
|
|
# ---------------------------------------------------------------- runners ---
|
|
# Both return 0 when the suite PASSED, which for a mutated tree is the bad news.
|
|
run_unit() { ( cd "$WORK/.tests" && sh test.sh ) >"$WORK/out" 2>&1; }
|
|
run_lab() { ( cd "$WORK/.tests/lab" && bash run.sh "$1" ) >"$WORK/out" 2>&1; }
|
|
run_suite() {
|
|
case $1 in
|
|
unit) run_unit ;;
|
|
lab:*) run_lab "${1#lab:}" ;;
|
|
*) echo "unknown suite: $1" >&2; return 125 ;;
|
|
esac
|
|
}
|
|
# What the suite said went wrong, in its own words. This is the evidence that a
|
|
# mutation was caught for a REASON rather than by a crash somewhere unrelated.
|
|
failures() {
|
|
sed 's/\x1b\[[0-9;]*m//g' "$WORK/out" \
|
|
| sed -n 's/^ FAIL \(.*\)$/\1/p; s/^FAIL: \(.*\)$/\1/p; s/^\(TIMEOUT: .*\)$/\1/p' \
|
|
| sed 's/ — .*//'
|
|
}
|
|
why_failed() { failures | head -3 | tr '\n' ';' | sed 's/;$//; s/;/; /g'; }
|
|
|
|
# --------------------------------------------------------------- baseline ---
|
|
echo "== baseline: the copy, unmutated =="
|
|
if run_unit; then
|
|
base=$(sed 's/\x1b\[[0-9;]*m//g' "$WORK/out" | sed -n 's/^\([0-9]* passed, [0-9]* failed\)$/\1/p' | tail -1)
|
|
printf ' %sunit%s %s\n' "$G" "$Z" "$base"
|
|
else
|
|
sed 's/\x1b\[[0-9;]*m//g' "$WORK/out" | tail -20
|
|
echo "${R_}the suite does not pass on an unmutated tree; nothing below would mean anything.${Z}"
|
|
exit 1
|
|
fi
|
|
|
|
# --------------------------------------------------------------- mutating ---
|
|
killed=0; survived=0; stale=0; wrong=0
|
|
: > "$WORK/table"
|
|
for f in "$MUTS"/*.mut; do
|
|
id=$(basename "$f" .mut)
|
|
if [ -n "$only" ]; then
|
|
want=0
|
|
for o in $only; do case $id in $o|$o-*|*"$o"*) want=1 ;; esac; done
|
|
[ "$want" = 1 ] || continue
|
|
fi
|
|
what=$(sed -n 's/^# what: //p' "$f")
|
|
suite=$(sed -n 's/^# run: //p' "$f")
|
|
kills=$(sed -n 's/^# kills: //p' "$f")
|
|
|
|
cp "$WORK/.dotup.pristine" "$D"
|
|
if ! python3 - "$f" "$D" <<'PY'
|
|
import sys
|
|
mut, target = sys.argv[1], sys.argv[2]
|
|
raw = open(mut).read()
|
|
body = raw.split('<<<OLD\n', 1)[1]
|
|
old, rest = body.split('<<<NEW\n', 1)
|
|
new = rest.split('<<<END\n', 1)[0]
|
|
src = open(target).read()
|
|
n = src.count(old)
|
|
if n != 1:
|
|
sys.exit("STALE: OLD block appears %d times, want exactly 1" % n)
|
|
open(target, 'w').write(src.replace(old, new, 1))
|
|
PY
|
|
then
|
|
printf ' %sSTALE%s %-38s %s\n' "$Y" "$Z" "$id" "$what"
|
|
printf 'STALE\t%s\t%s\t%s\t-\n' "$id" "$what" "$suite" >> "$WORK/table"
|
|
stale=$((stale + 1)); continue
|
|
fi
|
|
|
|
printf ' %s…%s %-38s %s' "$DIM" "$Z" "$id" "$DIM$suite$Z"
|
|
if run_suite "$suite"; then
|
|
printf '\r %sSURVIVED%s %-38s %s\n' "$R_" "$Z" "$id" "$what"
|
|
printf 'SURVIVED\t%s\t%s\t%s\t-\n' "$id" "$what" "$suite" >> "$WORK/table"
|
|
survived=$((survived + 1))
|
|
[ "$KEEP" = 1 ] && cp "$WORK/out" "$HERE/mutate-survived-$id.log"
|
|
# -e, not a bare pattern: several of these assertions begin with "--",
|
|
# which grep would read as its own flags.
|
|
elif [ -n "$kills" ] && ! failures | grep -qF -e "$kills"; then
|
|
# It broke something, but not the thing that is supposed to be watching
|
|
# it. That is luck, and luck is lost the next time the other test moves.
|
|
printf '\r %sWRONG-TEST%s %-36s %s\n' "$Y" "$Z" "$id" "expected [$kills], got [$(why_failed)]"
|
|
printf 'WRONG-TEST\t%s\t%s\t%s\t%s\n' "$id" "$what" "$suite" "$(why_failed)" >> "$WORK/table"
|
|
wrong=$((wrong + 1))
|
|
[ "$KEEP" = 1 ] && cp "$WORK/out" "$HERE/mutate-wrongtest-$id.log"
|
|
else
|
|
printf '\r %sKILLED%s %-38s %s\n' "$G" "$Z" "$id" "${kills:-$(why_failed)}"
|
|
printf 'KILLED\t%s\t%s\t%s\t%s\n' "$id" "$what" "$suite" "${kills:-$(why_failed)}" >> "$WORK/table"
|
|
killed=$((killed + 1))
|
|
fi
|
|
done
|
|
cp "$WORK/.dotup.pristine" "$D"
|
|
|
|
total=$((killed + survived + stale + wrong))
|
|
[ "$total" -gt 0 ] || { echo "no mutations selected"; exit 2; }
|
|
echo
|
|
printf '%sMUTATION\tCATCHES IT\tSUITE\tCAUGHT BY%s\n' "$B" "$Z"
|
|
awk -F'\t' '{printf "%-9s %-38s %-34s %s\n", $1, $2, $4, $5}' "$WORK/table"
|
|
echo
|
|
printf '%smutation score: %d/%d killed%s' "$B" "$killed" "$total" "$Z"
|
|
[ "$survived" -eq 0 ] || printf ' %s(%d SURVIVED)%s' "$R_" "$survived" "$Z"
|
|
[ "$wrong" -eq 0 ] || printf ' %s(%d wrong-test)%s' "$Y" "$wrong" "$Z"
|
|
[ "$stale" -eq 0 ] || printf ' %s(%d stale)%s' "$Y" "$stale" "$Z"
|
|
printf '\n\n'
|
|
[ "$survived" -eq 0 ] && [ "$stale" -eq 0 ] && [ "$wrong" -eq 0 ]
|