#!/bin/sh # Mutation testing for dotup. Break the code on purpose; a suite that still # passes has not been measuring what it claims to. # # sh .tests/mutate.sh every mutation in mutations/ # sh .tests/mutate.sh 02 05 13 just those # sh .tests/mutate.sh --list what each one does and why it matters # sh .tests/mutate.sh --keep leave the mutated tree behind on failure # # Each file in mutations/ is one deliberate bug: a header saying what it breaks # and why that matters, the ASSERTION that is supposed to catch it, an OLD # block that must appear EXACTLY ONCE in dot_local/bin/executable_dotup, and # the NEW text to put in its place. A # mutation whose OLD block no longer matches is reported as STALE rather than # quietly skipped -- the code moved, and the mutation has to move with it. # # Nothing here touches the working tree. The whole repo is copied once, the # copy's dotup is mutated, the copy's own tests run against it, and the copy's # dotup is restored from a pristine byte-for-byte spare between runs. # # A mutation is KILLED when the suite it names fails AND the named assertion is # one of the failures. A suite that fails for some unrelated reason is reported # as WRONG-TEST: the bug was noticed by accident, which is not the same as being # tested for, and the next edit to the unrelated test would lose it. SURVIVED # means the bug is live and nothing noticed at all. set -eu cd "$(dirname "$0")" HERE=$PWD ROOT=$(CDPATH= cd -- .. && pwd) MUTS=$HERE/mutations KEEP=0; only=; LIST=0 while [ $# -gt 0 ]; do case $1 in --keep) KEEP=1 ;; --list) LIST=1 ;; -*) echo "unknown flag $1" >&2; exit 2 ;; *) only="$only $1" ;; esac shift done G=$(printf '\033[32m'); R_=$(printf '\033[31m'); Y=$(printf '\033[33m') B=$(printf '\033[1m'); Z=$(printf '\033[0m'); DIM=$(printf '\033[2m') if [ "$LIST" = 1 ]; then for f in "$MUTS"/*.mut; do printf '%s%s%s\n' "$B" "$(basename "$f" .mut)" "$Z" sed -n 's/^# what: / breaks /p; s/^# why: / matters /p s/^# kills: / caught by /p; s/^# run: / suite /p' "$f" done exit 0 fi WORK=${TMPDIR:-/tmp}/dotup-mutate.$$ cleanup() { [ "$KEEP" = 1 ] || rm -rf "$WORK"; } trap cleanup EXIT INT TERM echo "== copying the tree ==" mkdir -p "$WORK" tar -C "$ROOT" --exclude=.git --exclude=tmp --exclude='.tests/state' -cf - . \ | tar -C "$WORK" -xf - D=$WORK/dot_local/bin/executable_dotup cp "$D" "$WORK/.dotup.pristine" # ---------------------------------------------------------------- runners --- # Both return 0 when the suite PASSED, which for a mutated tree is the bad news. run_unit() { ( cd "$WORK/.tests" && sh test.sh ) >"$WORK/out" 2>&1; } run_lab() { ( cd "$WORK/.tests/lab" && bash run.sh "$1" ) >"$WORK/out" 2>&1; } run_suite() { case $1 in unit) run_unit ;; lab:*) run_lab "${1#lab:}" ;; *) echo "unknown suite: $1" >&2; return 125 ;; esac } # What the suite said went wrong, in its own words. This is the evidence that a # mutation was caught for a REASON rather than by a crash somewhere unrelated. failures() { sed 's/\x1b\[[0-9;]*m//g' "$WORK/out" \ | sed -n 's/^ FAIL \(.*\)$/\1/p; s/^FAIL: \(.*\)$/\1/p; s/^\(TIMEOUT: .*\)$/\1/p' \ | sed 's/ — .*//' } why_failed() { failures | head -3 | tr '\n' ';' | sed 's/;$//; s/;/; /g'; } # --------------------------------------------------------------- baseline --- echo "== baseline: the copy, unmutated ==" if run_unit; then base=$(sed 's/\x1b\[[0-9;]*m//g' "$WORK/out" | sed -n 's/^\([0-9]* passed, [0-9]* failed\)$/\1/p' | tail -1) printf ' %sunit%s %s\n' "$G" "$Z" "$base" else sed 's/\x1b\[[0-9;]*m//g' "$WORK/out" | tail -20 echo "${R_}the suite does not pass on an unmutated tree; nothing below would mean anything.${Z}" exit 1 fi # --------------------------------------------------------------- mutating --- killed=0; survived=0; stale=0; wrong=0 : > "$WORK/table" for f in "$MUTS"/*.mut; do id=$(basename "$f" .mut) if [ -n "$only" ]; then want=0 for o in $only; do case $id in $o|$o-*|*"$o"*) want=1 ;; esac; done [ "$want" = 1 ] || continue fi what=$(sed -n 's/^# what: //p' "$f") suite=$(sed -n 's/^# run: //p' "$f") kills=$(sed -n 's/^# kills: //p' "$f") cp "$WORK/.dotup.pristine" "$D" if ! python3 - "$f" "$D" <<'PY' import sys mut, target = sys.argv[1], sys.argv[2] raw = open(mut).read() body = raw.split('<<> "$WORK/table" stale=$((stale + 1)); continue fi printf ' %s…%s %-38s %s' "$DIM" "$Z" "$id" "$DIM$suite$Z" if run_suite "$suite"; then printf '\r %sSURVIVED%s %-38s %s\n' "$R_" "$Z" "$id" "$what" printf 'SURVIVED\t%s\t%s\t%s\t-\n' "$id" "$what" "$suite" >> "$WORK/table" survived=$((survived + 1)) [ "$KEEP" = 1 ] && cp "$WORK/out" "$HERE/mutate-survived-$id.log" # -e, not a bare pattern: several of these assertions begin with "--", # which grep would read as its own flags. elif [ -n "$kills" ] && ! failures | grep -qF -e "$kills"; then # It broke something, but not the thing that is supposed to be watching # it. That is luck, and luck is lost the next time the other test moves. printf '\r %sWRONG-TEST%s %-36s %s\n' "$Y" "$Z" "$id" "expected [$kills], got [$(why_failed)]" printf 'WRONG-TEST\t%s\t%s\t%s\t%s\n' "$id" "$what" "$suite" "$(why_failed)" >> "$WORK/table" wrong=$((wrong + 1)) [ "$KEEP" = 1 ] && cp "$WORK/out" "$HERE/mutate-wrongtest-$id.log" else printf '\r %sKILLED%s %-38s %s\n' "$G" "$Z" "$id" "${kills:-$(why_failed)}" printf 'KILLED\t%s\t%s\t%s\t%s\n' "$id" "$what" "$suite" "${kills:-$(why_failed)}" >> "$WORK/table" killed=$((killed + 1)) fi done cp "$WORK/.dotup.pristine" "$D" total=$((killed + survived + stale + wrong)) [ "$total" -gt 0 ] || { echo "no mutations selected"; exit 2; } echo printf '%sMUTATION\tCATCHES IT\tSUITE\tCAUGHT BY%s\n' "$B" "$Z" awk -F'\t' '{printf "%-9s %-38s %-34s %s\n", $1, $2, $4, $5}' "$WORK/table" echo printf '%smutation score: %d/%d killed%s' "$B" "$killed" "$total" "$Z" [ "$survived" -eq 0 ] || printf ' %s(%d SURVIVED)%s' "$R_" "$survived" "$Z" [ "$wrong" -eq 0 ] || printf ' %s(%d wrong-test)%s' "$Y" "$wrong" "$Z" [ "$stale" -eq 0 ] || printf ' %s(%d stale)%s' "$Y" "$stale" "$Z" printf '\n\n' [ "$survived" -eq 0 ] && [ "$stale" -eq 0 ] && [ "$wrong" -eq 0 ]