diff --git a/.tests/mutate.sh b/.tests/mutate.sh new file mode 100755 index 0000000..7d63235 --- /dev/null +++ b/.tests/mutate.sh @@ -0,0 +1,165 @@ +#!/bin/sh +# Mutation testing for dotup. Break the code on purpose; a suite that still +# passes has not been measuring what it claims to. +# +# sh .tests/mutate.sh every mutation in mutations/ +# sh .tests/mutate.sh 02 05 13 just those +# sh .tests/mutate.sh --list what each one does and why it matters +# sh .tests/mutate.sh --keep leave the mutated tree behind on failure +# +# Each file in mutations/ is one deliberate bug: a header saying what it breaks +# and why that matters, the ASSERTION that is supposed to catch it, an OLD +# block that must appear EXACTLY ONCE in dot_local/bin/executable_dotup, and +# the NEW text to put in its place. A +# mutation whose OLD block no longer matches is reported as STALE rather than +# quietly skipped -- the code moved, and the mutation has to move with it. +# +# Nothing here touches the working tree. The whole repo is copied once, the +# copy's dotup is mutated, the copy's own tests run against it, and the copy's +# dotup is restored from a pristine byte-for-byte spare between runs. +# +# A mutation is KILLED when the suite it names fails AND the named assertion is +# one of the failures. A suite that fails for some unrelated reason is reported +# as WRONG-TEST: the bug was noticed by accident, which is not the same as being +# tested for, and the next edit to the unrelated test would lose it. SURVIVED +# means the bug is live and nothing noticed at all. +set -eu +cd "$(dirname "$0")" +HERE=$PWD +ROOT=$(CDPATH= cd -- .. && pwd) +MUTS=$HERE/mutations + +KEEP=0; only=; LIST=0 +while [ $# -gt 0 ]; do + case $1 in + --keep) KEEP=1 ;; + --list) LIST=1 ;; + -*) echo "unknown flag $1" >&2; exit 2 ;; + *) only="$only $1" ;; + esac + shift +done + +G=$(printf '\033[32m'); R_=$(printf '\033[31m'); Y=$(printf '\033[33m') +B=$(printf '\033[1m'); Z=$(printf '\033[0m'); DIM=$(printf '\033[2m') + +if [ "$LIST" = 1 ]; then + for f in "$MUTS"/*.mut; do + printf '%s%s%s\n' "$B" "$(basename "$f" .mut)" "$Z" + sed -n 's/^# what: / breaks /p; s/^# why: / matters /p + s/^# kills: / caught by /p; s/^# run: / suite /p' "$f" + done + exit 0 +fi + +WORK=${TMPDIR:-/tmp}/dotup-mutate.$$ +cleanup() { [ "$KEEP" = 1 ] || rm -rf "$WORK"; } +trap cleanup EXIT INT TERM + +echo "== copying the tree ==" +mkdir -p "$WORK" +tar -C "$ROOT" --exclude=.git --exclude=tmp --exclude='.tests/state' -cf - . \ + | tar -C "$WORK" -xf - +D=$WORK/dot_local/bin/executable_dotup +cp "$D" "$WORK/.dotup.pristine" + +# ---------------------------------------------------------------- runners --- +# Both return 0 when the suite PASSED, which for a mutated tree is the bad news. +run_unit() { ( cd "$WORK/.tests" && sh test.sh ) >"$WORK/out" 2>&1; } +run_lab() { ( cd "$WORK/.tests/lab" && bash run.sh "$1" ) >"$WORK/out" 2>&1; } +run_suite() { + case $1 in + unit) run_unit ;; + lab:*) run_lab "${1#lab:}" ;; + *) echo "unknown suite: $1" >&2; return 125 ;; + esac +} +# What the suite said went wrong, in its own words. This is the evidence that a +# mutation was caught for a REASON rather than by a crash somewhere unrelated. +failures() { + sed 's/\x1b\[[0-9;]*m//g' "$WORK/out" \ + | sed -n 's/^ FAIL \(.*\)$/\1/p; s/^FAIL: \(.*\)$/\1/p; s/^\(TIMEOUT: .*\)$/\1/p' \ + | sed 's/ — .*//' +} +why_failed() { failures | head -3 | tr '\n' ';' | sed 's/;$//; s/;/; /g'; } + +# --------------------------------------------------------------- baseline --- +echo "== baseline: the copy, unmutated ==" +if run_unit; then + base=$(sed 's/\x1b\[[0-9;]*m//g' "$WORK/out" | sed -n 's/^\([0-9]* passed, [0-9]* failed\)$/\1/p' | tail -1) + printf ' %sunit%s %s\n' "$G" "$Z" "$base" +else + sed 's/\x1b\[[0-9;]*m//g' "$WORK/out" | tail -20 + echo "${R_}the suite does not pass on an unmutated tree; nothing below would mean anything.${Z}" + exit 1 +fi + +# --------------------------------------------------------------- mutating --- +killed=0; survived=0; stale=0; wrong=0 +: > "$WORK/table" +for f in "$MUTS"/*.mut; do + id=$(basename "$f" .mut) + if [ -n "$only" ]; then + want=0 + for o in $only; do case $id in $o|$o-*|*"$o"*) want=1 ;; esac; done + [ "$want" = 1 ] || continue + fi + what=$(sed -n 's/^# what: //p' "$f") + suite=$(sed -n 's/^# run: //p' "$f") + kills=$(sed -n 's/^# kills: //p' "$f") + + cp "$WORK/.dotup.pristine" "$D" + if ! python3 - "$f" "$D" <<'PY' +import sys +mut, target = sys.argv[1], sys.argv[2] +raw = open(mut).read() +body = raw.split('<<> "$WORK/table" + stale=$((stale + 1)); continue + fi + + printf ' %s…%s %-38s %s' "$DIM" "$Z" "$id" "$DIM$suite$Z" + if run_suite "$suite"; then + printf '\r %sSURVIVED%s %-38s %s\n' "$R_" "$Z" "$id" "$what" + printf 'SURVIVED\t%s\t%s\t%s\t-\n' "$id" "$what" "$suite" >> "$WORK/table" + survived=$((survived + 1)) + [ "$KEEP" = 1 ] && cp "$WORK/out" "$HERE/mutate-survived-$id.log" + # -e, not a bare pattern: several of these assertions begin with "--", + # which grep would read as its own flags. + elif [ -n "$kills" ] && ! failures | grep -qF -e "$kills"; then + # It broke something, but not the thing that is supposed to be watching + # it. That is luck, and luck is lost the next time the other test moves. + printf '\r %sWRONG-TEST%s %-36s %s\n' "$Y" "$Z" "$id" "expected [$kills], got [$(why_failed)]" + printf 'WRONG-TEST\t%s\t%s\t%s\t%s\n' "$id" "$what" "$suite" "$(why_failed)" >> "$WORK/table" + wrong=$((wrong + 1)) + [ "$KEEP" = 1 ] && cp "$WORK/out" "$HERE/mutate-wrongtest-$id.log" + else + printf '\r %sKILLED%s %-38s %s\n' "$G" "$Z" "$id" "${kills:-$(why_failed)}" + printf 'KILLED\t%s\t%s\t%s\t%s\n' "$id" "$what" "$suite" "${kills:-$(why_failed)}" >> "$WORK/table" + killed=$((killed + 1)) + fi +done +cp "$WORK/.dotup.pristine" "$D" + +total=$((killed + survived + stale + wrong)) +[ "$total" -gt 0 ] || { echo "no mutations selected"; exit 2; } +echo +printf '%sMUTATION\tCATCHES IT\tSUITE\tCAUGHT BY%s\n' "$B" "$Z" +awk -F'\t' '{printf "%-9s %-38s %-34s %s\n", $1, $2, $4, $5}' "$WORK/table" +echo +printf '%smutation score: %d/%d killed%s' "$B" "$killed" "$total" "$Z" +[ "$survived" -eq 0 ] || printf ' %s(%d SURVIVED)%s' "$R_" "$survived" "$Z" +[ "$wrong" -eq 0 ] || printf ' %s(%d wrong-test)%s' "$Y" "$wrong" "$Z" +[ "$stale" -eq 0 ] || printf ' %s(%d stale)%s' "$Y" "$stale" "$Z" +printf '\n\n' +[ "$survived" -eq 0 ] && [ "$stale" -eq 0 ] && [ "$wrong" -eq 0 ] diff --git a/.tests/mutations/01-private-never-a-package.mut b/.tests/mutations/01-private-never-a-package.mut new file mode 100644 index 0000000..58a8213 --- /dev/null +++ b/.tests/mutations/01-private-never-a-package.mut @@ -0,0 +1,8 @@ +# what: the private filter in selected_packages, removed +# why: the "private is never a package" invariant is what makes --unattended safe +# kills: no private row reaches the plan +# run: unit +<</dev/null || : +<<0) sel[l]=1 } + !/^[#@]/ && NF>=3 && $3=="invasive" && (($1"/"$2) in sel) {print $1"/"$2}' "$MANIFEST") + if [ -n "$inv" ]; then + warn "unattended: refusing invasive packages$(printf ' %s' $inv)" + for k in $inv; do + awk -F'\t' -v k="$k" '$3!=k' "$tbl" > "$tbl.f"; mv "$tbl.f" "$tbl" + done + fi + fi + +<<&2 + return 1 +<<&2 + return 0 +<<> "$FAILED"; err "$1: $2"; } +<</dev/null 2>&1 \ +<</dev/null 2>&1 \ +<<