test: mutation testing — break dotup on purpose and check the suite notices
Each file in .tests/mutations/ is one deliberate bug: what it breaks, why that matters, the assertion meant to catch it, and an OLD/NEW pair applied to a copy of the tree. A mutation is KILLED only when the suite fails AND the named assertion is among the failures — failing for an unrelated reason is reported as WRONG-TEST, because that is luck, and luck is lost the next time the unrelated test moves. A mutation whose OLD block no longer matches is STALE rather than quietly skipped. 19 reproduces the DU-C1 shape: a comment moved back inside the fzf line-continuation. It is killed by "dotup pick exits 0".
This commit is contained in:
bcherb2
Executable
+165
@@ -0,0 +1,165 @@
|
|||||||
|
#!/bin/sh
|
||||||
|
# Mutation testing for dotup. Break the code on purpose; a suite that still
|
||||||
|
# passes has not been measuring what it claims to.
|
||||||
|
#
|
||||||
|
# sh .tests/mutate.sh every mutation in mutations/
|
||||||
|
# sh .tests/mutate.sh 02 05 13 just those
|
||||||
|
# sh .tests/mutate.sh --list what each one does and why it matters
|
||||||
|
# sh .tests/mutate.sh --keep leave the mutated tree behind on failure
|
||||||
|
#
|
||||||
|
# Each file in mutations/ is one deliberate bug: a header saying what it breaks
|
||||||
|
# and why that matters, the ASSERTION that is supposed to catch it, an OLD
|
||||||
|
# block that must appear EXACTLY ONCE in dot_local/bin/executable_dotup, and
|
||||||
|
# the NEW text to put in its place. A
|
||||||
|
# mutation whose OLD block no longer matches is reported as STALE rather than
|
||||||
|
# quietly skipped -- the code moved, and the mutation has to move with it.
|
||||||
|
#
|
||||||
|
# Nothing here touches the working tree. The whole repo is copied once, the
|
||||||
|
# copy's dotup is mutated, the copy's own tests run against it, and the copy's
|
||||||
|
# dotup is restored from a pristine byte-for-byte spare between runs.
|
||||||
|
#
|
||||||
|
# A mutation is KILLED when the suite it names fails AND the named assertion is
|
||||||
|
# one of the failures. A suite that fails for some unrelated reason is reported
|
||||||
|
# as WRONG-TEST: the bug was noticed by accident, which is not the same as being
|
||||||
|
# tested for, and the next edit to the unrelated test would lose it. SURVIVED
|
||||||
|
# means the bug is live and nothing noticed at all.
|
||||||
|
set -eu
|
||||||
|
cd "$(dirname "$0")"
|
||||||
|
HERE=$PWD
|
||||||
|
ROOT=$(CDPATH= cd -- .. && pwd)
|
||||||
|
MUTS=$HERE/mutations
|
||||||
|
|
||||||
|
KEEP=0; only=; LIST=0
|
||||||
|
while [ $# -gt 0 ]; do
|
||||||
|
case $1 in
|
||||||
|
--keep) KEEP=1 ;;
|
||||||
|
--list) LIST=1 ;;
|
||||||
|
-*) echo "unknown flag $1" >&2; exit 2 ;;
|
||||||
|
*) only="$only $1" ;;
|
||||||
|
esac
|
||||||
|
shift
|
||||||
|
done
|
||||||
|
|
||||||
|
G=$(printf '\033[32m'); R_=$(printf '\033[31m'); Y=$(printf '\033[33m')
|
||||||
|
B=$(printf '\033[1m'); Z=$(printf '\033[0m'); DIM=$(printf '\033[2m')
|
||||||
|
|
||||||
|
if [ "$LIST" = 1 ]; then
|
||||||
|
for f in "$MUTS"/*.mut; do
|
||||||
|
printf '%s%s%s\n' "$B" "$(basename "$f" .mut)" "$Z"
|
||||||
|
sed -n 's/^# what: / breaks /p; s/^# why: / matters /p
|
||||||
|
s/^# kills: / caught by /p; s/^# run: / suite /p' "$f"
|
||||||
|
done
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
WORK=${TMPDIR:-/tmp}/dotup-mutate.$$
|
||||||
|
cleanup() { [ "$KEEP" = 1 ] || rm -rf "$WORK"; }
|
||||||
|
trap cleanup EXIT INT TERM
|
||||||
|
|
||||||
|
echo "== copying the tree =="
|
||||||
|
mkdir -p "$WORK"
|
||||||
|
tar -C "$ROOT" --exclude=.git --exclude=tmp --exclude='.tests/state' -cf - . \
|
||||||
|
| tar -C "$WORK" -xf -
|
||||||
|
D=$WORK/dot_local/bin/executable_dotup
|
||||||
|
cp "$D" "$WORK/.dotup.pristine"
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------- runners ---
|
||||||
|
# Both return 0 when the suite PASSED, which for a mutated tree is the bad news.
|
||||||
|
run_unit() { ( cd "$WORK/.tests" && sh test.sh ) >"$WORK/out" 2>&1; }
|
||||||
|
run_lab() { ( cd "$WORK/.tests/lab" && bash run.sh "$1" ) >"$WORK/out" 2>&1; }
|
||||||
|
run_suite() {
|
||||||
|
case $1 in
|
||||||
|
unit) run_unit ;;
|
||||||
|
lab:*) run_lab "${1#lab:}" ;;
|
||||||
|
*) echo "unknown suite: $1" >&2; return 125 ;;
|
||||||
|
esac
|
||||||
|
}
|
||||||
|
# What the suite said went wrong, in its own words. This is the evidence that a
|
||||||
|
# mutation was caught for a REASON rather than by a crash somewhere unrelated.
|
||||||
|
failures() {
|
||||||
|
sed 's/\x1b\[[0-9;]*m//g' "$WORK/out" \
|
||||||
|
| sed -n 's/^ FAIL \(.*\)$/\1/p; s/^FAIL: \(.*\)$/\1/p; s/^\(TIMEOUT: .*\)$/\1/p' \
|
||||||
|
| sed 's/ — .*//'
|
||||||
|
}
|
||||||
|
why_failed() { failures | head -3 | tr '\n' ';' | sed 's/;$//; s/;/; /g'; }
|
||||||
|
|
||||||
|
# --------------------------------------------------------------- baseline ---
|
||||||
|
echo "== baseline: the copy, unmutated =="
|
||||||
|
if run_unit; then
|
||||||
|
base=$(sed 's/\x1b\[[0-9;]*m//g' "$WORK/out" | sed -n 's/^\([0-9]* passed, [0-9]* failed\)$/\1/p' | tail -1)
|
||||||
|
printf ' %sunit%s %s\n' "$G" "$Z" "$base"
|
||||||
|
else
|
||||||
|
sed 's/\x1b\[[0-9;]*m//g' "$WORK/out" | tail -20
|
||||||
|
echo "${R_}the suite does not pass on an unmutated tree; nothing below would mean anything.${Z}"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
# --------------------------------------------------------------- mutating ---
|
||||||
|
killed=0; survived=0; stale=0; wrong=0
|
||||||
|
: > "$WORK/table"
|
||||||
|
for f in "$MUTS"/*.mut; do
|
||||||
|
id=$(basename "$f" .mut)
|
||||||
|
if [ -n "$only" ]; then
|
||||||
|
want=0
|
||||||
|
for o in $only; do case $id in $o|$o-*|*"$o"*) want=1 ;; esac; done
|
||||||
|
[ "$want" = 1 ] || continue
|
||||||
|
fi
|
||||||
|
what=$(sed -n 's/^# what: //p' "$f")
|
||||||
|
suite=$(sed -n 's/^# run: //p' "$f")
|
||||||
|
kills=$(sed -n 's/^# kills: //p' "$f")
|
||||||
|
|
||||||
|
cp "$WORK/.dotup.pristine" "$D"
|
||||||
|
if ! python3 - "$f" "$D" <<'PY'
|
||||||
|
import sys
|
||||||
|
mut, target = sys.argv[1], sys.argv[2]
|
||||||
|
raw = open(mut).read()
|
||||||
|
body = raw.split('<<<OLD\n', 1)[1]
|
||||||
|
old, rest = body.split('<<<NEW\n', 1)
|
||||||
|
new = rest.split('<<<END\n', 1)[0]
|
||||||
|
src = open(target).read()
|
||||||
|
n = src.count(old)
|
||||||
|
if n != 1:
|
||||||
|
sys.exit("STALE: OLD block appears %d times, want exactly 1" % n)
|
||||||
|
open(target, 'w').write(src.replace(old, new, 1))
|
||||||
|
PY
|
||||||
|
then
|
||||||
|
printf ' %sSTALE%s %-38s %s\n' "$Y" "$Z" "$id" "$what"
|
||||||
|
printf 'STALE\t%s\t%s\t%s\t-\n' "$id" "$what" "$suite" >> "$WORK/table"
|
||||||
|
stale=$((stale + 1)); continue
|
||||||
|
fi
|
||||||
|
|
||||||
|
printf ' %s…%s %-38s %s' "$DIM" "$Z" "$id" "$DIM$suite$Z"
|
||||||
|
if run_suite "$suite"; then
|
||||||
|
printf '\r %sSURVIVED%s %-38s %s\n' "$R_" "$Z" "$id" "$what"
|
||||||
|
printf 'SURVIVED\t%s\t%s\t%s\t-\n' "$id" "$what" "$suite" >> "$WORK/table"
|
||||||
|
survived=$((survived + 1))
|
||||||
|
[ "$KEEP" = 1 ] && cp "$WORK/out" "$HERE/mutate-survived-$id.log"
|
||||||
|
# -e, not a bare pattern: several of these assertions begin with "--",
|
||||||
|
# which grep would read as its own flags.
|
||||||
|
elif [ -n "$kills" ] && ! failures | grep -qF -e "$kills"; then
|
||||||
|
# It broke something, but not the thing that is supposed to be watching
|
||||||
|
# it. That is luck, and luck is lost the next time the other test moves.
|
||||||
|
printf '\r %sWRONG-TEST%s %-36s %s\n' "$Y" "$Z" "$id" "expected [$kills], got [$(why_failed)]"
|
||||||
|
printf 'WRONG-TEST\t%s\t%s\t%s\t%s\n' "$id" "$what" "$suite" "$(why_failed)" >> "$WORK/table"
|
||||||
|
wrong=$((wrong + 1))
|
||||||
|
[ "$KEEP" = 1 ] && cp "$WORK/out" "$HERE/mutate-wrongtest-$id.log"
|
||||||
|
else
|
||||||
|
printf '\r %sKILLED%s %-38s %s\n' "$G" "$Z" "$id" "${kills:-$(why_failed)}"
|
||||||
|
printf 'KILLED\t%s\t%s\t%s\t%s\n' "$id" "$what" "$suite" "${kills:-$(why_failed)}" >> "$WORK/table"
|
||||||
|
killed=$((killed + 1))
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
cp "$WORK/.dotup.pristine" "$D"
|
||||||
|
|
||||||
|
total=$((killed + survived + stale + wrong))
|
||||||
|
[ "$total" -gt 0 ] || { echo "no mutations selected"; exit 2; }
|
||||||
|
echo
|
||||||
|
printf '%sMUTATION\tCATCHES IT\tSUITE\tCAUGHT BY%s\n' "$B" "$Z"
|
||||||
|
awk -F'\t' '{printf "%-9s %-38s %-34s %s\n", $1, $2, $4, $5}' "$WORK/table"
|
||||||
|
echo
|
||||||
|
printf '%smutation score: %d/%d killed%s' "$B" "$killed" "$total" "$Z"
|
||||||
|
[ "$survived" -eq 0 ] || printf ' %s(%d SURVIVED)%s' "$R_" "$survived" "$Z"
|
||||||
|
[ "$wrong" -eq 0 ] || printf ' %s(%d wrong-test)%s' "$Y" "$wrong" "$Z"
|
||||||
|
[ "$stale" -eq 0 ] || printf ' %s(%d stale)%s' "$Y" "$stale" "$Z"
|
||||||
|
printf '\n\n'
|
||||||
|
[ "$survived" -eq 0 ] && [ "$stale" -eq 0 ] && [ "$wrong" -eq 0 ]
|
||||||
@@ -0,0 +1,8 @@
|
|||||||
|
# what: the private filter in selected_packages, removed
|
||||||
|
# why: the "private is never a package" invariant is what makes --unattended safe
|
||||||
|
# kills: no private row reaches the plan
|
||||||
|
# run: unit
|
||||||
|
<<<OLD
|
||||||
|
$3=="private" { next }
|
||||||
|
<<<NEW
|
||||||
|
<<<END
|
||||||
@@ -0,0 +1,9 @@
|
|||||||
|
# what: the bws checksum comparison, inverted: accepts on mismatch, rejects on match
|
||||||
|
# why: the binary is about to hold the key to every other credential
|
||||||
|
# kills: a checksum mismatch refuses to install
|
||||||
|
# run: unit
|
||||||
|
<<<OLD
|
||||||
|
if [ -n "$want" ] && [ "$want" != "$got" ]; then
|
||||||
|
<<<NEW
|
||||||
|
if [ -n "$want" ] && [ "$want" = "$got" ]; then
|
||||||
|
<<<END
|
||||||
@@ -0,0 +1,9 @@
|
|||||||
|
# what: the bws token written 644 instead of 600
|
||||||
|
# why: any account on the box could then read the machine credential
|
||||||
|
# kills: the bws token is written 600
|
||||||
|
# run: unit
|
||||||
|
<<<OLD
|
||||||
|
chmod 600 "$BWS_TOKEN"
|
||||||
|
<<<NEW
|
||||||
|
chmod 644 "$BWS_TOKEN"
|
||||||
|
<<<END
|
||||||
@@ -0,0 +1,8 @@
|
|||||||
|
# what: the chmod -R go-rwx on the cloned private source, removed
|
||||||
|
# why: the clone lands at the caller umask -- 022 on stock Ubuntu -- holding ssh config and machine identity
|
||||||
|
# kills: the private source is unreadable to anyone else
|
||||||
|
# run: unit
|
||||||
|
<<<OLD
|
||||||
|
[ ! -d "$PRIV_SRC" ] || chmod -R go-rwx "$PRIV_SRC" 2>/dev/null || :
|
||||||
|
<<<NEW
|
||||||
|
<<<END
|
||||||
@@ -0,0 +1,13 @@
|
|||||||
|
# what: the password retry loop: any non-200 is fatal again
|
||||||
|
# why: these credentials are asked for after every package is installed; one typo used to cost the whole install
|
||||||
|
# kills: …then asks again, keeping the URL
|
||||||
|
# run: unit
|
||||||
|
<<<OLD
|
||||||
|
P_BLOB=''
|
||||||
|
say " Try again, or type q at the URL prompt to stay public-only."
|
||||||
|
done
|
||||||
|
<<<NEW
|
||||||
|
P_BLOB=''
|
||||||
|
return 1
|
||||||
|
done
|
||||||
|
<<<END
|
||||||
@@ -0,0 +1,11 @@
|
|||||||
|
# what: cmd_private's unattended guard, inverted
|
||||||
|
# why: an unattended run would sit at a password prompt nobody is there to answer
|
||||||
|
# kills: --unattended: nobody is here to type it
|
||||||
|
# run: unit
|
||||||
|
<<<OLD
|
||||||
|
if [ "$UNATTENDED" -eq 1 ]; then
|
||||||
|
warn "unattended: the private tier needs a password nobody is here to type — skipped"
|
||||||
|
<<<NEW
|
||||||
|
if [ "$UNATTENDED" -ne 1 ]; then
|
||||||
|
warn "unattended: the private tier needs a password nobody is here to type — skipped"
|
||||||
|
<<<END
|
||||||
@@ -0,0 +1,9 @@
|
|||||||
|
# what: cmd_private's terminal guard, inverted
|
||||||
|
# why: the private tier would run with no terminal, reading EOF for a password
|
||||||
|
# kills: no tty: it says THAT instead
|
||||||
|
# run: unit
|
||||||
|
<<<OLD
|
||||||
|
[ -t 0 ] || { warn "no terminal: the private tier needs a password — skipped"; return 0; }
|
||||||
|
<<<NEW
|
||||||
|
[ ! -t 0 ] || { warn "no terminal: the private tier needs a password — skipped"; return 0; }
|
||||||
|
<<<END
|
||||||
@@ -0,0 +1,19 @@
|
|||||||
|
# what: cmd_install's unattended invasive refusal, deleted
|
||||||
|
# why: a stale state file would install a docker daemon on a machine with nobody at the keyboard
|
||||||
|
# kills: the daemon never reaches a package manager
|
||||||
|
# run: unit
|
||||||
|
<<<OLD
|
||||||
|
if [ "$UNATTENDED" -eq 1 ]; then
|
||||||
|
inv=$(awk -F'\t' -v selfile="$SEL" '
|
||||||
|
BEGIN{ while((getline l < selfile)>0) sel[l]=1 }
|
||||||
|
!/^[#@]/ && NF>=3 && $3=="invasive" && (($1"/"$2) in sel) {print $1"/"$2}' "$MANIFEST")
|
||||||
|
if [ -n "$inv" ]; then
|
||||||
|
warn "unattended: refusing invasive packages$(printf ' %s' $inv)"
|
||||||
|
for k in $inv; do
|
||||||
|
awk -F'\t' -v k="$k" '$3!=k' "$tbl" > "$tbl.f"; mv "$tbl.f" "$tbl"
|
||||||
|
done
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
<<<NEW
|
||||||
|
<<<END
|
||||||
@@ -0,0 +1,11 @@
|
|||||||
|
# what: cmd_install always returns 0
|
||||||
|
# why: a CI run or a wrapper script cannot tell a clean install from six failures
|
||||||
|
# kills: the run exits non-zero
|
||||||
|
# run: unit
|
||||||
|
<<<OLD
|
||||||
|
printf '\n%s%d package(s) did not install.%s Everything else did.\n' "$YEL" "$n" "$R" >&2
|
||||||
|
return 1
|
||||||
|
<<<NEW
|
||||||
|
printf '\n%s%d package(s) did not install.%s Everything else did.\n' "$YEL" "$n" "$R" >&2
|
||||||
|
return 0
|
||||||
|
<<<END
|
||||||
@@ -0,0 +1,9 @@
|
|||||||
|
# what: note_fail() neutered: failures are dropped
|
||||||
|
# why: the summary, the count and the exit code all come from this one file
|
||||||
|
# kills: …under a heading you can find
|
||||||
|
# run: unit
|
||||||
|
<<<OLD
|
||||||
|
note_fail() { printf '%s\t%s\n' "$1" "$2" >> "$FAILED"; err "$1: $2"; }
|
||||||
|
<<<NEW
|
||||||
|
note_fail() { :; }
|
||||||
|
<<<END
|
||||||
@@ -0,0 +1,9 @@
|
|||||||
|
# what: ensure_chezmoi's installer call replaced with true
|
||||||
|
# why: the private tier died with `chezmoi: not found` AFTER writing the bws token
|
||||||
|
# kills: the installer call did not install anything
|
||||||
|
# run: lab:scenarios/31-chezmoi-absent.sh
|
||||||
|
<<<OLD
|
||||||
|
sh -c "$(curl -fsLS get.chezmoi.io)" -- -b "$HOME/.local/bin" >/dev/null 2>&1 \
|
||||||
|
<<<NEW
|
||||||
|
true >/dev/null 2>&1 \
|
||||||
|
<<<END
|
||||||
@@ -0,0 +1,9 @@
|
|||||||
|
# what: find_tool no longer looks in ~/.local/bin
|
||||||
|
# why: uv, bws and chezmoi all land there, on no PATH the running process has
|
||||||
|
# kills: uv is found in ~/.local/bin, off PATH
|
||||||
|
# run: unit
|
||||||
|
<<<OLD
|
||||||
|
for ft_c in "$HOME/.local/bin/$1" "$HOME/bin/$1" "$HOME/.npm-global/bin/$1" \
|
||||||
|
<<<NEW
|
||||||
|
for ft_c in "$HOME/bin/$1" "$HOME/.npm-global/bin/$1" \
|
||||||
|
<<<END
|
||||||
@@ -0,0 +1,11 @@
|
|||||||
|
# what: the apt one-at-a-time retry, removed
|
||||||
|
# why: apt refuses the whole batch when one name is unusable; the retry is the difference between 34 installed and 0
|
||||||
|
# kills: …then btop on its own
|
||||||
|
# run: unit
|
||||||
|
<<<OLD
|
||||||
|
for p in $keep; do
|
||||||
|
run_sh "${SUDO:+$SUDO }DEBIAN_FRONTEND=noninteractive apt-get install -y $p" \
|
||||||
|
|| note_fail "$p" "apt install failed"
|
||||||
|
done
|
||||||
|
<<<NEW
|
||||||
|
<<<END
|
||||||
@@ -0,0 +1,8 @@
|
|||||||
|
# what: the npm one-at-a-time retry, removed
|
||||||
|
# why: same shape as apt: one bad spec takes every other tool down with it
|
||||||
|
# kills: …then @openai/codex on its own
|
||||||
|
# run: unit
|
||||||
|
<<<OLD
|
||||||
|
for p in $specs; do run_sh "$NPM install -g $p" || note_fail "$(keyof npm "$p")" "npm install failed"; done
|
||||||
|
<<<NEW
|
||||||
|
<<<END
|
||||||
@@ -0,0 +1,9 @@
|
|||||||
|
# what: install_fzf's base URL pointed at a host that does not exist
|
||||||
|
# why: the picker cannot draw the list that installs fzf without fzf
|
||||||
|
# kills: the fetch goes to the fzf release download URL
|
||||||
|
# run: unit
|
||||||
|
<<<OLD
|
||||||
|
base=https://github.com/junegunn/fzf/releases/download
|
||||||
|
<<<NEW
|
||||||
|
base=https://github.invalid/junegunn/fzf/releases/download
|
||||||
|
<<<END
|
||||||
@@ -0,0 +1,9 @@
|
|||||||
|
# what: the neovim tarball extracted to /opt/nvim-TYPO
|
||||||
|
# why: tar succeeds, the symlink dangles, and nothing says so
|
||||||
|
# kills: the tarball is extracted where the symlink points
|
||||||
|
# run: unit
|
||||||
|
<<<OLD
|
||||||
|
run_sh "${SUDO:+$SUDO }rm -rf /opt/nvim && ${SUDO:+$SUDO }mkdir -p /opt/nvim && ${SUDO:+$SUDO }tar -xzf /tmp/nvim.tgz -C /opt/nvim --strip-components=1" \
|
||||||
|
<<<NEW
|
||||||
|
run_sh "${SUDO:+$SUDO }rm -rf /opt/nvim && ${SUDO:+$SUDO }mkdir -p /opt/nvim && ${SUDO:+$SUDO }tar -xzf /tmp/nvim.tgz -C /opt/nvim-TYPO --strip-components=1" \
|
||||||
|
<<<END
|
||||||
@@ -0,0 +1,8 @@
|
|||||||
|
# what: install_deb's install step, removed: it downloads and stops
|
||||||
|
# why: chrome and ghostty would report success having installed nothing
|
||||||
|
# kills: …and then handed to the package manager
|
||||||
|
# run: unit
|
||||||
|
<<<OLD
|
||||||
|
run_sh "${SUDO:+$SUDO }apt-get install -y '$f'" || note_fail "$(keyof deb "$s")" "dpkg install failed"
|
||||||
|
<<<NEW
|
||||||
|
<<<END
|
||||||
@@ -0,0 +1,10 @@
|
|||||||
|
# what: a comment moved back INSIDE the fzf line-continuation (the DU-C1 shape)
|
||||||
|
# why: the `\`-newline is stripped first, so the comment's own newline ends the command: fzf gets two binds and the lines below run as a command named --bind
|
||||||
|
# kills: dotup pick exits 0
|
||||||
|
# run: unit
|
||||||
|
<<<OLD
|
||||||
|
--bind "tab:execute-silent($SELF expand {2})+reload($SELF render)" \
|
||||||
|
<<<NEW
|
||||||
|
--bind "tab:execute-silent($SELF expand {2})+reload($SELF render)" \
|
||||||
|
# clear-query is not cosmetic: ^t toggles every row the filter shows.
|
||||||
|
<<<END
|
||||||
Reference in New Issue
Block a user