test: mutation testing — break dotup on purpose and check the suite notices
Each file in .tests/mutations/ is one deliberate bug: what it breaks, why that matters, the assertion meant to catch it, and an OLD/NEW pair applied to a copy of the tree. A mutation is KILLED only when the suite fails AND the named assertion is among the failures — failing for an unrelated reason is reported as WRONG-TEST, because that is luck, and luck is lost the next time the unrelated test moves. A mutation whose OLD block no longer matches is STALE rather than quietly skipped. 19 reproduces the DU-C1 shape: a comment moved back inside the fzf line-continuation. It is killed by "dotup pick exits 0".
This commit is contained in:
bcherb2
Executable
+165
@@ -0,0 +1,165 @@
|
||||
#!/bin/sh
|
||||
# Mutation testing for dotup. Break the code on purpose; a suite that still
|
||||
# passes has not been measuring what it claims to.
|
||||
#
|
||||
# sh .tests/mutate.sh every mutation in mutations/
|
||||
# sh .tests/mutate.sh 02 05 13 just those
|
||||
# sh .tests/mutate.sh --list what each one does and why it matters
|
||||
# sh .tests/mutate.sh --keep leave the mutated tree behind on failure
|
||||
#
|
||||
# Each file in mutations/ is one deliberate bug: a header saying what it breaks
|
||||
# and why that matters, the ASSERTION that is supposed to catch it, an OLD
|
||||
# block that must appear EXACTLY ONCE in dot_local/bin/executable_dotup, and
|
||||
# the NEW text to put in its place. A
|
||||
# mutation whose OLD block no longer matches is reported as STALE rather than
|
||||
# quietly skipped -- the code moved, and the mutation has to move with it.
|
||||
#
|
||||
# Nothing here touches the working tree. The whole repo is copied once, the
|
||||
# copy's dotup is mutated, the copy's own tests run against it, and the copy's
|
||||
# dotup is restored from a pristine byte-for-byte spare between runs.
|
||||
#
|
||||
# A mutation is KILLED when the suite it names fails AND the named assertion is
|
||||
# one of the failures. A suite that fails for some unrelated reason is reported
|
||||
# as WRONG-TEST: the bug was noticed by accident, which is not the same as being
|
||||
# tested for, and the next edit to the unrelated test would lose it. SURVIVED
|
||||
# means the bug is live and nothing noticed at all.
|
||||
set -eu
|
||||
cd "$(dirname "$0")"
|
||||
HERE=$PWD
|
||||
ROOT=$(CDPATH= cd -- .. && pwd)
|
||||
MUTS=$HERE/mutations
|
||||
|
||||
KEEP=0; only=; LIST=0
|
||||
while [ $# -gt 0 ]; do
|
||||
case $1 in
|
||||
--keep) KEEP=1 ;;
|
||||
--list) LIST=1 ;;
|
||||
-*) echo "unknown flag $1" >&2; exit 2 ;;
|
||||
*) only="$only $1" ;;
|
||||
esac
|
||||
shift
|
||||
done
|
||||
|
||||
G=$(printf '\033[32m'); R_=$(printf '\033[31m'); Y=$(printf '\033[33m')
|
||||
B=$(printf '\033[1m'); Z=$(printf '\033[0m'); DIM=$(printf '\033[2m')
|
||||
|
||||
if [ "$LIST" = 1 ]; then
|
||||
for f in "$MUTS"/*.mut; do
|
||||
printf '%s%s%s\n' "$B" "$(basename "$f" .mut)" "$Z"
|
||||
sed -n 's/^# what: / breaks /p; s/^# why: / matters /p
|
||||
s/^# kills: / caught by /p; s/^# run: / suite /p' "$f"
|
||||
done
|
||||
exit 0
|
||||
fi
|
||||
|
||||
WORK=${TMPDIR:-/tmp}/dotup-mutate.$$
|
||||
cleanup() { [ "$KEEP" = 1 ] || rm -rf "$WORK"; }
|
||||
trap cleanup EXIT INT TERM
|
||||
|
||||
echo "== copying the tree =="
|
||||
mkdir -p "$WORK"
|
||||
tar -C "$ROOT" --exclude=.git --exclude=tmp --exclude='.tests/state' -cf - . \
|
||||
| tar -C "$WORK" -xf -
|
||||
D=$WORK/dot_local/bin/executable_dotup
|
||||
cp "$D" "$WORK/.dotup.pristine"
|
||||
|
||||
# ---------------------------------------------------------------- runners ---
|
||||
# Both return 0 when the suite PASSED, which for a mutated tree is the bad news.
|
||||
run_unit() { ( cd "$WORK/.tests" && sh test.sh ) >"$WORK/out" 2>&1; }
|
||||
run_lab() { ( cd "$WORK/.tests/lab" && bash run.sh "$1" ) >"$WORK/out" 2>&1; }
|
||||
run_suite() {
|
||||
case $1 in
|
||||
unit) run_unit ;;
|
||||
lab:*) run_lab "${1#lab:}" ;;
|
||||
*) echo "unknown suite: $1" >&2; return 125 ;;
|
||||
esac
|
||||
}
|
||||
# What the suite said went wrong, in its own words. This is the evidence that a
|
||||
# mutation was caught for a REASON rather than by a crash somewhere unrelated.
|
||||
failures() {
|
||||
sed 's/\x1b\[[0-9;]*m//g' "$WORK/out" \
|
||||
| sed -n 's/^ FAIL \(.*\)$/\1/p; s/^FAIL: \(.*\)$/\1/p; s/^\(TIMEOUT: .*\)$/\1/p' \
|
||||
| sed 's/ — .*//'
|
||||
}
|
||||
why_failed() { failures | head -3 | tr '\n' ';' | sed 's/;$//; s/;/; /g'; }
|
||||
|
||||
# --------------------------------------------------------------- baseline ---
|
||||
echo "== baseline: the copy, unmutated =="
|
||||
if run_unit; then
|
||||
base=$(sed 's/\x1b\[[0-9;]*m//g' "$WORK/out" | sed -n 's/^\([0-9]* passed, [0-9]* failed\)$/\1/p' | tail -1)
|
||||
printf ' %sunit%s %s\n' "$G" "$Z" "$base"
|
||||
else
|
||||
sed 's/\x1b\[[0-9;]*m//g' "$WORK/out" | tail -20
|
||||
echo "${R_}the suite does not pass on an unmutated tree; nothing below would mean anything.${Z}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# --------------------------------------------------------------- mutating ---
|
||||
killed=0; survived=0; stale=0; wrong=0
|
||||
: > "$WORK/table"
|
||||
for f in "$MUTS"/*.mut; do
|
||||
id=$(basename "$f" .mut)
|
||||
if [ -n "$only" ]; then
|
||||
want=0
|
||||
for o in $only; do case $id in $o|$o-*|*"$o"*) want=1 ;; esac; done
|
||||
[ "$want" = 1 ] || continue
|
||||
fi
|
||||
what=$(sed -n 's/^# what: //p' "$f")
|
||||
suite=$(sed -n 's/^# run: //p' "$f")
|
||||
kills=$(sed -n 's/^# kills: //p' "$f")
|
||||
|
||||
cp "$WORK/.dotup.pristine" "$D"
|
||||
if ! python3 - "$f" "$D" <<'PY'
|
||||
import sys
|
||||
mut, target = sys.argv[1], sys.argv[2]
|
||||
raw = open(mut).read()
|
||||
body = raw.split('<<<OLD\n', 1)[1]
|
||||
old, rest = body.split('<<<NEW\n', 1)
|
||||
new = rest.split('<<<END\n', 1)[0]
|
||||
src = open(target).read()
|
||||
n = src.count(old)
|
||||
if n != 1:
|
||||
sys.exit("STALE: OLD block appears %d times, want exactly 1" % n)
|
||||
open(target, 'w').write(src.replace(old, new, 1))
|
||||
PY
|
||||
then
|
||||
printf ' %sSTALE%s %-38s %s\n' "$Y" "$Z" "$id" "$what"
|
||||
printf 'STALE\t%s\t%s\t%s\t-\n' "$id" "$what" "$suite" >> "$WORK/table"
|
||||
stale=$((stale + 1)); continue
|
||||
fi
|
||||
|
||||
printf ' %s…%s %-38s %s' "$DIM" "$Z" "$id" "$DIM$suite$Z"
|
||||
if run_suite "$suite"; then
|
||||
printf '\r %sSURVIVED%s %-38s %s\n' "$R_" "$Z" "$id" "$what"
|
||||
printf 'SURVIVED\t%s\t%s\t%s\t-\n' "$id" "$what" "$suite" >> "$WORK/table"
|
||||
survived=$((survived + 1))
|
||||
[ "$KEEP" = 1 ] && cp "$WORK/out" "$HERE/mutate-survived-$id.log"
|
||||
# -e, not a bare pattern: several of these assertions begin with "--",
|
||||
# which grep would read as its own flags.
|
||||
elif [ -n "$kills" ] && ! failures | grep -qF -e "$kills"; then
|
||||
# It broke something, but not the thing that is supposed to be watching
|
||||
# it. That is luck, and luck is lost the next time the other test moves.
|
||||
printf '\r %sWRONG-TEST%s %-36s %s\n' "$Y" "$Z" "$id" "expected [$kills], got [$(why_failed)]"
|
||||
printf 'WRONG-TEST\t%s\t%s\t%s\t%s\n' "$id" "$what" "$suite" "$(why_failed)" >> "$WORK/table"
|
||||
wrong=$((wrong + 1))
|
||||
[ "$KEEP" = 1 ] && cp "$WORK/out" "$HERE/mutate-wrongtest-$id.log"
|
||||
else
|
||||
printf '\r %sKILLED%s %-38s %s\n' "$G" "$Z" "$id" "${kills:-$(why_failed)}"
|
||||
printf 'KILLED\t%s\t%s\t%s\t%s\n' "$id" "$what" "$suite" "${kills:-$(why_failed)}" >> "$WORK/table"
|
||||
killed=$((killed + 1))
|
||||
fi
|
||||
done
|
||||
cp "$WORK/.dotup.pristine" "$D"
|
||||
|
||||
total=$((killed + survived + stale + wrong))
|
||||
[ "$total" -gt 0 ] || { echo "no mutations selected"; exit 2; }
|
||||
echo
|
||||
printf '%sMUTATION\tCATCHES IT\tSUITE\tCAUGHT BY%s\n' "$B" "$Z"
|
||||
awk -F'\t' '{printf "%-9s %-38s %-34s %s\n", $1, $2, $4, $5}' "$WORK/table"
|
||||
echo
|
||||
printf '%smutation score: %d/%d killed%s' "$B" "$killed" "$total" "$Z"
|
||||
[ "$survived" -eq 0 ] || printf ' %s(%d SURVIVED)%s' "$R_" "$survived" "$Z"
|
||||
[ "$wrong" -eq 0 ] || printf ' %s(%d wrong-test)%s' "$Y" "$wrong" "$Z"
|
||||
[ "$stale" -eq 0 ] || printf ' %s(%d stale)%s' "$Y" "$stale" "$Z"
|
||||
printf '\n\n'
|
||||
[ "$survived" -eq 0 ] && [ "$stale" -eq 0 ] && [ "$wrong" -eq 0 ]
|
||||
@@ -0,0 +1,8 @@
|
||||
# what: the private filter in selected_packages, removed
|
||||
# why: the "private is never a package" invariant is what makes --unattended safe
|
||||
# kills: no private row reaches the plan
|
||||
# run: unit
|
||||
<<<OLD
|
||||
$3=="private" { next }
|
||||
<<<NEW
|
||||
<<<END
|
||||
@@ -0,0 +1,9 @@
|
||||
# what: the bws checksum comparison, inverted: accepts on mismatch, rejects on match
|
||||
# why: the binary is about to hold the key to every other credential
|
||||
# kills: a checksum mismatch refuses to install
|
||||
# run: unit
|
||||
<<<OLD
|
||||
if [ -n "$want" ] && [ "$want" != "$got" ]; then
|
||||
<<<NEW
|
||||
if [ -n "$want" ] && [ "$want" = "$got" ]; then
|
||||
<<<END
|
||||
@@ -0,0 +1,9 @@
|
||||
# what: the bws token written 644 instead of 600
|
||||
# why: any account on the box could then read the machine credential
|
||||
# kills: the bws token is written 600
|
||||
# run: unit
|
||||
<<<OLD
|
||||
chmod 600 "$BWS_TOKEN"
|
||||
<<<NEW
|
||||
chmod 644 "$BWS_TOKEN"
|
||||
<<<END
|
||||
@@ -0,0 +1,8 @@
|
||||
# what: the chmod -R go-rwx on the cloned private source, removed
|
||||
# why: the clone lands at the caller umask -- 022 on stock Ubuntu -- holding ssh config and machine identity
|
||||
# kills: the private source is unreadable to anyone else
|
||||
# run: unit
|
||||
<<<OLD
|
||||
[ ! -d "$PRIV_SRC" ] || chmod -R go-rwx "$PRIV_SRC" 2>/dev/null || :
|
||||
<<<NEW
|
||||
<<<END
|
||||
@@ -0,0 +1,13 @@
|
||||
# what: the password retry loop: any non-200 is fatal again
|
||||
# why: these credentials are asked for after every package is installed; one typo used to cost the whole install
|
||||
# kills: …then asks again, keeping the URL
|
||||
# run: unit
|
||||
<<<OLD
|
||||
P_BLOB=''
|
||||
say " Try again, or type q at the URL prompt to stay public-only."
|
||||
done
|
||||
<<<NEW
|
||||
P_BLOB=''
|
||||
return 1
|
||||
done
|
||||
<<<END
|
||||
@@ -0,0 +1,11 @@
|
||||
# what: cmd_private's unattended guard, inverted
|
||||
# why: an unattended run would sit at a password prompt nobody is there to answer
|
||||
# kills: --unattended: nobody is here to type it
|
||||
# run: unit
|
||||
<<<OLD
|
||||
if [ "$UNATTENDED" -eq 1 ]; then
|
||||
warn "unattended: the private tier needs a password nobody is here to type — skipped"
|
||||
<<<NEW
|
||||
if [ "$UNATTENDED" -ne 1 ]; then
|
||||
warn "unattended: the private tier needs a password nobody is here to type — skipped"
|
||||
<<<END
|
||||
@@ -0,0 +1,9 @@
|
||||
# what: cmd_private's terminal guard, inverted
|
||||
# why: the private tier would run with no terminal, reading EOF for a password
|
||||
# kills: no tty: it says THAT instead
|
||||
# run: unit
|
||||
<<<OLD
|
||||
[ -t 0 ] || { warn "no terminal: the private tier needs a password — skipped"; return 0; }
|
||||
<<<NEW
|
||||
[ ! -t 0 ] || { warn "no terminal: the private tier needs a password — skipped"; return 0; }
|
||||
<<<END
|
||||
@@ -0,0 +1,19 @@
|
||||
# what: cmd_install's unattended invasive refusal, deleted
|
||||
# why: a stale state file would install a docker daemon on a machine with nobody at the keyboard
|
||||
# kills: the daemon never reaches a package manager
|
||||
# run: unit
|
||||
<<<OLD
|
||||
if [ "$UNATTENDED" -eq 1 ]; then
|
||||
inv=$(awk -F'\t' -v selfile="$SEL" '
|
||||
BEGIN{ while((getline l < selfile)>0) sel[l]=1 }
|
||||
!/^[#@]/ && NF>=3 && $3=="invasive" && (($1"/"$2) in sel) {print $1"/"$2}' "$MANIFEST")
|
||||
if [ -n "$inv" ]; then
|
||||
warn "unattended: refusing invasive packages$(printf ' %s' $inv)"
|
||||
for k in $inv; do
|
||||
awk -F'\t' -v k="$k" '$3!=k' "$tbl" > "$tbl.f"; mv "$tbl.f" "$tbl"
|
||||
done
|
||||
fi
|
||||
fi
|
||||
|
||||
<<<NEW
|
||||
<<<END
|
||||
@@ -0,0 +1,11 @@
|
||||
# what: cmd_install always returns 0
|
||||
# why: a CI run or a wrapper script cannot tell a clean install from six failures
|
||||
# kills: the run exits non-zero
|
||||
# run: unit
|
||||
<<<OLD
|
||||
printf '\n%s%d package(s) did not install.%s Everything else did.\n' "$YEL" "$n" "$R" >&2
|
||||
return 1
|
||||
<<<NEW
|
||||
printf '\n%s%d package(s) did not install.%s Everything else did.\n' "$YEL" "$n" "$R" >&2
|
||||
return 0
|
||||
<<<END
|
||||
@@ -0,0 +1,9 @@
|
||||
# what: note_fail() neutered: failures are dropped
|
||||
# why: the summary, the count and the exit code all come from this one file
|
||||
# kills: …under a heading you can find
|
||||
# run: unit
|
||||
<<<OLD
|
||||
note_fail() { printf '%s\t%s\n' "$1" "$2" >> "$FAILED"; err "$1: $2"; }
|
||||
<<<NEW
|
||||
note_fail() { :; }
|
||||
<<<END
|
||||
@@ -0,0 +1,9 @@
|
||||
# what: ensure_chezmoi's installer call replaced with true
|
||||
# why: the private tier died with `chezmoi: not found` AFTER writing the bws token
|
||||
# kills: the installer call did not install anything
|
||||
# run: lab:scenarios/31-chezmoi-absent.sh
|
||||
<<<OLD
|
||||
sh -c "$(curl -fsLS get.chezmoi.io)" -- -b "$HOME/.local/bin" >/dev/null 2>&1 \
|
||||
<<<NEW
|
||||
true >/dev/null 2>&1 \
|
||||
<<<END
|
||||
@@ -0,0 +1,9 @@
|
||||
# what: find_tool no longer looks in ~/.local/bin
|
||||
# why: uv, bws and chezmoi all land there, on no PATH the running process has
|
||||
# kills: uv is found in ~/.local/bin, off PATH
|
||||
# run: unit
|
||||
<<<OLD
|
||||
for ft_c in "$HOME/.local/bin/$1" "$HOME/bin/$1" "$HOME/.npm-global/bin/$1" \
|
||||
<<<NEW
|
||||
for ft_c in "$HOME/bin/$1" "$HOME/.npm-global/bin/$1" \
|
||||
<<<END
|
||||
@@ -0,0 +1,11 @@
|
||||
# what: the apt one-at-a-time retry, removed
|
||||
# why: apt refuses the whole batch when one name is unusable; the retry is the difference between 34 installed and 0
|
||||
# kills: …then btop on its own
|
||||
# run: unit
|
||||
<<<OLD
|
||||
for p in $keep; do
|
||||
run_sh "${SUDO:+$SUDO }DEBIAN_FRONTEND=noninteractive apt-get install -y $p" \
|
||||
|| note_fail "$p" "apt install failed"
|
||||
done
|
||||
<<<NEW
|
||||
<<<END
|
||||
@@ -0,0 +1,8 @@
|
||||
# what: the npm one-at-a-time retry, removed
|
||||
# why: same shape as apt: one bad spec takes every other tool down with it
|
||||
# kills: …then @openai/codex on its own
|
||||
# run: unit
|
||||
<<<OLD
|
||||
for p in $specs; do run_sh "$NPM install -g $p" || note_fail "$(keyof npm "$p")" "npm install failed"; done
|
||||
<<<NEW
|
||||
<<<END
|
||||
@@ -0,0 +1,9 @@
|
||||
# what: install_fzf's base URL pointed at a host that does not exist
|
||||
# why: the picker cannot draw the list that installs fzf without fzf
|
||||
# kills: the fetch goes to the fzf release download URL
|
||||
# run: unit
|
||||
<<<OLD
|
||||
base=https://github.com/junegunn/fzf/releases/download
|
||||
<<<NEW
|
||||
base=https://github.invalid/junegunn/fzf/releases/download
|
||||
<<<END
|
||||
@@ -0,0 +1,9 @@
|
||||
# what: the neovim tarball extracted to /opt/nvim-TYPO
|
||||
# why: tar succeeds, the symlink dangles, and nothing says so
|
||||
# kills: the tarball is extracted where the symlink points
|
||||
# run: unit
|
||||
<<<OLD
|
||||
run_sh "${SUDO:+$SUDO }rm -rf /opt/nvim && ${SUDO:+$SUDO }mkdir -p /opt/nvim && ${SUDO:+$SUDO }tar -xzf /tmp/nvim.tgz -C /opt/nvim --strip-components=1" \
|
||||
<<<NEW
|
||||
run_sh "${SUDO:+$SUDO }rm -rf /opt/nvim && ${SUDO:+$SUDO }mkdir -p /opt/nvim && ${SUDO:+$SUDO }tar -xzf /tmp/nvim.tgz -C /opt/nvim-TYPO --strip-components=1" \
|
||||
<<<END
|
||||
@@ -0,0 +1,8 @@
|
||||
# what: install_deb's install step, removed: it downloads and stops
|
||||
# why: chrome and ghostty would report success having installed nothing
|
||||
# kills: …and then handed to the package manager
|
||||
# run: unit
|
||||
<<<OLD
|
||||
run_sh "${SUDO:+$SUDO }apt-get install -y '$f'" || note_fail "$(keyof deb "$s")" "dpkg install failed"
|
||||
<<<NEW
|
||||
<<<END
|
||||
@@ -0,0 +1,10 @@
|
||||
# what: a comment moved back INSIDE the fzf line-continuation (the DU-C1 shape)
|
||||
# why: the `\`-newline is stripped first, so the comment's own newline ends the command: fzf gets two binds and the lines below run as a command named --bind
|
||||
# kills: dotup pick exits 0
|
||||
# run: unit
|
||||
<<<OLD
|
||||
--bind "tab:execute-silent($SELF expand {2})+reload($SELF render)" \
|
||||
<<<NEW
|
||||
--bind "tab:execute-silent($SELF expand {2})+reload($SELF render)" \
|
||||
# clear-query is not cosmetic: ^t toggles every row the filter shows.
|
||||
<<<END
|
||||
Reference in New Issue
Block a user