From d1e3f8bce2671be91a74265d8baa2b90c947b6f0 Mon Sep 17 00:00:00 2001 From: bcherb2 Date: Fri, 21 Aug 2026 22:34:53 -0400 Subject: [PATCH] =?UTF-8?q?test:=20mutation=20testing=20=E2=80=94=20break?= =?UTF-8?q?=20dotup=20on=20purpose=20and=20check=20the=20suite=20notices?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Each file in .tests/mutations/ is one deliberate bug: what it breaks, why that matters, the assertion meant to catch it, and an OLD/NEW pair applied to a copy of the tree. A mutation is KILLED only when the suite fails AND the named assertion is among the failures — failing for an unrelated reason is reported as WRONG-TEST, because that is luck, and luck is lost the next time the unrelated test moves. A mutation whose OLD block no longer matches is STALE rather than quietly skipped. 19 reproduces the DU-C1 shape: a comment moved back inside the fzf line-continuation. It is killed by "dotup pick exits 0". --- .tests/mutate.sh | 165 ++++++++++++++++++ .../mutations/01-private-never-a-package.mut | 8 + .tests/mutations/02-bws-checksum-inverted.mut | 9 + .tests/mutations/03-bws-token-644.mut | 9 + .../mutations/04-private-src-not-locked.mut | 8 + .tests/mutations/05-no-password-retry.mut | 13 ++ .../06-private-unattended-guard-inverted.mut | 11 ++ .../07-private-tty-guard-inverted.mut | 9 + .../08-no-unattended-invasive-filter.mut | 19 ++ .../mutations/09-install-always-exits-0.mut | 11 ++ .tests/mutations/10-note-fail-silent.mut | 9 + .../11-ensure-chezmoi-no-install.mut | 9 + .../12-find-tool-skips-local-bin.mut | 9 + .tests/mutations/13-no-apt-one-at-a-time.mut | 11 ++ .tests/mutations/14-no-npm-one-at-a-time.mut | 8 + .tests/mutations/16-fzf-bad-base-url.mut | 9 + .tests/mutations/17-nvim-path-typo.mut | 9 + .tests/mutations/18-deb-never-installed.mut | 8 + .../19-pick-comment-in-continuation.mut | 10 ++ 19 files changed, 344 insertions(+) create mode 100755 .tests/mutate.sh create mode 100644 .tests/mutations/01-private-never-a-package.mut create mode 100644 .tests/mutations/02-bws-checksum-inverted.mut create mode 100644 .tests/mutations/03-bws-token-644.mut create mode 100644 .tests/mutations/04-private-src-not-locked.mut create mode 100644 .tests/mutations/05-no-password-retry.mut create mode 100644 .tests/mutations/06-private-unattended-guard-inverted.mut create mode 100644 .tests/mutations/07-private-tty-guard-inverted.mut create mode 100644 .tests/mutations/08-no-unattended-invasive-filter.mut create mode 100644 .tests/mutations/09-install-always-exits-0.mut create mode 100644 .tests/mutations/10-note-fail-silent.mut create mode 100644 .tests/mutations/11-ensure-chezmoi-no-install.mut create mode 100644 .tests/mutations/12-find-tool-skips-local-bin.mut create mode 100644 .tests/mutations/13-no-apt-one-at-a-time.mut create mode 100644 .tests/mutations/14-no-npm-one-at-a-time.mut create mode 100644 .tests/mutations/16-fzf-bad-base-url.mut create mode 100644 .tests/mutations/17-nvim-path-typo.mut create mode 100644 .tests/mutations/18-deb-never-installed.mut create mode 100644 .tests/mutations/19-pick-comment-in-continuation.mut diff --git a/.tests/mutate.sh b/.tests/mutate.sh new file mode 100755 index 0000000..7d63235 --- /dev/null +++ b/.tests/mutate.sh @@ -0,0 +1,165 @@ +#!/bin/sh +# Mutation testing for dotup. Break the code on purpose; a suite that still +# passes has not been measuring what it claims to. +# +# sh .tests/mutate.sh every mutation in mutations/ +# sh .tests/mutate.sh 02 05 13 just those +# sh .tests/mutate.sh --list what each one does and why it matters +# sh .tests/mutate.sh --keep leave the mutated tree behind on failure +# +# Each file in mutations/ is one deliberate bug: a header saying what it breaks +# and why that matters, the ASSERTION that is supposed to catch it, an OLD +# block that must appear EXACTLY ONCE in dot_local/bin/executable_dotup, and +# the NEW text to put in its place. A +# mutation whose OLD block no longer matches is reported as STALE rather than +# quietly skipped -- the code moved, and the mutation has to move with it. +# +# Nothing here touches the working tree. The whole repo is copied once, the +# copy's dotup is mutated, the copy's own tests run against it, and the copy's +# dotup is restored from a pristine byte-for-byte spare between runs. +# +# A mutation is KILLED when the suite it names fails AND the named assertion is +# one of the failures. A suite that fails for some unrelated reason is reported +# as WRONG-TEST: the bug was noticed by accident, which is not the same as being +# tested for, and the next edit to the unrelated test would lose it. SURVIVED +# means the bug is live and nothing noticed at all. +set -eu +cd "$(dirname "$0")" +HERE=$PWD +ROOT=$(CDPATH= cd -- .. && pwd) +MUTS=$HERE/mutations + +KEEP=0; only=; LIST=0 +while [ $# -gt 0 ]; do + case $1 in + --keep) KEEP=1 ;; + --list) LIST=1 ;; + -*) echo "unknown flag $1" >&2; exit 2 ;; + *) only="$only $1" ;; + esac + shift +done + +G=$(printf '\033[32m'); R_=$(printf '\033[31m'); Y=$(printf '\033[33m') +B=$(printf '\033[1m'); Z=$(printf '\033[0m'); DIM=$(printf '\033[2m') + +if [ "$LIST" = 1 ]; then + for f in "$MUTS"/*.mut; do + printf '%s%s%s\n' "$B" "$(basename "$f" .mut)" "$Z" + sed -n 's/^# what: / breaks /p; s/^# why: / matters /p + s/^# kills: / caught by /p; s/^# run: / suite /p' "$f" + done + exit 0 +fi + +WORK=${TMPDIR:-/tmp}/dotup-mutate.$$ +cleanup() { [ "$KEEP" = 1 ] || rm -rf "$WORK"; } +trap cleanup EXIT INT TERM + +echo "== copying the tree ==" +mkdir -p "$WORK" +tar -C "$ROOT" --exclude=.git --exclude=tmp --exclude='.tests/state' -cf - . \ + | tar -C "$WORK" -xf - +D=$WORK/dot_local/bin/executable_dotup +cp "$D" "$WORK/.dotup.pristine" + +# ---------------------------------------------------------------- runners --- +# Both return 0 when the suite PASSED, which for a mutated tree is the bad news. +run_unit() { ( cd "$WORK/.tests" && sh test.sh ) >"$WORK/out" 2>&1; } +run_lab() { ( cd "$WORK/.tests/lab" && bash run.sh "$1" ) >"$WORK/out" 2>&1; } +run_suite() { + case $1 in + unit) run_unit ;; + lab:*) run_lab "${1#lab:}" ;; + *) echo "unknown suite: $1" >&2; return 125 ;; + esac +} +# What the suite said went wrong, in its own words. This is the evidence that a +# mutation was caught for a REASON rather than by a crash somewhere unrelated. +failures() { + sed 's/\x1b\[[0-9;]*m//g' "$WORK/out" \ + | sed -n 's/^ FAIL \(.*\)$/\1/p; s/^FAIL: \(.*\)$/\1/p; s/^\(TIMEOUT: .*\)$/\1/p' \ + | sed 's/ — .*//' +} +why_failed() { failures | head -3 | tr '\n' ';' | sed 's/;$//; s/;/; /g'; } + +# --------------------------------------------------------------- baseline --- +echo "== baseline: the copy, unmutated ==" +if run_unit; then + base=$(sed 's/\x1b\[[0-9;]*m//g' "$WORK/out" | sed -n 's/^\([0-9]* passed, [0-9]* failed\)$/\1/p' | tail -1) + printf ' %sunit%s %s\n' "$G" "$Z" "$base" +else + sed 's/\x1b\[[0-9;]*m//g' "$WORK/out" | tail -20 + echo "${R_}the suite does not pass on an unmutated tree; nothing below would mean anything.${Z}" + exit 1 +fi + +# --------------------------------------------------------------- mutating --- +killed=0; survived=0; stale=0; wrong=0 +: > "$WORK/table" +for f in "$MUTS"/*.mut; do + id=$(basename "$f" .mut) + if [ -n "$only" ]; then + want=0 + for o in $only; do case $id in $o|$o-*|*"$o"*) want=1 ;; esac; done + [ "$want" = 1 ] || continue + fi + what=$(sed -n 's/^# what: //p' "$f") + suite=$(sed -n 's/^# run: //p' "$f") + kills=$(sed -n 's/^# kills: //p' "$f") + + cp "$WORK/.dotup.pristine" "$D" + if ! python3 - "$f" "$D" <<'PY' +import sys +mut, target = sys.argv[1], sys.argv[2] +raw = open(mut).read() +body = raw.split('<<> "$WORK/table" + stale=$((stale + 1)); continue + fi + + printf ' %s…%s %-38s %s' "$DIM" "$Z" "$id" "$DIM$suite$Z" + if run_suite "$suite"; then + printf '\r %sSURVIVED%s %-38s %s\n' "$R_" "$Z" "$id" "$what" + printf 'SURVIVED\t%s\t%s\t%s\t-\n' "$id" "$what" "$suite" >> "$WORK/table" + survived=$((survived + 1)) + [ "$KEEP" = 1 ] && cp "$WORK/out" "$HERE/mutate-survived-$id.log" + # -e, not a bare pattern: several of these assertions begin with "--", + # which grep would read as its own flags. + elif [ -n "$kills" ] && ! failures | grep -qF -e "$kills"; then + # It broke something, but not the thing that is supposed to be watching + # it. That is luck, and luck is lost the next time the other test moves. + printf '\r %sWRONG-TEST%s %-36s %s\n' "$Y" "$Z" "$id" "expected [$kills], got [$(why_failed)]" + printf 'WRONG-TEST\t%s\t%s\t%s\t%s\n' "$id" "$what" "$suite" "$(why_failed)" >> "$WORK/table" + wrong=$((wrong + 1)) + [ "$KEEP" = 1 ] && cp "$WORK/out" "$HERE/mutate-wrongtest-$id.log" + else + printf '\r %sKILLED%s %-38s %s\n' "$G" "$Z" "$id" "${kills:-$(why_failed)}" + printf 'KILLED\t%s\t%s\t%s\t%s\n' "$id" "$what" "$suite" "${kills:-$(why_failed)}" >> "$WORK/table" + killed=$((killed + 1)) + fi +done +cp "$WORK/.dotup.pristine" "$D" + +total=$((killed + survived + stale + wrong)) +[ "$total" -gt 0 ] || { echo "no mutations selected"; exit 2; } +echo +printf '%sMUTATION\tCATCHES IT\tSUITE\tCAUGHT BY%s\n' "$B" "$Z" +awk -F'\t' '{printf "%-9s %-38s %-34s %s\n", $1, $2, $4, $5}' "$WORK/table" +echo +printf '%smutation score: %d/%d killed%s' "$B" "$killed" "$total" "$Z" +[ "$survived" -eq 0 ] || printf ' %s(%d SURVIVED)%s' "$R_" "$survived" "$Z" +[ "$wrong" -eq 0 ] || printf ' %s(%d wrong-test)%s' "$Y" "$wrong" "$Z" +[ "$stale" -eq 0 ] || printf ' %s(%d stale)%s' "$Y" "$stale" "$Z" +printf '\n\n' +[ "$survived" -eq 0 ] && [ "$stale" -eq 0 ] && [ "$wrong" -eq 0 ] diff --git a/.tests/mutations/01-private-never-a-package.mut b/.tests/mutations/01-private-never-a-package.mut new file mode 100644 index 0000000..58a8213 --- /dev/null +++ b/.tests/mutations/01-private-never-a-package.mut @@ -0,0 +1,8 @@ +# what: the private filter in selected_packages, removed +# why: the "private is never a package" invariant is what makes --unattended safe +# kills: no private row reaches the plan +# run: unit +<</dev/null || : +<<0) sel[l]=1 } + !/^[#@]/ && NF>=3 && $3=="invasive" && (($1"/"$2) in sel) {print $1"/"$2}' "$MANIFEST") + if [ -n "$inv" ]; then + warn "unattended: refusing invasive packages$(printf ' %s' $inv)" + for k in $inv; do + awk -F'\t' -v k="$k" '$3!=k' "$tbl" > "$tbl.f"; mv "$tbl.f" "$tbl" + done + fi + fi + +<<&2 + return 1 +<<&2 + return 0 +<<> "$FAILED"; err "$1: $2"; } +<</dev/null 2>&1 \ +<</dev/null 2>&1 \ +<<