Files
dotfiles-public/.tests/test.sh
T
bcherb2 90bca39396 fix: install bws, isolate the private config, keep credentials out of logs
Three defects the private tier could not survive a fresh machine with.

bws was never installed. The private/bws-secrets row promised seven API
keys and shipped no way to fetch them: dotsecrets shells out to `bws
secret get`, and nothing put that binary on the box. Added as a tarball
channel with checksum verification, musl rather than gnu so it does not
pin a glibc newer than an older LTS carries, plus core/unzip as a real
@needs dependency since the release is a .zip and 24.04 minimal has no
unzip. The releases API needs filtering by tag prefix: sdk-sm is a
monorepo and `latest` usually points at a python SDK, not bws.

Both tiers rendered their config to ~/.config/chezmoi/chezmoi.toml, so
re-running the public installer overwrote the private config and took its
seven promptStringOnce answers with it. Silently -- the templates degrade
politely when data is missing, so the symptom was `git commit` not knowing
who you are, days later. The private tier now renders to private.toml and
the cmp alias carries the matching -c.

`run` echoes its argv to stderr, and the private init argv ends in
https://user:TOKEN@host -- into scrollback, any `dotup 2>log`, and any
agent transcript. Traced through redact_url instead, in both the live and
dry-run branches. The clone is also chmod -R go-rwx afterwards: it lands
at the caller umask, and .git/config stores that same credential URL.

Tests: 90 passing, 9 new covering all three.
2026-08-17 11:15:48 -04:00

380 lines
20 KiB
Bash
Executable File

#!/bin/sh
# dotup test suite.
#
# Everything here is pty-free and runs unchanged in a container: the toggle
# rule is pure state-file logic, the match-confinement test uses
# `fzf --filter`, which is non-interactive, and the installer is driven against
# a directory of fake package managers that record what they were asked to do
# and install nothing. The one test that genuinely needs a terminal is fenced
# off at the bottom and skips cleanly without one.
#
# sh .tests/test.sh on the host
# sh .tests/test.sh --docker in a clean ubuntu:24.04 container
#
# NOTHING IN THIS FILE INSTALLS A PACKAGE. If you add a case that could, it
# belongs in a container and behind an explicit opt-in, not here.
set -eu
cd "$(dirname "$0")"
if [ "${1:-}" = "--docker" ]; then
# curl and script are here so the pty test runs too: the container has
# apt's fzf 0.44.1, not whatever the dev box has, and the cursor-on-reload
# behaviour is the one property that must hold on the older release.
exec docker run --rm -v "$(cd .. && pwd):/w:ro" "${DOTUP_TEST_IMAGE:-ubuntu:24.04}" sh -c '
apt-get update -qq
apt-get install -y -qq fzf curl ca-certificates >/dev/null 2>&1
cp -r /w /work && cd /work/.tests && rm -rf state fzfver
echo " distro fzf: $(fzf --version 2>/dev/null || echo none)"
sh ../dot_local/bin/executable_dotup preflight # exactly what a real run does first
exec sh test.sh'
fi
ROOT=$(CDPATH= cd -- .. && pwd)
D=$ROOT/dot_local/bin/executable_dotup
M=$ROOT/dot_local/share/dotup/packages.tsv
export DOTUP_STATE=$PWD/state
export DOTUP_MANIFEST=$M
pass=0; fail=0
ok() { pass=$((pass+1)); printf ' \033[32mok\033[0m %s\n' "$1"; }
no() { fail=$((fail+1)); printf ' \033[31mFAIL\033[0m %s\n' "$1"; }
is() { # is <desc> <expected> <actual>
if [ "$2" = "$3" ]; then ok "$1"; else no "$1 — want [$2] got [$3]"; fi
}
has() { # has <desc> <needle> <haystack>
case $3 in *"$2"*) ok "$1" ;; *) no "$1 — [$2] not in output" ;; esac
}
hasnt(){ # hasnt <desc> <needle> <haystack>
case $3 in *"$2"*) no "$1 — [$2] IS in output" ;; *) ok "$1" ;; esac
}
# Normalise before parsing: "[x]" is one awk field but "[ ]" is two, so column
# positions shift with tick state. Collapse the box to a single token first.
rows() { sh $D render | sed 's/\x1b\[[0-9;]*m//g; s/\[[x~ ]\]/BOX/'; }
grp() { rows | awk -v g="$1" '$1!="BOX" && $3==g {print $4}'; }
# Every selected key matching a flag. Trailing ":" keeps a failed grep on the
# last item from tripping set -e via the loop's exit status.
picked(){ awk -F'\t' -v f="$1" '!/^#/ && !/^@/ && NF>=3 && $3==f {print $1"/"$2}' "$M" \
| while read -r k; do grep -qxF "$k" state/selected && echo "$k"; done; :; }
missed(){ awk -F'\t' -v f="$1" '!/^#/ && !/^@/ && NF>=3 && $3==f {print $1"/"$2}' "$M" \
| while read -r k; do grep -qxF "$k" state/selected || echo "$k"; done; :; }
reset() { rm -rf state; sh $D preset defaults; }
printf '\n\033[1mtoggle rule\033[0m\n'
reset
is "package off -> group goes partial" "2/3" \
"$(sh $D toggle p:media/sox >/dev/null; grp media)"
is "group partial -> all on" "3/3" \
"$(sh $D toggle g:media >/dev/null; grp media)"
is "group full -> all off" "0/3" \
"$(sh $D toggle g:media >/dev/null; grp media)"
is "group empty -> all on" "3/3" \
"$(sh $D toggle g:media >/dev/null; grp media)"
is "bulk, mixed set -> all on" "3/3" \
"$(sh $D toggle p:gpu/cuda-toolkit >/dev/null
sh $D toggle p:gpu/nvidia-driver p:gpu/cuda-toolkit p:gpu/container-toolkit >/dev/null
grp gpu)"
is "bulk, all on -> all off" "0/3" \
"$(sh $D toggle p:gpu/nvidia-driver p:gpu/cuda-toolkit p:gpu/container-toolkit >/dev/null
grp gpu)"
printf '\n\033[1mdependencies — @needs auto-ticks, both directions\033[0m\n'
on() { grep -qxF "$1" state/selected && echo on || echo off; }
reset; sh $D toggle p:networking/xrdp >/dev/null
is "ticking xrdp pulls in the desktop group" "2/2" "$(grp desktop)"
sh $D toggle g:desktop >/dev/null
is "unticking desktop drops xrdp" "off" "$(on networking/xrdp)"
is "unticking desktop drops xorgxrdp" "off" "$(on networking/xorgxrdp)"
reset; sh $D toggle p:gpu/container-toolkit >/dev/null
is "container-toolkit pulls in docker" "3/3" "$(grp docker)"
reset; sh $D toggle p:core/node >/dev/null
is "dropping node drops codex" "off" "$(on agents/codex)"
is "dropping node drops mermaid-cli" "off" "$(on core/mermaid-cli)"
is "…and transitively drops neovim" "off" "$(on core/neovim)"
is "but leaves unrelated packages be" "on" "$(on core/ripgrep)"
# The directives must not leak into package parsing.
is "@needs lines are not packages" "" \
"$(rows | awk '$1!="BOX" && $3 ~ /@/ {print $3}')"
is "group count unchanged by directives" "11" \
"$(awk -F'\t' '!/^[#@]/ && NF>=3 {print $1}' "$M" | uniq | grep -c .)"
# @spec is the second directive kind and must be as invisible as the first.
is "@spec lines are not packages" "0" \
"$(sh $D render | cut -f2 | grep -c 'spec' || true)"
reset; sh $D toggle p:core/uv >/dev/null
is "dropping uv drops specify-cli" "off" "$(on agents/specify-cli)"
# bws-secrets promised seven API keys and installed nothing to fetch them with.
# The binary ships as a .zip and 24.04 minimal has no unzip, so the dependency
# is real rather than decorative.
reset; sh $D toggle p:private/bws-secrets >/dev/null
is "ticking bws-secrets pulls in unzip" "on" "$(on core/unzip)"
is "bws-secrets has an install channel" "-tarball" \
"$(awk -F'\t' '$1=="private" && $2=="bws-secrets" {print $4}' "$M")"
printf '\n\033[1mcredentials never reach a log\033[0m\n'
# `run` echoes its whole argv to stderr. The private init's argv ends in
# https://user:TOKEN@host, so it must not go through `run`.
is "private init does not use run()" "0" \
"$(grep -c 'run chezmoi init' $D)"
# Two, not one: the dry-run branch traces it too, and a dry run that printed a
# live token would be the worse leak of the pair.
is "both traced URLs are redacted" "2" \
"$(grep -c 'redact_url "\$p_repo"' $D)"
# Exercise the real implementation lifted straight out of dotup. A copy of the
# sed expression here would keep passing after someone edited the original.
rd() { eval "$(sed -n '/^redact_url()/p' $D)"; redact_url "$1"; }
is "redact_url strips userinfo" "https://<redacted>@git.example.com/x.git" \
"$(rd 'https://ben:deadbeefcafe@git.example.com/x.git')"
is "...and leaves a credential-free URL alone" "https://git.example.com/x.git" \
"$(rd 'https://git.example.com/x.git')"
# Two tiers, two configs. Sharing one meant the public installer silently ate
# the private tier's seven promptStringOnce answers.
is "private init carries its own -c" "1" \
"$(grep -c '\-c "\$PRIV_CFG"' $D)"
is "PRIV_CFG is not the default config path" "0" \
"$(grep -c 'PRIV_CFG=.*chezmoi/chezmoi.toml' $D)"
is "the private source is locked down after clone" "1" \
"$(grep -c 'chmod -R go-rwx "\$PRIV_SRC"' $D)"
printf '\n\033[1mrisk model — the invariants that matter\033[0m\n'
reset
is "defaults tick nothing invasive" "" "$(picked invasive)"
is "defaults tick nothing private" "" "$(picked private)"
is "defaults tick every safe package" "" "$(missed safe)"
# A group must never render a flag milder than something inside it.
bad=$(rows | awk '$1!="BOX" && $2=="BOX" {print $3, $5}' \
| while read -r g f; do
worst=$(awk -F'\t' -v g="$g" '!/^#/ && !/^@/ && NF>=3 && $1==g {
s=($3=="invasive")?3:($3=="private")?2:($3=="gui")?1:0
if (s>m) m=s } END{ print (m==3)?"invasive":(m==2)?"private":(m==1)?"gui":"safe" }' "$M")
[ "$f" = "$worst" ] || echo "$g shows $f, worst child is $worst"
done; :)
is "no group looks safer than its contents" "" "$bad"
printf '\n\033[1mmatch confinement — regression for the ^t bug\033[0m\n'
if command -v fzf >/dev/null; then
sh $D expand-all >/dev/null # every package row visible
grep -q gpu/nvidia-driver <<-EOF || sh $D expand-all >/dev/null
$(sh $D render | cut -f2)
EOF
rows=$(sh $D render | sed 's/\x1b\[[0-9;]*m//g')
hit=$(printf '%s\n' "$rows" | fzf --exact --filter=nvidia --no-sort | cut -f2 | sort)
is "--exact: 'nvidia' hits only gpu packages" \
"p:gpu/container-toolkit
p:gpu/cuda-toolkit
p:gpu/nvidia-driver" "$hit"
# The bug: without --exact this also matches docker, networking and desktop,
# and ^t silently switched all three on.
loose=$(printf '%s\n' "$rows" | fzf --filter=nvidia --no-sort | cut -f2 \
| grep -cv '^p:gpu/' || true)
if [ "$loose" -gt 0 ]; then
ok "fuzzy would over-match $loose non-gpu rows (why --exact is required)"
else
no "fuzzy no longer over-matches — re-check whether --exact is still load-bearing"
fi
is "the fzf invocation carries --exact" "1" \
"$(grep -c '\--ansi --exact' $D)"
# Every action the picker binds must exist on THIS fzf. Ubuntu 24.04 ships
# 0.44.1; a dev box may be 28 releases ahead. fzf validates binds at
# startup, and empty stdin makes it exit without needing a terminal.
badbind=$(sed -n 's/.*--bind "\([^"]*\)".*/\1/p; s/.*--bind '"'"'\([^'"'"']*\)'"'"'.*/\1/p' $D \
| sed "s|\$SELF|$D|g" \
| while read -r b; do
out=$(fzf --bind "$b" </dev/null 2>&1 || true)
case $out in *nvalid*|*nknown*|*nsupported*) echo "$b" ;; esac
done; :)
is "every binding parses on fzf $(fzf --version | cut -d' ' -f1)" "" "$badbind"
else
printf ' \033[33mskip\033[0m fzf not installed\n'
fi
printf '\n\033[1mpreflight — the picker brings its own fzf\033[0m\n'
vc() { sh $D vercmp "$1" "$2" && echo ge || echo lt; }
is "0.44.1 clears the floor" "ge" "$(vc 0.44.1 0.44.0)"
is "0.29 (jammy) does not" "lt" "$(vc 0.29 0.44.0)"
is "0.38 (bookworm) does not" "lt" "$(vc 0.38.0 0.44.0)"
is "compare is numeric, not lexical" "ge" "$(vc 0.100.0 0.44.0)"
is "0.9.5 is not above 0.44.0" "lt" "$(vc 0.9.5 0.44.0)"
is "preflight resolves a fzf above the floor" "ge" \
"$(vc "$(sh $D fzf-path | xargs -r -I{} sh -c '{} --version' | awk '{print $1}')" 0.44.0)"
# The picker must call its resolved binary, never whatever PATH happens to hold.
is "picker invokes the resolved binary" "1" \
"$(grep -c 'cmd_render | "\$FZF"' $D)"
is "picker never calls bare fzf" "0" \
"$(grep -c 'render | fzf ' $D)"
# The whole point of the cache: PATH and ~/.local/bin are left alone.
is "preflight writes nothing to ~/.local/bin" "absent" \
"$(sh $D preflight >/dev/null 2>&1; [ -e "$HOME/.local/bin/fzf" ] && echo present || echo absent)"
is "preflight does not export PATH" "0" \
"$(grep -c 'PATH=.*export PATH' $D)"
# fzf is still offered as a normal package — for your shell, not for the picker.
is "manifest still offers fzf to you" "core/fzf" \
"$(awk -F'\t' '!/^#/ && !/^@/ && NF>=3 && $2=="fzf" {print $1"/"$2}' "$M")"
printf '\n\033[1mplatform resolution\033[0m\n'
is "brew-only pkg resolves on linux (omp)" "" \
"$(sh $D expand g:agents >/dev/null
sh $D render | sed 's/\x1b\[[0-9;]*m//g' | grep 'omp' | grep -o 'unavailable' || true)"
# One place decides what installs a package; everything downstream reads it.
rs() { sh $D resolve "$1" | tr '\t' ' '; }
is "plain apt name resolves to apt" "apt sox" "$(rs media/sox)"
is "no apt package falls through to brew" "brew lazygit" "$(rs core/lazygit)"
is "tap-only pkg falls through to brew" "brew can1357/tap/omp" "$(rs agents/omp)"
is "special channel carries its @spec" "npm @openai/codex" "$(rs agents/codex)"
is "…and the scoped mermaid name" "npm @mermaid-js/mermaid-cli" "$(rs core/mermaid-cli)"
is "deb channel carries a source" "deb gh:mkasberg/ghostty-ubuntu:_amd64.deb" "$(rs apps/ghostty)"
is "snap channel renames to the binary" "snap bw" "$(rs core/bitwarden-cli)"
is "tarball is dispatched, not named" "tarball neovim" "$(rs core/neovim)"
is "an unknown key says so" "missing" "$(rs nope/nope | tr -d ' ')"
# The macOS half of the manifest is exercised nowhere else. Resolution is pure
# data, so a fake `uname` is enough to check it — and it caught a real bug: the
# apt/brew fallback used to run both ways, so a Linux-only package resolved to
# `apt install davfs2` on a machine that has never had apt.
DARWIN=$PWD/fakeuname
drs() { PATH="$DARWIN:$PATH" sh $D resolve "$1" | tr '\t' ' '; }
is "darwin: tap-only pkg is a brew tap" "brew can1357/tap/omp" "$(drs agents/omp)"
is "darwin: zsh is built in" "builtin zsh" "$(drs core/zsh)"
is "darwin: no apt package is not a reason to try apt" "unavailable" \
"$(drs networking/davfs2 | tr -d ' ')"
# Nothing in the default set may be unresolvable — that is a manifest bug, and
# it is silent until someone runs the installer on a fresh machine.
reset
is "every default package resolves" "" \
"$(sh $D plan 2>/dev/null | sed -n '/no source on this platform/,$p' | grep -v 'no source' || true)"
printf '\n\033[1mthe installer — driven against fake package managers\033[0m\n'
FAKE=$PWD/fakebin
export DOTUP_TEST_LOG=$PWD/state/calls.log
# --print resolves everything and must call nothing at all. The fakes are first
# on PATH, so any call whatsoever leaves a trace.
reset
: > "$DOTUP_TEST_LOG"
out=$(PATH="$FAKE:$PATH" sh $D --unattended --print 2>&1 || true)
is "--print calls no package manager" "0" "$(grep -c . "$DOTUP_TEST_LOG" || true)"
has "--print still shows the apt batch" "apt-get install -y" "$out"
has "--print shows the npm batch" "npm install -g" "$out"
hasnt "--print never reaches the private stage" "Bootstrap URL:" "$out"
# Now the real engine, over a subset chosen to exercise every non-destructive
# channel. Deliberately not the whole default set: the tarball and script
# handlers write outside $HOME, and a test suite has no business doing that.
reset
sh $D preset none
cat > state/selected <<'EOF'
agents/omp
agents/specify-cli
apps/obsidian
core/bitwarden-cli
core/gh
core/lazygit
core/mermaid-cli
core/ripgrep
EOF
: > "$DOTUP_TEST_LOG"
PATH="$FAKE:$PATH" sh $D install >/dev/null 2>&1 || true
log=$(cat "$DOTUP_TEST_LOG")
has "apt batches its packages in one call" "apt-get install -y ripgrep" "$log"
has "brew gets the tap-only package" "brew install can1357/tap/omp" "$log"
has "brew gets the apt-less package" "brew install lazygit" "$log"
has "npm gets the scoped spec" "@mermaid-js/mermaid-cli" "$log"
has "uv installs the tool, not the package" "uv tool install specify-cli" "$log"
has "snap gets the renamed binary" "snap install bw" "$log"
has "flatpak gets the app id" "flatpak install" "$log"
has "…with the real obsidian app id" "md.obsidian.Obsidian" "$log"
# The one that matters on Linux: apt not knowing a name is not a dead end.
has "apt probes before it batches" "apt-cache show gh" "$log"
has "an apt name apt rejects moves to brew" "brew install gh" "$log"
hasnt "…and is not left in the apt batch" "apt-get install -y gh" "$log"
# npm cannot run before node, uv cannot run before uv. The pipeline is fixed
# rather than sorted, so assert the order it actually produces.
is "apt runs before npm" "yes" \
"$(a=$(grep -n 'apt-get install' "$DOTUP_TEST_LOG" | head -1 | cut -d: -f1)
b=$(grep -n 'npm install' "$DOTUP_TEST_LOG" | head -1 | cut -d: -f1)
[ -n "$a" ] && [ -n "$b" ] && [ "$a" -lt "$b" ] && echo yes || echo no)"
# Nothing runs as root that does not have to.
hasnt "brew is never run through sudo" "sudo brew" "$log"
printf '\n\033[1munattended — the boundary holds because of what is missing\033[0m\n'
# A stale state file is the adversary here: it ticks the two things an
# unattended run must never act on, and the run has to refuse both anyway.
rm -rf state; mkdir -p state
cat > state/selected <<'EOF'
core/ripgrep
docker/docker-ce
networking/openssh-server
private/bws-secrets
private/private-repo
EOF
: > "$DOTUP_TEST_LOG"
out=$(PATH="$FAKE:$PATH" sh $D --unattended --print 2>&1 || true)
hasnt "unattended never installs a private row" "bws-secrets" "$out"
hasnt "unattended never prompts for a password" "Password:" "$out"
hasnt "unattended refuses a stale invasive tick" "docker-ce" "$out"
hasnt "…including a listening ssh port" "openssh-server" "$out"
has "…but still installs the safe defaults" "ripgrep" "$out"
# Determinism: the same command twice, on the same machine, means the same
# thing. A state file left by an interactive run must not change it.
a=$(PATH="$FAKE:$PATH" sh $D --unattended --print 2>/dev/null || true)
sh $D preset none
b=$(PATH="$FAKE:$PATH" sh $D --unattended --print 2>/dev/null || true)
is "unattended is computed, not inherited" "same" \
"$([ "$a" = "$b" ] && echo same || echo different)"
# `set -e` would kill the subshell at the failing command, so the status is
# captured through a || branch rather than read from $? afterwards.
rc() { c=0; "$@" >/dev/null 2>&1 || c=$?; echo "$c"; }
is "unattended exits clean when nothing fails" "0" \
"$(PATH="$FAKE:$PATH" rc sh $D --unattended --print)"
is "an unknown flag is refused" "2" "$(rc sh $D --nonsense)"
# The negative test in PLAN.md phase 3 runs `zsh -ic exit` after an unattended
# run. That only passes if zsh is in the unattended set.
has "unattended installs the shell it configures" "zsh" \
"$(reset; PATH="$FAKE:$PATH" sh $D --unattended --print 2>&1 | grep 'apt-get install' || true)"
printf '\n\033[1mruns where it has to run\033[0m\n'
# A machine with no sudo and no root is a real case -- a locked-down work box, a
# rootless container. `A || { B && C; }` is one || list, so when `command -v
# sudo` failed the whole list failed and `set -e` killed dotup at load, before
# it printed anything. A minimal PATH is the only way to see that.
# Not under state/ -- reset() removes that whole directory, and a PATH that
# vanishes mid-test looks exactly like the bug being tested for.
MIN=${TMPDIR:-/tmp}/dotup-minbin.$$
rm -rf "$MIN"; mkdir -p "$MIN"
# Resolved by looking, not by `command -v`: an interactive shell can report an
# alias or a bare name for these, and a symlink to a bare name is a loop.
for t in awk grep sed sort cut tr id uname mkdir cp mv rm cat head sh dirname basename; do
for bd in /usr/bin /bin /usr/local/bin; do
[ -x "$bd/$t" ] && { ln -sf "$bd/$t" "$MIN/$t"; break; }
done
done
reset >/dev/null 2>&1
is "runs as a non-root user with no sudo on PATH" "0" \
"$(c=0; env -i HOME="$HOME" PATH="$MIN" DOTUP_STATE="$DOTUP_STATE" DOTUP_MANIFEST="$M" \
"$MIN/sh" $D --unattended --print >/dev/null 2>&1 || c=$?
echo "$c")"
is "…and still resolves the whole default set" "1" \
"$(env -i HOME="$HOME" PATH="$MIN" DOTUP_STATE="$DOTUP_STATE" DOTUP_MANIFEST="$M" \
"$MIN/sh" $D --unattended --print 2>/dev/null | grep -c 'apt-get install -y ')"
rm -rf "$MIN"
# `sudo` must not be glued onto anything that is not a system package manager.
is "sudo is only ever used for apt, snap and dpkg" "" \
"$(grep -oE '\$\{SUDO:\+\$SUDO \}[a-z-]+' $D | sed 's/.*}//' | sort -u \
| grep -vE '^(apt-get|snap|rm|mkdir|tar|ln)$' || true)"
printf '\n\033[1minteractive loop (needs a pty)\033[0m\n'
if command -v fzf >/dev/null && command -v curl >/dev/null && command -v script >/dev/null; then
out=$(sh ./listen-test.sh 2>/dev/null | tr -d '\r' || true)
case $out in
*CURSOR_HELD*) ok "cursor survives toggle+reload" ;;
*) no "cursor position lost across reload — rapid ticking breaks" ;;
esac
else
printf ' \033[33mskip\033[0m needs fzf, curl and script\n'
fi
printf '\n%d passed, %d failed\n\n' "$pass" "$fail"
[ "$fail" -eq 0 ]