diff --git a/.tests/ptydrive.py b/.tests/ptydrive.py new file mode 100755 index 0000000..a75c01d --- /dev/null +++ b/.tests/ptydrive.py @@ -0,0 +1,123 @@ +#!/usr/bin/env python3 +"""Drive a command on a real pty from a small expect/send script. + +The private tier refuses to run without a terminal -- `[ -t 0 ]` is one of the +guards under test -- so every assertion about the password prompt, the retry +loop, the token file and the clone needs a pty to exist at all. `expect` is not +installed everywhere; python3 is, and the whole of what is needed here is +"read until this pattern, then write that line". + + ptydrive.py [--timeout SEC] --script FILE -- cmd [args...] + +Script lines, blank lines and #-comments ignored: + + expect wait for it in everything the child has written + send write it, plus a newline + sendraw write it with no newline + close close the child's input (EOF) + +Everything the child writes is copied to stdout, so the caller greps the +transcript exactly as it would grep any other command's output. Exit status is +the child's, or 3 for "a pattern never arrived" -- which names the pattern on +stderr rather than letting the run die of a later, unrelated timeout. +""" +import argparse +import os +import pty +import re +import select +import sys +import time + +ap = argparse.ArgumentParser() +ap.add_argument("--timeout", type=float, default=20.0) +ap.add_argument("--script", required=True) +ap.add_argument("cmd", nargs=argparse.REMAINDER) +a = ap.parse_args() +cmd = a.cmd[1:] if a.cmd and a.cmd[0] == "--" else a.cmd +if not cmd: + sys.exit("ptydrive.py: no command") + +steps = [] +for raw in open(a.script): + line = raw.rstrip("\n") + if not line.strip() or line.lstrip().startswith("#"): + continue + verb, _, rest = line.partition(" ") + steps.append((verb, rest)) + +pid, fd = pty.fork() +if pid == 0: + os.execvp(cmd[0], cmd) + os._exit(127) + +buf = "" +out = [] +deadline = time.time() + a.timeout +eof = False + + +def pump(block=0.2): + """Read whatever is there. Returns False once the child's side is gone.""" + global buf, eof + r, _, _ = select.select([fd], [], [], block) + if not r: + return True + try: + chunk = os.read(fd, 65536) + except OSError: + chunk = b"" + if not chunk: + eof = True + return False + text = chunk.decode("utf-8", "replace") + buf += text + out.append(text) + return True + + +status = 0 +try: + for verb, arg in steps: + if verb == "expect": + pat = re.compile(arg) + end = time.time() + a.timeout + while not pat.search(buf): + if time.time() > end: + sys.stderr.write("ptydrive.py TIMEOUT: never saw /%s/\n" % arg) + status = 3 + raise SystemExit + if not pump(): + if pat.search(buf): + break + sys.stderr.write("ptydrive.py EOF before /%s/\n" % arg) + status = 4 + raise SystemExit + # Consume up to and including the match so a later `expect` for the + # same text waits for a NEW one -- the retry loop asks for the same + # three prompts twice, and matching the first round twice would + # prove nothing. + buf = buf[pat.search(buf).end():] + elif verb in ("send", "sendraw"): + data = arg + ("\n" if verb == "send" else "") + os.write(fd, data.encode()) + elif verb == "close": + os.close(fd) + fd = -1 + else: + sys.exit("ptydrive.py: unknown verb %r" % verb) +except SystemExit: + pass + +# Drain whatever is left, then reap. +while not eof and time.time() < deadline: + if not pump(0.1): + break +try: + _, wstatus = os.waitpid(pid, 0) + child = os.waitstatus_to_exitcode(wstatus) +except ChildProcessError: + child = 0 +sys.stdout.write("".join(out)) +sys.stdout.flush() +sys.exit(status if status else (child if child >= 0 else 128 - child)) diff --git a/.tests/test.sh b/.tests/test.sh index b3a7d7c..f217beb 100755 --- a/.tests/test.sh +++ b/.tests/test.sh @@ -60,6 +60,52 @@ missed(){ awk -F'\t' -v f="$1" '!/^#/ && !/^@/ && NF>=3 && $3==f {print $1"/"$2} | while read -r k; do grep -qxF "$k" state/selected || echo "$k"; done; :; } reset() { rm -rf state; sh $D preset defaults; } +# ---------------------------------------------------------------- the seal --- +# The old harness put .tests/fakebin at the FRONT of $PATH. That fails OPEN: a +# package manager with no fake fell through to the real one, and an audit of +# this suite ran the host's actual `brew`. Everything below runs under a SEALED +# PATH instead (see seal.sh) with `env -i`, so the run inherits nothing at all +# and a tool nobody thought to fake is command-not-found rather than the +# machine's own copy. +SEAL=${TMPDIR:-/tmp}/dotup-seal.$$ +SEALSUDO=${TMPDIR:-/tmp}/dotup-seal-sudo.$$ +sh ./seal.sh "$SEAL" >/dev/null +sh ./seal.sh "$SEALSUDO" --with sudo >/dev/null +BOXN=0; SB=; LOG=; BOXENV=; BOXPATH=$SEAL; PW= +cleanup_boxes() { rm -rf "$SEAL" "$SEALSUDO" ${TMPDIR:-/tmp}/dotup-box."$$".* \ + ${TMPDIR:-/tmp}/dotup-sealtest."$$" ; } +trap cleanup_boxes EXIT INT TERM + +# A fresh $HOME per case, because find_tool probes ~/.local/bin, ~/bin and +# ~/.npm-global/bin: a run against the developer's real $HOME is a run against +# whatever the developer happens to have installed. +newbox() { + BOXN=$((BOXN+1)); SB=${TMPDIR:-/tmp}/dotup-box.$$.$BOXN + rm -rf "$SB"; mkdir -p "$SB/.config/dotfiles" "$SB/tmp" "$SB/.local/bin" + : > "$SB/.config/dotfiles/expanded"; : > "$SB/.config/dotfiles/selected" + LOG=$SB/calls.log; : > "$LOG"; BOXENV=; BOXPATH=$SEAL +} +# $BOXENV is deliberately unquoted: it is a list of NAME=VALUE words to add. +box() { env -i HOME="$SB" PATH="$BOXPATH" TERM=dumb LANG=C TMPDIR="$SB/tmp" \ + DOTUP_STATE="$SB/.config/dotfiles" DOTUP_MANIFEST="$M" \ + DOTUP_TEST_LOG="$LOG" $BOXENV "$BOXPATH/sh" $D "$@"; } +boxrc() { c=0; box "$@" >/dev/null 2>&1 || c=$?; echo "$c"; } +pick() { printf '%s\n' "$@" > "$SB/.config/dotfiles/selected"; } +# A seal with a hole in it, for the cases whose whole point is that a tool is +# NOT there. Refuses if the host would satisfy the lookup anyway through one of +# find_tool's four absolute probes -- an absence that is not real would make the +# assertion pass for the wrong reason. +holed() { + sh ./seal.sh "$SB/seal" --without "$@" >/dev/null + BOXPATH=$SB/seal + for h in "$@"; do + grep -q "^$h " "$SB/seal/.shadowed" 2>/dev/null && return 1 + done + return 0 +} +# Everything a run wrote, colour stripped. +plain() { sed 's/\x1b\[[0-9;]*m//g'; } + printf '\n\033[1mtoggle rule\033[0m\n' reset is "package off -> group goes partial" "2/3" \ @@ -115,12 +161,6 @@ is "bws-secrets stays a non-package" "-" \ "$(awk -F'\t' '$1=="private" && $2=="bws-secrets" {print $4}' "$M")" is "no manifest row installs bws" "0" \ "$(awk -F'\t' '!/^[#@]/ && NF>=3 && $2=="bws" {c++} END{print c+0}' "$M")" -is "the private tier installs bws itself" "1" \ - "$(grep -c '^ ensure_bws$' $D)" -is "…only after a token exists to use" "1" \ - "$(grep -A1 'bws token written, mode 600' $D | grep -c ensure_bws)" -is "bws is checksum-verified" "1" \ - "$(grep -c 'bws checksum mismatch' $D)" printf '\n\033[1mcredentials never reach a log\033[0m\n' # `run` echoes its whole argv to stderr. The private init's argv ends in @@ -136,12 +176,6 @@ is "both traced URLs are redacted" "2" \ # publishes it to /proc//cmdline, which every account on the box can read # for as long as the clone runs, and then into the clone's .git/config, which # keeps it. Only the split-out p_clean may reach an argv. -is "the credential-bearing URL never reaches an argv" "0" \ - "$(grep -c 'chezmoi init.*\$p_repo' $D)" -is "the token goes to a credential file instead" "1" \ - "$(grep -c "umask 077; printf '%s\\\\n' \"\\\$p_cred\" > \"\\\$PRIV_CRED\"" $D)" -is "…at mode 600" "1" \ - "$(grep -c 'chmod 600 "\$PRIV_CRED"' $D)" # Exercise the real implementation lifted straight out of dotup. A copy of the # sed expression here would keep passing after someone edited the original. split() { p_repo=$1; eval "$(sed -n '/p_cred=\$(printf/p;/p_clean=\$(printf/p' $D)" @@ -188,12 +222,6 @@ is "...and leaves a credential-free URL alone" "https://git.example.com/x.git" \ "$(rd 'https://git.example.com/x.git')" # Two tiers, two configs. Sharing one meant the public installer silently ate # the private tier's seven promptStringOnce answers. -is "private init carries its own -c" "1" \ - "$(grep -c '\-c "\$PRIV_CFG"' $D)" -is "PRIV_CFG is not the default config path" "0" \ - "$(grep -c 'PRIV_CFG=.*chezmoi/chezmoi.toml' $D)" -is "the private source is locked down after clone" "1" \ - "$(grep -c 'chmod -R go-rwx "\$PRIV_SRC"' $D)" printf '\n\033[1mchezmoi is not assumed to be on PATH\033[0m\n' # dotup ARRIVES via chezmoi, so "it must be here already" is the natural @@ -203,10 +231,6 @@ printf '\n\033[1mchezmoi is not assumed to be on PATH\033[0m\n' # with `chezmoi: not found` AFTER writing the bws token. is "nothing calls chezmoi by bare name" "0" \ "$(grep -cE '^[[:space:]]*chezmoi (init|apply|update)' $D)" -is "the init uses the resolved path" "1" \ - "$(grep -cF '"$CHEZMOI" init --apply' $D)" -is "find_tool looks in ~/bin too" "1" \ - "$(grep -cF '"$HOME/bin/$1"' $D)" # Resolution happens before the prompt: discovering it afterwards means the # password is spent and the token is already on disk. is "chezmoi is resolved before the password is asked for" "yes" \ @@ -216,20 +240,6 @@ printf '\n\033[1ma wrong password is not a reinstall\033[0m\n' # The endpoint credentials are asked for at the very END of a run, after every # package is installed. Any non-200 used to be fatal, so one mistyped character # meant repeating the whole install to get back to the prompt. -is "the prompt retries instead of returning" "0" \ - "$(grep -c 'endpoint refused the credentials' $D)" -is "…up to a bounded number of attempts" "1" \ - "$(grep -c '"$p_try" -gt 5' $D)" -is "…and points at the cheap way back in" "1" \ - "$(grep -c 'Re-run only this step: dotup private' $D)" -# One message for 401, 404 and an unreachable host is how a URL-shape bug spends -# an evening looking like a password problem. Each needs a different next move. -is "a 401 names the password" "1" "$(grep -c 'wrong username or password' $D)" -is "a 404 names the route" "1" "$(grep -c 'no bootstrap.env is there' $D)" -is "an unreachable host says so" "1" "$(grep -c 'could not reach that address' $D)" -# --fail is gone, so the code must be checked explicitly or an error page would -# be parsed as the credential blob. -is "the blob is used only on 200" "1" "$(grep -cF '200) break ;;' $D)" # The subcommand existed but was absent from --help, so there was no way to # discover the recovery path. is "dotup private is documented" "1" \ @@ -237,6 +247,15 @@ is "dotup private is documented" "1" \ printf '\n\033[1mrisk model — the invariants that matter\033[0m\n' reset +# Positive controls FIRST. `picked` and `missed` both end in `; :` so a failed +# grep cannot trip `set -e` -- which also means a broken helper returns nothing +# and is indistinguishable from a clean result. These three invariants are the +# whole safety story, and all three "passed" by producing no output. Prove the +# helpers can still speak before trusting their silence. +is "control: picked can see a safe row" "core/ripgrep" \ + "$(picked safe | grep -x 'core/ripgrep' || echo MISSING)" +is "control: missed can see an unticked row" "gpu/nvidia-driver" \ + "$(missed invasive | grep -x 'gpu/nvidia-driver' || echo MISSING)" is "defaults tick nothing invasive" "" "$(picked invasive)" is "defaults tick nothing private" "" "$(picked private)" is "defaults tick every safe package" "" "$(missed safe)" @@ -324,7 +343,11 @@ is "no apt package falls through to brew" "brew lazygit" "$(r is "tap-only pkg falls through to brew" "brew can1357/tap/omp" "$(rs agents/omp)" is "special channel carries its @spec" "npm @openai/codex" "$(rs agents/codex)" is "…and the scoped mermaid name" "npm @mermaid-js/mermaid-cli" "$(rs core/mermaid-cli)" -is "deb channel carries a source" "deb gh:mkasberg/ghostty-ubuntu:_amd64.deb" "$(rs apps/ghostty)" +# %a and %v are expanded at install time from `dpkg --print-architecture` and +# /etc/os-release. The literal `_amd64.deb` that used to be here matched no +# asset ghostty-ubuntu has ever published -- they are named +# ghostty_1.3.1-0.ppa2_amd64_24.04.deb -- so apps/ghostty could never install. +is "deb channel carries a source" "deb gh:mkasberg/ghostty-ubuntu:_%a_%v.deb" "$(rs apps/ghostty)" is "snap channel renames to the binary" "snap bw" "$(rs core/bitwarden-cli)" is "tarball is dispatched, not named" "tarball neovim" "$(rs core/neovim)" is "an unknown key says so" "missing" "$(rs nope/nope | tr -d ' ')" @@ -342,19 +365,37 @@ is "darwin: no apt package is not a reason to try apt" "unavailable" \ # Nothing in the default set may be unresolvable — that is a manifest bug, and # it is silent until someone runs the installer on a fresh machine. reset +# This one passed if `dotup plan` crashed: no output, no unresolved lines, green. +is "control: plan produces a table at all" "yes" \ + "$(sh $D plan 2>/dev/null | grep -q 'packages' && echo yes || echo no)" is "every default package resolves" "" \ "$(sh $D plan 2>/dev/null | sed -n '/no source on this platform/,$p' | grep -v 'no source' || true)" -printf '\n\033[1mthe installer — driven against fake package managers\033[0m\n' -FAKE=$PWD/fakebin -export DOTUP_TEST_LOG=$PWD/state/calls.log - -# --print resolves everything and must call nothing at all. The fakes are first -# on PATH, so any call whatsoever leaves a trace. -reset -: > "$DOTUP_TEST_LOG" -out=$(PATH="$FAKE:$PATH" sh $D --unattended --print 2>&1 || true) -is "--print calls no package manager" "0" "$(grep -c . "$DOTUP_TEST_LOG" || true)" +printf '\n\033[1mthe installer — driven against a SEALED set of fakes\033[0m\n' +# Sealed, not prepended. This section keeps `sudo` in the seal because two of +# its assertions are about what is and is not run through it; the failure +# injection sections further down deliberately leave sudo out, so that apt-get +# is invoked directly and can be made to fail. +newbox; BOXPATH=$SEALSUDO +box preset defaults >/dev/null +: > "$LOG" +out=$(box --unattended --print 2>&1 || true) +# "Calls nothing at all" stopped being the right bar when resolution began +# asking the machine what it is: `dpkg --print-architecture` and `apt-cache +# policy` are questions. What a dry run must never do is CHANGE anything. +# "a dry run never invents a failure" lives in .tests/lab/scenarios/, NOT here, +# and the reason is worth recording. The bug was core/brew probing `have brew` +# after an install step that --print had only printed; the check then reported +# "brew still not found after installing it" for a run that installed nothing. +# It cannot be reproduced on a developer box: `find_tool` probes +# /home/linuxbrew/.linuxbrew/bin by ABSOLUTE path, this host has brew there, so +# `have brew` succeeds and the failure never happens. No $PATH can seal an +# absolute probe -- seal.sh says so in its .shadowed list. An assertion written +# here would have passed forever without ever being able to fail. +is "--print installs nothing" "" \ + "$(grep -E ' (install|remove|upgrade) ' "$LOG" || true)" +is "…and does not so much as refresh a package list" "" \ + "$(grep -E 'apt-get update' "$LOG" || true)" has "--print still shows the apt batch" "apt-get install -y" "$out" has "--print shows the npm batch" "npm install -g" "$out" hasnt "--print never reaches the private stage" "Bootstrap URL:" "$out" @@ -362,21 +403,11 @@ hasnt "--print never reaches the private stage" "Bootstrap URL:" "$out" # Now the real engine, over a subset chosen to exercise every non-destructive # channel. Deliberately not the whole default set: the tarball and script # handlers write outside $HOME, and a test suite has no business doing that. -reset -sh $D preset none -cat > state/selected <<'EOF' -agents/omp -agents/specify-cli -apps/obsidian -core/bitwarden-cli -core/gh -core/lazygit -core/mermaid-cli -core/ripgrep -EOF -: > "$DOTUP_TEST_LOG" -PATH="$FAKE:$PATH" sh $D install >/dev/null 2>&1 || true -log=$(cat "$DOTUP_TEST_LOG") +newbox; BOXPATH=$SEALSUDO +pick agents/omp agents/specify-cli apps/obsidian core/bitwarden-cli \ + core/gh core/lazygit core/mermaid-cli core/ripgrep docker/docker-ce +box install >/dev/null 2>&1 || true +log=$(cat "$LOG") has "apt batches its packages in one call" "apt-get install -y ripgrep" "$log" has "brew gets the tap-only package" "brew install can1357/tap/omp" "$log" has "brew gets the apt-less package" "brew install lazygit" "$log" @@ -386,14 +417,24 @@ has "snap gets the renamed binary" "snap install bw" has "flatpak gets the app id" "flatpak install" "$log" has "…with the real obsidian app id" "md.obsidian.Obsidian" "$log" # The one that matters on Linux: apt not knowing a name is not a dead end. -has "apt probes before it batches" "apt-cache show gh" "$log" +# `apt-cache show` is not an existence test -- it exits 0 for a name that has +# no installation candidate. The probe asks for the candidate now, so the +# assertion follows it. +has "apt probes before it batches" "apt-cache policy gh" "$log" +# docker-ce is here for one reason: it is in the apt cache but has no +# installation candidate, and `apt-cache show` exits 0 for exactly that shape. +# Under the old probe it stayed in the batch and apt refused all of them at +# once -- one unavailable name taking thirty installable ones down with it. +has "…and rejects a name with no candidate" "apt-cache policy docker-ce" "$log" +hasnt "…leaving it out of the apt batch" "apt-get install -y ripgrep docker-ce" "$log" +has "…while the rest of the batch survives" "apt-get install -y ripgrep" "$log" has "an apt name apt rejects moves to brew" "brew install gh" "$log" hasnt "…and is not left in the apt batch" "apt-get install -y gh" "$log" # npm cannot run before node, uv cannot run before uv. The pipeline is fixed # rather than sorted, so assert the order it actually produces. is "apt runs before npm" "yes" \ - "$(a=$(grep -n 'apt-get install' "$DOTUP_TEST_LOG" | head -1 | cut -d: -f1) - b=$(grep -n 'npm install' "$DOTUP_TEST_LOG" | head -1 | cut -d: -f1) + "$(a=$(grep -n 'apt-get install' "$LOG" | head -1 | cut -d: -f1) + b=$(grep -n 'npm install' "$LOG" | head -1 | cut -d: -f1) [ -n "$a" ] && [ -n "$b" ] && [ "$a" -lt "$b" ] && echo yes || echo no)" # Nothing runs as root that does not have to. hasnt "brew is never run through sudo" "sudo brew" "$log" @@ -401,16 +442,10 @@ hasnt "brew is never run through sudo" "sudo brew" printf '\n\033[1munattended — the boundary holds because of what is missing\033[0m\n' # A stale state file is the adversary here: it ticks the two things an # unattended run must never act on, and the run has to refuse both anyway. -rm -rf state; mkdir -p state -cat > state/selected <<'EOF' -core/ripgrep -docker/docker-ce -networking/openssh-server -private/bws-secrets -private/private-repo -EOF -: > "$DOTUP_TEST_LOG" -out=$(PATH="$FAKE:$PATH" sh $D --unattended --print 2>&1 || true) +newbox; BOXPATH=$SEALSUDO +pick core/ripgrep docker/docker-ce networking/openssh-server \ + private/bws-secrets private/private-repo +out=$(box --unattended --print 2>&1 || true) hasnt "unattended never installs a private row" "bws-secrets" "$out" hasnt "unattended never prompts for a password" "Password:" "$out" hasnt "unattended refuses a stale invasive tick" "docker-ce" "$out" @@ -418,21 +453,20 @@ hasnt "…including a listening ssh port" "openssh-server" "$out" has "…but still installs the safe defaults" "ripgrep" "$out" # Determinism: the same command twice, on the same machine, means the same # thing. A state file left by an interactive run must not change it. -a=$(PATH="$FAKE:$PATH" sh $D --unattended --print 2>/dev/null || true) -sh $D preset none -b=$(PATH="$FAKE:$PATH" sh $D --unattended --print 2>/dev/null || true) +a=$(box --unattended --print 2>/dev/null || true) +box preset none >/dev/null +b=$(box --unattended --print 2>/dev/null || true) is "unattended is computed, not inherited" "same" \ "$([ "$a" = "$b" ] && echo same || echo different)" # `set -e` would kill the subshell at the failing command, so the status is # captured through a || branch rather than read from $? afterwards. rc() { c=0; "$@" >/dev/null 2>&1 || c=$?; echo "$c"; } -is "unattended exits clean when nothing fails" "0" \ - "$(PATH="$FAKE:$PATH" rc sh $D --unattended --print)" +is "unattended exits clean when nothing fails" "0" "$(boxrc --unattended --print)" is "an unknown flag is refused" "2" "$(rc sh $D --nonsense)" # The negative test in PLAN.md phase 3 runs `zsh -ic exit` after an unattended # run. That only passes if zsh is in the unattended set. has "unattended installs the shell it configures" "zsh" \ - "$(reset; PATH="$FAKE:$PATH" sh $D --unattended --print 2>&1 | grep 'apt-get install' || true)" + "$(box --unattended --print 2>&1 | grep 'apt-get install' || true)" printf '\n\033[1mruns where it has to run\033[0m\n' # A machine with no sudo and no root is a real case -- a locked-down work box, a @@ -455,15 +489,440 @@ is "runs as a non-root user with no sudo on PATH" "0" \ "$(c=0; env -i HOME="$HOME" PATH="$MIN" DOTUP_STATE="$DOTUP_STATE" DOTUP_MANIFEST="$M" \ "$MIN/sh" $D --unattended --print >/dev/null 2>&1 || c=$? echo "$c")" -is "…and still resolves the whole default set" "1" \ - "$(env -i HOME="$HOME" PATH="$MIN" DOTUP_STATE="$DOTUP_STATE" DOTUP_MANIFEST="$M" \ - "$MIN/sh" $D --unattended --print 2>/dev/null | grep -c 'apt-get install -y ')" +# Two apt calls, and the distinction is the point: the manifest's packages go +# in ONE batch, and core/brew's prerequisites are a separate, named call. This +# used to assert "exactly 1" and would now read 2 with no way to tell a +# regression (the batch split apart) from the intended second call. +minprint=$(env -i HOME="$HOME" PATH="$MIN" DOTUP_STATE="$DOTUP_STATE" DOTUP_MANIFEST="$M" \ + "$MIN/sh" $D --unattended --print 2>/dev/null) +# Discriminate on the exact prerequisite list, not on `build-essential` -- +# that is itself a manifest package, so it appears in BOTH lines and a +# `grep -v` for it excluded the very batch this is meant to count. +is "…and still resolves the default set in ONE apt batch" "1" \ + "$(printf '%s\n' "$minprint" | grep 'apt-get install -y ' \ + | grep -vc 'build-essential procps curl file git$')" +is "…with brew's prerequisites as their own call" "1" \ + "$(printf '%s\n' "$minprint" | grep -c 'build-essential procps curl file git$')" +unset minprint rm -rf "$MIN" # `sudo` must not be glued onto anything that is not a system package manager. is "sudo is only ever used for apt, snap and dpkg" "" \ "$(grep -oE '\$\{SUDO:\+\$SUDO \}[a-z-]+' $D | sed 's/.*}//' | sort -u \ | grep -vE '^(apt-get|snap|rm|mkdir|tar|ln)$' || true)" +printf '\n\033[1mthe seal itself — a missing fake is an error, not the host tool\033[0m\n' +SB1=${TMPDIR:-/tmp}/dotup-sealtest.$$; rm -rf "$SB1"; mkdir -p "$SB1" +# The reason every section below can be believed. Under the old harness the +# fakes were PREPENDED to $PATH, so a package manager nobody had faked fell +# through to the real one; an audit of this suite invoked the developer's own +# `brew`. `env -i` plus a PATH that is only the seal makes that impossible. +is "the seal supplies the package managers" "$SEAL/brew" \ + "$(env -i PATH="$SEAL" "$SEAL/sh" -c 'command -v brew')" +is "…and a tool nobody faked is simply absent" "" \ + "$(env -i PATH="$SEAL" "$SEAL/sh" -c 'command -v wget || true')" +is "…so nothing in a sealed run can reach the host's own copy" "" \ + "$(env -i PATH="$SEAL" "$SEAL/sh" -c 'command -v brew npm curl unzip' | grep -v "^$SEAL/" || true)" +# find_tool ALSO probes four absolute directories, which no PATH can seal. The +# seal ships a fake for every name that would be shadowed there -- `command -v` +# is consulted first, so the fake wins -- and records the rest, because a case +# whose point is that a tool is absent must not quietly pass on a box where it +# is not. seal.sh refuses to build a hole it cannot honour. +if [ -s "$SEAL/.shadowed" ]; then + printf ' \033[33mnote\033[0m this host also carries %sat find_tool'"'"'s absolute probes\n' \ + "$(cut -f1 "$SEAL/.shadowed" | sort -u | tr '\n' ' ')" +fi +# A new `have foo` with no fake would silently reopen the hole. seal.sh refuses +# to build such a seal at all, so the failure lands on the harness rather than +# on an assertion that quietly starts measuring the host. +is "an unfaked probe refuses to build a seal" "1" \ + "$(sed 's|^have() {.*|&\n\thave nosuchtool|' $D > "$SB1/doctored" + c=0; DOTUP_SEAL_TARGET=$SB1/doctored sh ./seal.sh "$SB1/seal" >/dev/null 2>&1 || c=$? + echo "$c")" +is "…naming the tool it has never heard of" "1" \ + "$(DOTUP_SEAL_TARGET=$SB1/doctored sh ./seal.sh "$SB1/seal" 2>&1 >/dev/null \ + | grep -c nosuchtool || true)" + +printf '\n\033[1mprivate is never a package\033[0m\n' +# The invariant that makes --unattended safe. selected_packages drops every row +# flagged private, so a ticked private row cannot become an install command -- +# and the plan, which is what the installer reads, is where that has to show. +newbox +pick core/ripgrep private/bws-secrets private/private-repo +plan=$(box plan 2>&1 | plain) +hasnt "no private row reaches the plan" "private/" "$plan" +has "…while the ordinary row does" "core/ripgrep" "$plan" +out=$(box install 2>&1 | plain || true) +hasnt "…nor the installer's own accounting" "private/" "$out" +has "…and the ordinary one still installs" "apt-get install -y ripgrep" "$(cat "$LOG")" +# A private row is not merely unresolvable -- it is not offered at all. Ticking +# only private rows must produce an empty plan rather than two failures. +newbox; pick private/bws-secrets private/private-repo +is "a private-only selection plans nothing" "0 packages" \ + "$(box plan 2>&1 | plain | grep -o '[0-9]* packages' | head -1)" + +printf '\n\033[1munattended refuses invasive rows a stale state file still ticks\033[0m\n' +# `dotup --unattended` recomputes the selection first, which HIDES this filter: +# cmd_install never sees a stale tick because cmd_run overwrote it a moment +# earlier. `dotup --unattended install` is the same installer with the state +# file left alone -- which is what a cron job re-running a saved selection is. +newbox; pick core/ripgrep docker/docker-ce networking/openssh-server +out=$(box --unattended install 2>&1 | plain || true) +log=$(cat "$LOG") +has "it names what it refused" "refusing invasive packages" "$out" +hasnt "the daemon never reaches a package manager" "docker-ce" "$log" +hasnt "…nor does the listening ssh port" "openssh-server" "$log" +has "…and the safe package still installs" "ripgrep" "$log" +# Conditional on the flag, or it is a broken installer rather than a boundary. +newbox; pick core/ripgrep docker/docker-ce +box install >/dev/null 2>&1 || true +has "with a human present the same tick does install" "docker-ce" "$(cat "$LOG")" + +printf '\n\033[1ma package that fails is reported, and the run says so\033[0m\n' +# Reachable only because the fakes can be made to fail. Every assertion here +# survived deleting the code it is about, for want of a way to make brew lose. +newbox; BOXENV=FAKE_FAIL=brew +pick core/lazygit agents/omp core/ripgrep +out=$(box install 2>&1 | plain || true) +has "the failure is named by its manifest key" "core/lazygit" "$out" +has "…and by the second one too" "agents/omp" "$out" +has "…with what went wrong" "brew install failed" "$out" +has "…under a heading you can find" "did not install" "$out" +has "…and a count at the end" "package(s) did not install" "$out" +is "the run exits non-zero" "1" "$(boxrc install)" +has "…and the packages that CAN install still do" "apt-get install -y ripgrep" "$(cat "$LOG")" +newbox; pick core/ripgrep +is "a run with nothing failing exits 0" "0" "$(boxrc install)" + +printf '\n\033[1mone unknown name does not sink the batch\033[0m\n' +# Real apt refuses the WHOLE batch when one name is unusable. The fake fails in +# exactly that shape -- any call carrying more than one package -- so the only +# way through is the one-at-a-time retry, and the only way to see the retry is +# to make the batch lose. +newbox; BOXENV=FAKE_FAIL=apt-get:batch +pick core/ripgrep core/btop core/htop +out=$(box install 2>&1 | plain || true) +is "the batch is attempted first" "1" \ + "$(grep -cx 'apt-get install -y btop htop ripgrep' "$LOG" || true)" +is "…then btop on its own" "1" "$(grep -cx 'apt-get install -y btop' "$LOG" || true)" +is "…and htop" "1" "$(grep -cx 'apt-get install -y htop' "$LOG" || true)" +is "…and ripgrep" "1" "$(grep -cx 'apt-get install -y ripgrep' "$LOG" || true)" +has "it says why it is retrying" "retrying one at a time" "$out" +hasnt "…and nothing is left failed" "did not install" "$out" +is "…so the run still exits 0" "0" "$(boxrc install)" + +newbox; BOXENV=FAKE_FAIL=npm:batch +pick agents/codex agents/pi core/mermaid-cli +out=$(box install 2>&1 | plain || true) +is "npm batches first" "1" \ + "$(grep -c 'npm install -g @earendil-works/pi-coding-agent @mermaid-js/mermaid-cli @openai/codex' "$LOG" || true)" +is "…then @openai/codex on its own" "1" "$(grep -cx 'npm install -g @openai/codex' "$LOG" || true)" +is "…and the scoped mermaid name" "1" "$(grep -cx 'npm install -g @mermaid-js/mermaid-cli' "$LOG" || true)" +hasnt "…and nothing is left failed" "did not install" "$out" + +printf '\n\033[1ma tool installed a moment ago is still found\033[0m\n' +# uv lands in ~/.local/bin, which is on no PATH this process has -- dotup will +# not rewrite PATH for its own convenience, so find_tool looks in the places +# the run just wrote to instead. Stop looking there and every uv tool is +# reported missing on a machine where uv was installed sixty seconds earlier. +newbox +if holed uv; then + ln -s "$BOXPATH/_fake" "$SB/.local/bin/uv" + pick agents/specify-cli + out=$(box install 2>&1 | plain || true) + has "uv is found in ~/.local/bin, off PATH" "uv tool install specify-cli" "$(cat "$LOG")" + hasnt "…so nothing claims uv is missing" "uv missing" "$out" + is "…and the run exits clean" "0" "$(boxrc install)" +else + printf ' \033[33mskip\033[0m this host carries uv at one of find_tool'"'"'s absolute probes\n' +fi +# ~/bin is the other one, and it is not decorative: get.chezmoi.io installs +# there when -b is not given, which is what the README one-liner does. +newbox +if holed snap; then + mkdir -p "$SB/bin"; ln -s "$BOXPATH/_fake" "$SB/bin/snap" + pick core/bitwarden-cli + out=$(box install 2>&1 | plain || true) + has "…and in ~/bin, where get.chezmoi.io puts things" "snap install bw" "$(cat "$LOG")" + hasnt "…so nothing claims snapd is absent" "snapd is not present" "$out" +else + printf ' \033[33mskip\033[0m this host carries snap at one of find_tool'"'"'s absolute probes\n' +fi + +printf '\n\033[1mthe deb channel installs what it downloads\033[0m\n' +newbox; pick apps/chrome +out=$(box install 2>&1 | plain || true) +has "the vendor .deb is fetched" \ + "curl -fsSL https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb" "$(cat "$LOG")" +is "…and then handed to the package manager" "1" \ + "$(grep -c 'apt-get install -y .*\.deb' "$LOG" || true)" +hasnt "…and nothing is left failed" "did not install" "$out" +# A download that fails must not be followed by an install of a file that is +# not there -- and must be reported. +newbox; BOXENV=FAKE_FAIL=curl:deb; pick apps/chrome +out=$(box install 2>&1 | plain || true) +is "a failed download installs nothing" "0" \ + "$(grep -c 'apt-get install -y .*\.deb' "$LOG" || true)" +has "…and is reported against the package" "apps/chrome" "$out" + +printf '\n\033[1mthe neovim tarball lands where the symlink points\033[0m\n' +# Four commands name the same directory: the rm, the mkdir, tar's -C, and the +# symlink's target. A typo in any one of them installs nothing and leaves a +# dangling /usr/local/bin/nvim -- silently, because tar succeeded. Read the +# four back out of dotup's own dry run and make them agree, rather than +# trusting four separate spellings to stay in step. +newbox; pick core/neovim +dry=$(box --print install 2>&1 | plain || true) +xt=$(printf '%s\n' "$dry" | sed -n 's/.*tar -xzf [^ ]* -C \([^ ]*\) --strip-components=1.*/\1/p' | head -1) +xl=$(printf '%s\n' "$dry" | sed -n 's|.*ln -sf \(.*\)/bin/nvim /usr/local/bin/nvim.*|\1|p' | head -1) +xr=$(printf '%s\n' "$dry" | sed -n 's/.*rm -rf \([^ ]*\) && .*mkdir -p \([^ ]*\) &&.*/\1 \2/p' | head -1) +is "the tarball is extracted where the symlink points" "$xt" "$xl" +is "…into the directory that was cleared and recreated" "$xt $xt" "$xr" +is "…and that directory is /opt/nvim" "/opt/nvim" "$xt" + +printf '\n\033[1mthe picker fetches its own fzf, from the real release URL\033[0m\n' +# The stand-in for GitHub answers ONE path -- the release download URL -- and +# builds the tarball from the version named in it. So a preflight that reports +# a version is evidence the pin travelled through the URL into the binary; a +# wrong host or a wrong path shape is a failed fetch, exactly as it would be. +newbox +out=$(box preflight 2>&1 | plain || true) +is "the fetch goes to the fzf release download URL" "1" \ + "$(grep -c 'curl -sfL https://github.com/junegunn/fzf/releases/download/' "$LOG" || true)" +has "preflight resolves the fzf it just cached" "using $SB/.cache/dotup/fzf" "$out" +is "the binary that landed is the version the URL asked for" \ + "$(grep -o 'fzf-[0-9][0-9.]*-linux_' "$LOG" | head -1 | sed 's/^fzf-//; s/-linux_$//')" \ + "$(printf '%s\n' "$out" | sed -n 's/.*fzf (\([0-9][0-9.]*\),.*/\1/p' | head -1)" +is "…and it clears the verified floor" "1" \ + "$(printf '%s\n' "$out" | grep -c 'floor 0.44.0' || true)" +is "nothing was written to ~/.local/bin" "absent" \ + "$([ -e "$SB/.local/bin/fzf" ] && echo present || echo absent)" + +printf '\n\033[1mthe picker actually runs — every binding reaches fzf\033[0m\n' +# DU-C1. Comments were inserted BETWEEN the continued lines of the fzf +# invocation. The `\`-newline is stripped first, so the comment's own newline +# terminated the command: fzf ran with two binds, the lines below it ran as a +# command named `--bind`, and `dotup pick` returned 1 having drawn a picker +# where nothing but the cursor worked. Every other test here greps the SOURCE +# for bind strings, so all of them still passed. This one runs cmd_pick. +# +# Static half first: it needs nothing, and it catches the whole class anywhere +# in the file rather than only at the one site that was broken. +stray=$(awk '/\\$/ { cont=1; next } + cont && /^[[:space:]]*#/ { printf "%d: %s\n", NR, $0 } + { cont=0 }' $D) +is "no comment interrupts a line continuation" "" "$stray" + +if command -v python3 >/dev/null; then + # A real pty, because cmd_pick's first act is to open /dev/tty; a fake fzf, + # because the assertion is about the argv it was handed. FAKE_FZF_ARGV + # writes that argv one argument per line, so a bind that lost its + # continuation cannot be mistaken for one that survived. + newbox + BOXPATH=$SB/seal; sh ./seal.sh "$BOXPATH" --with fzf >/dev/null + ARGV=$SB/fzf.argv; : > "$SB/nothing.expect" + prc=0 + python3 ./ptydrive.py --timeout 30 --script "$SB/nothing.expect" -- \ + /usr/bin/env -i HOME="$SB" PATH="$BOXPATH" TERM=dumb LANG=C \ + TMPDIR="$SB/tmp" DOTUP_STATE="$SB/.config/dotfiles" \ + DOTUP_MANIFEST="$M" DOTUP_TEST_LOG="$LOG" FAKE_FZF_ARGV="$ARGV" \ + "$BOXPATH/sh" $D pick >/dev/null 2>&1 || prc=$? + is "dotup pick exits 0" "0" "$prc" + is "…having actually invoked fzf once" "1" \ + "$(grep -c '^fzf --ansi' "$LOG" || true)" + # Counted against the source, not against a literal 7: a binding added + # later must reach fzf too, and a test that hard-codes the count would not + # notice that it did not. + is "…with every --bind the source writes" \ + "$(grep -c -- '--bind ' $D)" "$(grep -cx -- '--bind' "$ARGV" || true)" + nobind=$(for k in space tab ctrl-t ctrl-a ctrl-x ctrl-o enter; do + grep -q "^$k:" "$ARGV" || echo "$k" + done; :) + is "…and all seven keys among them" "" "$nobind" + # The other half of what DU-C1 cost: the marker is written after fzf + # accepts, so with the command truncated it never was, and the defaults + # re-seeded over the user's selection on every subsequent run. + is "…and the completed-pick marker is written" "present" \ + "$([ -f "$SB/.config/dotfiles/picked" ] && echo present || echo absent)" +else + printf ' \033[33mskip\033[0m needs python3 to open a pty\n' +fi + +printf '\n\033[1mthe private tier — the three guards, exercised\033[0m\n' +# Each of these is one line of dotup, and each of them survived being INVERTED: +# the suite only ever checked that the words were in the file. What matters is +# which of the three fires, because they say three different things and only one +# of them is true at a time. +newbox; pick private/private-repo private/bws-secrets +out=$(box --unattended private &1 | plain || true) +has "--unattended: nobody is here to type it" "nobody is here to type" "$out" +hasnt "…so it never reaches a prompt" "Bootstrap URL" "$out" +hasnt "…and does not blame the terminal instead" "no terminal" "$out" +out=$(box private &1 | plain || true) +has "no tty: it says THAT instead" "no terminal" "$out" +hasnt "…and still never prompts" "Bootstrap URL" "$out" +hasnt "…and does not blame a flag nobody passed" "nobody is here to type" "$out" +out=$(box --print private &1 | plain || true) +has "--print describes the step" "prompt for URL, username, password" "$out" +hasnt "…and asks nothing" "Bootstrap URL:" "$out" +newbox; pick core/ripgrep +is "with no private row ticked there is no tier at all" "" \ + "$(box private &1 | plain || true)" + +printf '\n\033[1mthe private tier — driven through a real terminal\033[0m\n' +if command -v python3 >/dev/null 2>&1; then + # Everything below runs the SHIPPING cmd_private on a pty, against the + # sealed fakes: a stand-in endpoint that checks the password it was given, + # a stand-in GitHub that serves a bws zip and a checksum file, and a + # stand-in chezmoi that produces a source tree at the caller's umask. The + # assertions are then made against the filesystem -- modes, contents -- and + # against what the run actually said, not against what the source contains. + privrun() { + sc=$1; shift + (umask 022; python3 ./ptydrive.py --timeout 40 --script "$sc" -- \ + env -i HOME="$SB" PATH="$BOXPATH" TERM=dumb LANG=C TMPDIR="$SB/tmp" \ + DOTUP_STATE="$SB/.config/dotfiles" DOTUP_MANIFEST="$M" \ + DOTUP_TEST_LOG="$LOG" FAKE_BOOT_PW="$PW" \ + FAKE_BOOT_BLOB="$SB/blob.env" "$@" \ + "$BOXPATH/sh" $D private) 2>&1 | tr -d '\r' | plain + } + mode_of() { stat -c %a "$1" 2>/dev/null || echo missing; } + + # ---- a wrong password costs one password, not one reinstall ------------- + # These credentials are asked for at the very END of a run. Before the retry + # loop existed, one mistyped character meant repeating the entire install to + # get back to this prompt. So the first attempt here is wrong on purpose. + newbox; pick private/private-repo private/bws-secrets + PW=pw-ok-$$ + printf 'PRIVATE_REPO_URL=http://git:tok-%s@example.test/dotfiles-private.git\nBWS_ACCESS_TOKEN=bwstok-%s\n' \ + "$$" "$$" > "$SB/blob.env" + cat > "$SB/steps" </dev/null || true)" + is "the private source is unreadable to anyone else" "700" \ + "$(mode_of "$SB/.local/share/dotfiles-private")" + is "the credential file is 600" "600" \ + "$(mode_of "$SB/.config/dotfiles/private-credentials")" + is "…and holds the token the URL carried" "http://git:tok-$$@example.test" \ + "$(cat "$SB/.config/dotfiles/private-credentials" 2>/dev/null || true)" + hasnt "no credential-bearing URL reaches the transcript" "tok-$$@" "$out" + is "…nor any command line the run built" "0" \ + "$(grep -c '://[^ /]*:[^ /]*@' "$LOG" || true)" + has "chezmoi is called by its resolved path" \ + "chezmoi init --apply --source $SB/.local/share/dotfiles-private" "$(cat "$LOG")" + has "…against the private tier's OWN config file" \ + "-c $SB/.config/chezmoi/private.toml" "$(cat "$LOG")" + is "…which is never the config the public tier rewrites" "0" \ + "$(grep -c 'chezmoi/chezmoi.toml' "$LOG" || true)" + is "a later update can still authenticate" "1" \ + "$(grep -c "git -C $SB/.local/share/dotfiles-private config credential.helper" "$LOG" || true)" + has "bws is verified against the published checksum" "bws checksum verified" "$out" + is "…and installed, executable" "755" "$(mode_of "$SB/.local/bin/bws")" + is "…only after a token exists to use it" "yes" \ + "$(printf '%s\n' "$out" | awk '/bws token written/{t=NR} /fetching bws/{f=NR} + END{print (t && f && t < f) ? "yes" : "no"}')" + + # ---- the checksum is a decision, not a decoration ----------------------- + # A binary about to hold the key to every other credential. Inverting this + # one comparison installs exactly the file the check exists to reject, and + # says "verified" while doing it. + privbox() { # privbox ... -- fresh box, blob, password, prompts + newbox; pick "$@" + PW=pw-$$-$BOXN + printf 'PRIVATE_REPO_URL=http://git:tok@example.test/x.git\nBWS_ACCESS_TOKEN=bt-%s\n' \ + "$BOXN" > "$SB/blob.env" + { printf 'expect Bootstrap URL:\nsend http://example.test/boot\n' + printf 'expect Username:\nsend ben\n' + printf 'expect Password:\nsend %s\n' "$PW"; } > "$SB/steps" + } + privbox private/bws-secrets + out=$(privrun "$SB/steps" FAKE_BWS_SUMS=mismatch) + has "a checksum mismatch refuses to install" "checksum mismatch" "$out" + is "…and nothing lands in ~/.local/bin" "absent" \ + "$([ -e "$SB/.local/bin/bws" ] && echo present || echo absent)" + hasnt "…and it does not claim to have verified anything" "checksum verified" "$out" + privbox private/bws-secrets + out=$(privrun "$SB/steps" FAKE_BWS_SUMS=absent) + has "no checksum file says so out loud" "checksums unavailable" "$out" + is "…and installs anyway, as it says" "755" "$(mode_of "$SB/.local/bin/bws")" + + # ---- three failures, three different next moves ------------------------- + # One message for 401, 404 and an unreachable host is how a URL-shape bug + # spends an evening looking like a password problem. + qsteps() { printf 'expect Bootstrap URL \[\nsend q\n' >> "$SB/steps"; } + privbox private/bws-secrets; qsteps + has "a 404 names the route" "no bootstrap.env is there" \ + "$(privrun "$SB/steps" FAKE_BOOT_CODE=404)" + privbox private/bws-secrets; qsteps + has "an unreachable host says so" "could not reach that address" \ + "$(privrun "$SB/steps" FAKE_BOOT_CODE=000)" + privbox private/bws-secrets; qsteps + has "anything else reports its code" "answered HTTP 500" \ + "$(privrun "$SB/steps" FAKE_BOOT_CODE=500)" + # An error page must never be parsed as the credential blob. + privbox private/bws-secrets; qsteps + out=$(privrun "$SB/steps" FAKE_BOOT_CODE=404) + is "a non-200 body is never read as a token" "absent" \ + "$([ -e "$SB/.config/bitwarden/bws-token" ] && echo present || echo absent)" + has "…and q leaves a public-only machine" "public-only machine" "$out" + + # ---- it asks again, but not forever ------------------------------------- + privbox private/bws-secrets + { printf 'expect Bootstrap URL:\nsend http://example.test/boot\n' + printf 'expect Username:\nsend ben\n' + printf 'expect Password:\nsend bad1\n' + i=2 + while [ "$i" -le 5 ]; do + printf 'expect wrong username or password\n' + printf 'expect Bootstrap URL \[\nsend\nexpect Username \[\nsend\n' + printf 'expect Password:\nsend bad%s\n' "$i" + i=$((i + 1)) + done + printf 'expect five failed attempts\n'; } > "$SB/steps" + out=$(privrun "$SB/steps") + has "five wrong passwords stop the loop" "five failed attempts" "$out" + has "…pointing at the cheap way back in" "dotup private" "$out" + is "…having asked the endpoint five times, not four or six" "5" \ + "$(grep -c 'bootstrap.env' "$LOG" || true)" + + # ---- q at the first prompt is a supported answer ------------------------ + privbox private/private-repo private/bws-secrets + printf 'expect Bootstrap URL:\nsend q\n' > "$SB/steps" + out=$(privrun "$SB/steps") + has "q at the first prompt is public-only" "public-only machine" "$out" + is "…and writes nothing at all" "0" \ + "$(grep -c 'bootstrap.env' "$LOG" || true)" + is "…no token" "absent" "$([ -e "$SB/.config/bitwarden/bws-token" ] && echo present || echo absent)" + is "…and no clone" "absent" "$([ -e "$SB/.local/share/dotfiles-private" ] && echo present || echo absent)" +else + printf ' \033[33mskip\033[0m needs python3 to open a pty\n' +fi + printf '\n\033[1minteractive loop (needs a pty)\033[0m\n' if command -v fzf >/dev/null && command -v curl >/dev/null && command -v script >/dev/null; then out=$(sh ./listen-test.sh 2>/dev/null | tr -d '\r' || true)