From 042320208e5b8e8ebc05a4290f68a23e203599bf Mon Sep 17 00:00:00 2001 From: bcherb2 Date: Fri, 21 Aug 2026 22:34:30 -0400 Subject: [PATCH] test: seal the fake PATH instead of prepending it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The old harness put .tests/fakebin at the front of $PATH, which fails OPEN: a package manager with no fake fell through to the real one, and an audit of this suite invoked the host's actual `brew`. The twelve one-line fakes are replaced by a single dispatcher every fake symlinks to, plus seal.sh, which builds the WHOLE of $PATH for a run — real binaries for the pure tools, fakes for anything that installs, downloads or needs root, and command-not-found for everything else. Runs go under `env -i` so they inherit nothing. Two audits ship with it: any `have`/`find_tool` probe the seal has never heard of is an error, and the four absolute paths find_tool probes are written to .shadowed so a test needing a tool to be genuinely absent can say so. The dispatcher also fails on demand (FAKE_FAIL), which is what makes the installers' failure paths reachable at all — the apt and npm one-at-a-time retries, note_fail, and cmd_install's non-zero exit had all been unreachable, and all three survived being deleted outright. _curl stands in for every remote the installer talks to and for nothing else: an unrecognised URL is a failed download, so a typo'd host shows up as the failure it would really be. The fzf fake records its argv, which is what makes the picker's bindings testable. --- .tests/fakebin/_curl | 96 ++++++++++++++++++++++++++ .tests/fakebin/_fake | 130 ++++++++++++++++++++++++++++++++++++ .tests/fakebin/_log | 32 --------- .tests/fakebin/apt-cache | 1 - .tests/fakebin/apt-get | 1 - .tests/fakebin/brew | 1 - .tests/fakebin/curl | 1 - .tests/fakebin/flatpak | 1 - .tests/fakebin/npm | 1 - .tests/fakebin/snap | 1 - .tests/fakebin/sudo | 1 - .tests/fakebin/tar | 1 - .tests/fakebin/uv | 1 - .tests/fakebin/xcode-select | 1 - .tests/seal.sh | 109 ++++++++++++++++++++++++++++++ 15 files changed, 335 insertions(+), 43 deletions(-) create mode 100644 .tests/fakebin/_curl create mode 100755 .tests/fakebin/_fake delete mode 100755 .tests/fakebin/_log delete mode 120000 .tests/fakebin/apt-cache delete mode 120000 .tests/fakebin/apt-get delete mode 120000 .tests/fakebin/brew delete mode 120000 .tests/fakebin/curl delete mode 120000 .tests/fakebin/flatpak delete mode 120000 .tests/fakebin/npm delete mode 120000 .tests/fakebin/snap delete mode 120000 .tests/fakebin/sudo delete mode 120000 .tests/fakebin/tar delete mode 120000 .tests/fakebin/uv delete mode 120000 .tests/fakebin/xcode-select create mode 100755 .tests/seal.sh diff --git a/.tests/fakebin/_curl b/.tests/fakebin/_curl new file mode 100644 index 0000000..930b52c --- /dev/null +++ b/.tests/fakebin/_curl @@ -0,0 +1,96 @@ +# Sourced by _fake when it is invoked as `curl`. Stands in for every remote the +# installer talks to, and for nothing else: a URL this file does not recognise +# is a failed download, not a silent success, so a typo'd host or a moved path +# shows up as the failure it would really be. +# +# Knobs, all set by the caller: +# FAKE_BOOT_BLOB file whose contents the bootstrap endpoint returns on 200 +# FAKE_BOOT_PW the password that gets a 200; anything else gets a 401 +# FAKE_BOOT_CODE force a code (404, 000, 500 ...) whatever the password +# FAKE_BWS_SUMS match (default) | mismatch | absent +# FAKE_FAIL=curl:deb | curl:chezmoi | curl:fzf fail just that one download + +_fb=$(dirname "$0") # the fakebin directory +url=; out=; prev=; readcfg=0 +for a in "$@"; do + case $prev in -o) out=$a ;; -K) [ "$a" = - ] && readcfg=1 ;; esac + case $a in http://*|https://*|get.chezmoi.io|*.io|*.sh) [ -z "$url" ] && url=$a ;; esac + prev=$a +done +cfg= +[ "$readcfg" = 1 ] && cfg=$(cat) # always drain: the writer is a pipe + +emit() { if [ -n "$out" ]; then cat > "$out"; else cat; fi; } + +# A tiny installer script, printed the way get.chezmoi.io and astral.sh print +# theirs. It installs the FAKE of the same name, so whatever it drops behaves +# like every other fake in this directory. +installer() { # $1 tool, $2 default dir + printf '%s\n' \ + "b=$2" \ + 'while [ $# -gt 0 ]; do case $1 in -b) b=$2; shift 2 ;; *) shift ;; esac; done' \ + "mkdir -p \"\$b\" && ln -sf $_fb/_fake \"\$b/$1\" && echo \"installed $1 to \$b\"" +} + +case $url in +*get.chezmoi.io*) + [ "$mode" = chezmoi ] && exit 22 + installer chezmoi "\$HOME/.local/bin"; exit 0 ;; +*astral.sh/uv/install.sh*) + installer uv "\$HOME/.local/bin"; exit 0 ;; +*go.dev/VERSION*) + echo go1.99.0; exit 0 ;; +*api.github.com*) + echo ' "browser_download_url": "https://example.invalid/fake_amd64.deb"'; exit 0 ;; + +# ------------------------------------------------------------------- fzf ---- +# Only the real release path answers. The tarball is built here from the +# version in the URL, so `dotup preflight` reporting "0.74.2" is evidence that +# the pinned version travelled through the URL and into the binary -- not that +# a fixture happened to say so. +https://github.com/junegunn/fzf/releases/download/*) + [ "$mode" = fzf ] && exit 22 + f=${url##*/} # fzf-0.74.2-linux_amd64.tar.gz + v=${f#fzf-}; v=${v%%-*} + d=$(mktemp -d); printf '#!/bin/sh\ncase ${1:-} in --version) echo "%s (fake)" ;; esac\nexit 0\n' "$v" > "$d/fzf" + chmod 755 "$d/fzf"; tar -czf - -C "$d" fzf; rm -rf "$d"; exit 0 ;; + +# ------------------------------------------------------------------- bws ---- +*/sdk-sm/releases/download/*bws-sha256-checksums-*) + case ${FAKE_BWS_SUMS:-match} in + absent) exit 22 ;; + mismatch) s=0000000000000000000000000000000000000000000000000000000000000000 ;; + *) s=$(printf 'PK-fake-bws-zip\n' | sha256sum | cut -d' ' -f1) ;; + esac + v=${url##*bws-sha256-checksums-}; v=${v%.txt} + for t in x86_64-unknown-linux-musl aarch64-unknown-linux-musl macos-universal; do + printf '%s bws-%s-%s.zip\n' "$s" "$t" "$v" + done | emit + exit 0 ;; +*/sdk-sm/releases/download/*.zip) + printf 'PK-fake-bws-zip\n' | emit; exit 0 ;; + +# -------------------------------------------------------------- bootstrap --- +*/bootstrap.env) + pw=$(printf '%s\n' "$cfg" | sed -n 's/^user = "[^:]*:\(.*\)"$/\1/p' | head -1) + code=${FAKE_BOOT_CODE:-} + if [ -z "$code" ]; then + if [ "$pw" = "${FAKE_BOOT_PW:-}" ]; then code=200; else code=401; fi + fi + if [ "$code" = 200 ]; then cat "${FAKE_BOOT_BLOB:?FAKE_BOOT_BLOB unset}"; fi + printf '\n%s' "$code" + [ "$code" = 000 ] && exit 7 + exit 0 ;; +esac + +# --------------------------------------------------------------- anything --- +# A download to a file still has to produce the file; a download of something +# this stand-in has never heard of is a failure, which is the honest answer. +case $url in +*.deb|*.tar.gz|*.tgz) + [ "$mode" = deb ] && exit 22 + [ -n "$out" ] && { : > "$out"; exit 0; } + exit 0 ;; +esac +[ -n "$out" ] && { : > "$out"; exit 0; } +exit 22 diff --git a/.tests/fakebin/_fake b/.tests/fakebin/_fake new file mode 100755 index 0000000..a544c93 --- /dev/null +++ b/.tests/fakebin/_fake @@ -0,0 +1,130 @@ +#!/bin/sh +# One dispatcher; every fake in this directory is a symlink to it. +# +# The old version of this file did two things wrong, and both let real bugs +# through: +# +# it always succeeded so no test could ever reach an installer's failure +# path -- the apt and npm one-at-a-time retries, the +# "did not install" report, and cmd_install's non-zero +# exit were all unreachable, and all three survived +# being deleted outright. +# it failed OPEN the fakes were PREPENDED to $PATH, so a package +# manager with no fake here fell through to the real +# one. An audit of this suite invoked the host's real +# `brew`. A fake that is missing must be an ERROR, not +# a silent hand-off to the machine. +# +# The second is fixed by seal.sh, which builds a directory holding exactly the +# tools a run may touch -- fakes for anything that installs, downloads or needs +# root, real binaries for the pure ones -- and runs dotup under `env -i` with +# that directory as the WHOLE of PATH. Anything not listed is command-not-found. +# +# ---------------------------------------------------------------- failure --- +# FAKE_FAIL is a comma-separated list of injections: +# +# FAKE_FAIL=brew brew exits 1 every time +# FAKE_FAIL=apt-get:batch apt-get fails only when asked for >1 package, +# which is what a real "unable to locate package" +# inside a batch looks like: the batch dies and +# the one-at-a-time retry is the only thing that +# saves the other thirty. +# FAKE_FAIL=npm:batch the same shape for npm. +# FAKE_FAIL=curl:deb curl fails only for the .deb download. +# +# Several may be combined: FAKE_FAIL=brew,apt-get:batch +set -u +me=${0##*/} +printf '%s %s\n' "$me" "$*" >> "${DOTUP_TEST_LOG:?DOTUP_TEST_LOG unset}" + +# The mode this run injects for THIS tool: "always", a tool-specific word, or +# empty for "behave". +mode= +_oldifs=$IFS +IFS=, +for _e in ${FAKE_FAIL:-}; do + case $_e in + "$me") mode=always; break ;; + "$me":*) mode=${_e#*:}; break ;; + esac +done +IFS=$_oldifs +[ "$mode" = always ] && exit 1 + +# Count the package-ish arguments of a batch call: everything after the +# subcommand that is not a flag and not the flag's value. +count_pkgs() { + n=0 + for a in "$@"; do + case $a in + -*|install|update|show|get|tool|-g|-y|--*) continue ;; + *=*) continue ;; + esac + n=$((n + 1)) + done + printf '%s\n' "$n" +} + +case $me in +apt-get) + case ${1:-} in update) exit 0 ;; esac + [ "$mode" = batch ] && [ "$(count_pkgs "$@")" -gt 1 ] && exit 1 + exit 0 ;; +npm) + [ "$mode" = batch ] && [ "$(count_pkgs "$@")" -gt 1 ] && exit 1 + exit 0 ;; +apt-cache) + # Stand in for a real apt cache. Two names are special, and both are real + # behaviours of stock Ubuntu that the installer has to tell apart: + # + # gh genuinely absent -- `apt-cache policy` prints nothing. The + # manifest's real example of a name that falls through to brew. + # docker-ce present in the cache but with `Candidate: (none)`, because + # something else Conflicts/Replaces it. `apt-cache show` exits + # 0 for this, which is why the old probe kept it in the batch + # and let apt refuse all thirty packages at once. + case ${1:-} in + policy) + case ${2:-} in + gh) exit 0 ;; + docker-ce) printf '%s:\n Installed: (none)\n Candidate: (none)\n' "$2"; exit 0 ;; + esac + printf '%s:\n Installed: (none)\n Candidate: 1.0-fake\n' "$2"; exit 0 ;; + esac + case ${2:-} in gh) exit 100 ;; esac + printf 'Package: %s\n' "${2:-}"; exit 0 ;; +unzip) + # `unzip -oq -d ` -- produce the one member ensure_bws looks for. + d=; prev= + for a in "$@"; do [ "$prev" = -d ] && { d=$a; break; }; prev=$a; done + [ -n "$d" ] || exit 1 + mkdir -p "$d" && printf '#!/bin/sh\necho "bws 2.1.0"\n' > "$d/bws" \ + && chmod 755 "$d/bws" && exit 0 + exit 1 ;; +dpkg) + # install_deb fills %a in a gh: asset name from this. + case ${1:-} in --print-architecture) echo amd64 ;; esac + exit 0 ;; +chezmoi) + # `chezmoi init --apply --source DIR -c CFG URL`. Produce the shape the + # caller checks for -- a source tree with a .git in it -- and NOTHING else, + # at the caller's umask, so a missing `chmod -R go-rwx` is visible as a mode. + src=; prev= + for a in "$@"; do [ "$prev" = --source ] && { src=$a; break; }; prev=$a; done + [ -n "$src" ] || exit 0 + mkdir -p "$src/.git" && : > "$src/.git/config" && : > "$src/README" || exit 1 + exit 0 ;; +curl) . "$(dirname "$0")/_curl" ;; +fzf) + # The picker's own fzf, and the only fake here whose ARGV is the thing + # under test. `--bind` strings are built across a dozen continued lines, so + # one stray comment between them silently truncates the command and the + # picker draws with two binds instead of seven. Dump argv one argument per + # line -- a bind that lost its continuation cannot masquerade as one that + # survived -- and behave like an accept: read the piped rows, print nothing. + case ${1:-} in --version) echo "${FAKE_FZF_VERSION:-9.9.9} (fake)"; exit 0 ;; esac + [ -n "${FAKE_FZF_ARGV:-}" ] && printf '%s\n' "$@" > "$FAKE_FZF_ARGV" + cat > /dev/null + exit "${FAKE_FZF_RC:-0}" ;; +esac +exit 0 diff --git a/.tests/fakebin/_log b/.tests/fakebin/_log deleted file mode 100755 index 43ae177..0000000 --- a/.tests/fakebin/_log +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/sh -# Every fake in this directory is a symlink to this file. It records the call -# and succeeds, so the installer can be driven end to end without a single -# package being installed, on this machine or in a container. -# -# `sudo` is faked too, and deliberately does NOT exec its argument: a run as -# root in a container would otherwise reach the real `rm -rf /opt/nvim`. -printf '%s %s\n' "${0##*/}" "$*" >> "${DOTUP_TEST_LOG:?DOTUP_TEST_LOG unset}" - -case ${0##*/} in -apt-cache) - # Stand in for a real apt cache: everything is known except `gh`, which is - # genuinely absent from stock Ubuntu and is the manifest's real example of a - # name that has to fall through to brew. - case $2 in gh) exit 100 ;; esac - printf 'Package: %s\n' "$2"; exit 0 ;; -curl) - # Enough shape for the two bespoke handlers that parse a response. - for a in "$@"; do - case $a in - *go.dev/VERSION*) echo go1.99.0; exit 0 ;; - *api.github.com*) echo ' "browser_download_url": "https://example.invalid/fake_amd64.deb"'; exit 0 ;; - esac - done - prev= - for a in "$@"; do - [ "$prev" = "-o" ] && { : > "$a"; exit 0; } - prev=$a - done - exit 0 ;; -esac -exit 0 diff --git a/.tests/fakebin/apt-cache b/.tests/fakebin/apt-cache deleted file mode 120000 index 46acdae..0000000 --- a/.tests/fakebin/apt-cache +++ /dev/null @@ -1 +0,0 @@ -_log \ No newline at end of file diff --git a/.tests/fakebin/apt-get b/.tests/fakebin/apt-get deleted file mode 120000 index 46acdae..0000000 --- a/.tests/fakebin/apt-get +++ /dev/null @@ -1 +0,0 @@ -_log \ No newline at end of file diff --git a/.tests/fakebin/brew b/.tests/fakebin/brew deleted file mode 120000 index 46acdae..0000000 --- a/.tests/fakebin/brew +++ /dev/null @@ -1 +0,0 @@ -_log \ No newline at end of file diff --git a/.tests/fakebin/curl b/.tests/fakebin/curl deleted file mode 120000 index 46acdae..0000000 --- a/.tests/fakebin/curl +++ /dev/null @@ -1 +0,0 @@ -_log \ No newline at end of file diff --git a/.tests/fakebin/flatpak b/.tests/fakebin/flatpak deleted file mode 120000 index 46acdae..0000000 --- a/.tests/fakebin/flatpak +++ /dev/null @@ -1 +0,0 @@ -_log \ No newline at end of file diff --git a/.tests/fakebin/npm b/.tests/fakebin/npm deleted file mode 120000 index 46acdae..0000000 --- a/.tests/fakebin/npm +++ /dev/null @@ -1 +0,0 @@ -_log \ No newline at end of file diff --git a/.tests/fakebin/snap b/.tests/fakebin/snap deleted file mode 120000 index 46acdae..0000000 --- a/.tests/fakebin/snap +++ /dev/null @@ -1 +0,0 @@ -_log \ No newline at end of file diff --git a/.tests/fakebin/sudo b/.tests/fakebin/sudo deleted file mode 120000 index 46acdae..0000000 --- a/.tests/fakebin/sudo +++ /dev/null @@ -1 +0,0 @@ -_log \ No newline at end of file diff --git a/.tests/fakebin/tar b/.tests/fakebin/tar deleted file mode 120000 index 46acdae..0000000 --- a/.tests/fakebin/tar +++ /dev/null @@ -1 +0,0 @@ -_log \ No newline at end of file diff --git a/.tests/fakebin/uv b/.tests/fakebin/uv deleted file mode 120000 index 46acdae..0000000 --- a/.tests/fakebin/uv +++ /dev/null @@ -1 +0,0 @@ -_log \ No newline at end of file diff --git a/.tests/fakebin/xcode-select b/.tests/fakebin/xcode-select deleted file mode 120000 index 46acdae..0000000 --- a/.tests/fakebin/xcode-select +++ /dev/null @@ -1 +0,0 @@ -_log \ No newline at end of file diff --git a/.tests/seal.sh b/.tests/seal.sh new file mode 100755 index 0000000..138f0fe --- /dev/null +++ b/.tests/seal.sh @@ -0,0 +1,109 @@ +#!/bin/sh +# Build a SEALED bin directory: the whole of $PATH for a run under test. +# +# sh seal.sh [--with name ...] [--without name ...] +# +# The old harness prepended .tests/fakebin to the caller's $PATH. That fails +# OPEN: a package manager with no fake here simply fell through to the real one, +# and an audit of this suite invoked the host's actual `brew`. The seal fails +# CLOSED. It holds exactly two kinds of thing -- +# +# REAL pure, local, side-effect-free tools dotup is entitled to use: awk, +# sed, install, sha256sum, tar. Faking these would weaken the test; +# `sha256sum` in particular has to be the real one or the checksum +# comparison is checking a fixture against itself. +# FAKE anything that installs, downloads, or needs root. Symlinks to _fake. +# +# -- and nothing else. A tool that is neither is command-not-found, which is a +# visible failure rather than a silent hand-off to the machine. +# +# One hole cannot be closed from here and is reported rather than hidden: +# find_tool probes /usr/local/bin, /usr/local/go/bin, /home/linuxbrew/... and +# /opt/homebrew/bin by ABSOLUTE path, so on a developer box those directories +# can satisfy a lookup no matter what PATH says. The seal therefore ships a fake +# for every name that is shadowed there -- `command -v` is tried first, so the +# fake wins -- and writes the shadow list to /.shadowed so a test that +# needs a tool to be genuinely ABSENT can say so instead of quietly passing. +set -eu + +# gzip is here because tar shells out to it: without it `tar xz` dies with +# "gzip: not found" -- which is the seal working, and is how it was found. +REAL='sh awk grep sed sort cut tr head tail id uname mkdir cp mv rm cat + dirname basename chmod mktemp find install sha256sum stty tar gzip ln + touch true false env expr wc' +# Deliberately NOT here by default, each for a reason a test depends on: +# sudo absent means SUDO='' and apt-get is invoked directly, which is the +# only way FAKE_FAIL=apt-get can reach the installer at all. +# fzf absent is what makes ensure_fzf actually fetch one. +# bws absent is what makes the private tier actually install it. +FAKE='curl apt-get apt-cache brew npm uv snap flatpak xcode-select + git dpkg unzip chezmoi' +OPTIONAL='sudo fzf bws nvim go node' + +dir=${1:?usage: seal.sh [--with name ...] [--without name ...]}; shift +# Everything the seal KNOWS about, fixed before --without takes anything out: a +# hole punched on purpose is still an accounted-for tool, and must not trip the +# coverage audit below. +KNOWN="$REAL $FAKE $OPTIONAL" +mode= +for a in "$@"; do + case $a in + --with) mode=with ;; + --without) mode=without ;; + *) case $mode in + with) FAKE="$FAKE $a"; KNOWN="$KNOWN $a" ;; + without) FAKE=$(printf '%s\n' $FAKE | grep -vx "$a" | tr '\n' ' ') ;; + *) echo "seal.sh: stray argument $a" >&2; exit 2 ;; + esac ;; + esac +done + +here=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +FB=$here/fakebin +[ -x "$FB/_fake" ] || { echo "seal.sh: no $FB/_fake" >&2; exit 1; } + +rm -rf "$dir"; mkdir -p "$dir" +ln -s "$FB/_fake" "$dir/_fake" +ln -s "$FB/_curl" "$dir/_curl" + +# Resolved by looking, not by `command -v`: an interactive shell reports +# aliases and functions, and a symlink to a bare name is a loop. +miss= +for t in $REAL; do + found= + for bd in /usr/bin /bin /usr/local/bin /sbin /usr/sbin; do + [ -x "$bd/$t" ] && { ln -sf "$bd/$t" "$dir/$t"; found=1; break; } + done + [ -n "$found" ] || miss="$miss $t" +done +[ -z "$miss" ] || { echo "seal.sh: host is missing real tools:$miss" >&2; exit 1; } + +for t in $FAKE; do ln -sf "$dir/_fake" "$dir/$t"; done + +# ------------------------------------------------------------- the audits --- +# 1. Anything dotup probes for must be accounted for. A new `have foo` with no +# fake would otherwise fall through to the host the moment someone adds one. +D=${DOTUP_SEAL_TARGET:-$here/../dot_local/bin/executable_dotup} +probes=$(grep -v '^[[:space:]]*#' "$D" \ + | grep -oE '\b(have|find_tool) [a-z][a-z0-9-]*' \ + | awk '{print $2}' | sort -u || :) +unknown= +for p in $probes; do + case " $KNOWN " in *" $p "*) ;; *) unknown="$unknown $p" ;; esac +done +[ -z "$unknown" ] || { + echo "seal.sh: dotup probes for tools the seal has never heard of:$unknown" >&2 + echo " add them to REAL, FAKE or OPTIONAL -- do not let them fall through" >&2 + exit 1; } + +# 2. The absolute-path hole, reported rather than papered over. $KNOWN, not +# $FAKE: a name taken out with --without is exactly the name a test wants to +# be absent, so it is the one that most needs checking. +: > "$dir/.shadowed" +for t in $KNOWN; do + for bd in /usr/local/bin /usr/local/go/bin /home/linuxbrew/.linuxbrew/bin /opt/homebrew/bin; do + [ -x "$bd/$t" ] || continue + [ -e "$dir/$t" ] || printf '%s\t%s\n' "$t" "$bd/$t" >> "$dir/.shadowed" + done +done +printf '%s\n' "$dir"