ok 0.2s MASTER/tools/rhythm_lint.rb ok 0.7s MASTER/tools/rails/operator/fallback_drift_lint.rb ok 0.1s MASTER/tools/empty_state_lint.rb ok 0.1s MASTER/tools/adhoc_empty_lint.rb ok 0.4s MASTER/tools/chrome_i18n_lint.rb ok 0.4s MASTER/tools/rails/operator/dialect_token_drift_check.rb ok 0.4s OPENBSD/test/test_deploy_paths_exist.rb FAIL 4.3s OPENBSD/test/test_vps_deploy_contract.rb Run options: --seed 2837 # Running: F...Ffinish:log:66: too many arguments ........F...........................................................F......... Finished in 4.155771s, 19.9722 runs/s, 173.2530 assertions/s. 1) Failure: VpsDeployContractTest#test_recovery_plan_is_explicit_and_never_deploys_or_checks_out_code [test/test_vps_deploy_contract.rb:1097]: Expected "#!/usr/bin/env zsh\n# Serial VPS deploy entrypoint — sync, CI, restart, health, deploy stamp.\n# Usage: zsh OPENBSD/bin/vps-deploy all|GROUP|APP\nset -euo pipefail\n\n# SSH executes this as a non-login shell on vm23; pin the OpenBSD package path.\nexport PATH=/usr/local/bin:/usr/local/sbin:/usr/bin:/usr/sbin:/bin:/sbin\n\nusage() {\n print -r -- \"usage: zsh OPENBSD/bin/vps-deploy all|GROUP|APP (runs as root; re-execs via doas)\n\n all every active registered surface, including MASTER web\n GROUP every registered app whose deploy_root belongs to that namespace\n APP one exact app ID from RAILS/apps.yml\n recover --plan inspect interrupted-deploy recovery receipts\n recover --apply restore only services recorded as up before that run\n VPS_DEPLOY_SHA= deploy exactly this commit (default: origin/main, fetched once)\n VPS_DEPLOY_PREFLIGHT_SHA= the operator's machine already proved this commit boots\n VPS_DEPLOY_CI_SHA= the operator's machine ran the app's bin/ci green on this commit; the box skips its suite\n VPS_DEPLOY_SKIP_PREFLIGHT=1 break-glass, needs I_UNDERSTAND_FAST_DEPLOY=1\n VPS_DEPLOY_SKIP_TESTS=1 break-glass: no test suite anywhere, needs I_UNDERSTAND_FAST_DEPLOY=1\n SKIP_CI=1 skip vps_ci.sh; \\${app}.sh and its runtime check still run\n VPS_DEPLOY_DRIFT=fail refuse to deploy when the box's configs differ from the repo\n PUB5_DEPLOY_BROWSER_CHECKS=1 add the browser checks after restart\n VPS_DEPLOY_LOG=/path aggregate log (default /var/log/pub5/vps-deploy.log)\n VPS_DEPLOY_LOG_DIR=/path log directory (default /var/log/pub5)\n VPS_DEPLOY_RUN_LOG=/path per-run log (default auto under LOG_DIR/runs/)\n\n Git fetch/checkout always runs as user dev so the /home/dev/pub5 tree and the\n GitHub deploy key stay owned and reachable. Service control uses pub5-admin\n directly when already root.\"\n}\ncase ${1:-} in\n -h|--help) usage; exit 0 ;;\nesac\n\n# Prefer root. Passwordless doas (dev in wheel / permit nopass) re-execs so the\n# operator can still type the command as dev. Git never runs as root: root has\n# no GitHub host key by design, and a root checkout would leave root-owned files\n# in the dev tree. Service control calls the broker directly when uid 0.\n#\n# doas strips the caller environment (the dev rule has no keepenv — intentional,\n# so RUBYOPT and friends cannot cross the privilege boundary). Re-exec through\n# /usr/bin/env with a prefix allowlist (VPS_DEPLOY_*, PUB5_DEPLOY_*, plus PUB5_ROOT,\n# SKIP_CI, I_UNDERSTAND_FAST_DEPLOY) so pin, preflight, canary and logging\n# knobs survive. Not keepenv: RUBYOPT and friends still cannot cross.\nif [[ $(id -u) -ne 0 ]]; then\n if command -v doas >/dev/null 2>&1; then\n self=$0\n if [[ $self != /* ]]; then\n self=\"$(cd -- \"$(dirname -- \"$self\")\" && pwd)/$(basename -- \"$self\")\"\n fi\n typeset -a deploy_env\n # Prefix allowlist (not keepenv): every VPS_DEPLOY_* and PUB5_DEPLOY_* that\n # is set in the caller crosses doas via /usr/bin/env. Plus a few one-offs\n # the deploy reads. New knobs named under those prefixes need no edit here.\n for _name in ${(k)parameters}; do\n case $_name in\n VPS_DEPLOY_*|PUB5_DEPLOY_*|PUB5_ROOT|SKIP_CI|I_UNDERSTAND_FAST_DEPLOY)\n deploy_env+=(\"$_name=${(P)_name}\")\n ;;\n esac\n done\n exec doas -n /usr/bin/env \"${deploy_env[@]}\" zsh \"$self\" \"$@\"\n fi\n print -u2 \"vps-deploy: must run as root (or with passwordless doas)\"\n print -u2 \" try: doas zsh OPENBSD/bin/vps-deploy ${1:-all}\"\n exit 2\nfi\n\n# Unprivileged owner of the checkout and GitHub credentials on this box.\nDEPLOY_USER=${VPS_DEPLOY_USER:-dev}\nDEPLOY_HOME=${VPS_DEPLOY_HOME:-/home/$DEPLOY_USER}\nrepo=${PUB5_ROOT:-/home/dev/pub5}\nsource \"$repo/OPENBSD/lib/ruby_select.sh\"\ninventory_target_kind() {\n \"$RUBY\" -r\"$repo/OPENBSD/lib/deploy_inventory\" -e \\\n 'print Deploy::Inventory.new(root: ARGV[0]).target_kind(ARGV[1])' \"$repo\" \"$1\"\n}\n\ninventory_app_group() {\n \"$RUBY\" -r\"$repo/OPENBSD/lib/deploy_inventory\" -e '\n app = Deploy::Inventory.new(root: ARGV[0]).apps.find { |row| row.name == ARGV[1] }\n print app&.deploy_group\n ' \"$repo\" \"$1\"\n}\n\ninventory_names() {\n local target=$1 degraded=${2:-0}\n \"$RUBY\" -r\"$repo/OPENBSD/lib/deploy_inventory\" -e '\n inventory = Deploy::Inventory.new(root: ARGV[0])\n rows = ARGV[1] == \"all\" ? inventory.ordered_apps(degraded: ARGV[2] == \"1\") :\n inventory.ordered_apps(group: ARGV[1])\n puts rows.map(&:name)\n ' \"$repo\" \"$target\" \"$degraded\"\n}\n\n# Run a shell snippet as DEPLOY_USER without pulling root's environment into\n# the checkout. Always su: root may change to DEPLOY_USER without a doas rule.\nrun_as_deploy_user() {\n # Root may su to DEPLOY_USER without a doas rule. Avoid doas -u: the box\n # only permits dev→root and root→root by default.\n local snippet=$1\n su -s /bin/sh \"$DEPLOY_USER\" -c \"$snippet\"\n}\n\ngit_as_deploy_user() {\n # All git mutations against the production checkout run as DEPLOY_USER.\n local args_q\n args_q=$(printf '%q ' \"$@\")\n run_as_deploy_user \"cd $(printf '%q' \"$repo\") && git $args_q\"\n}\n\napp=${1:-}\n[[ -n $app ]] || { usage >&2; exit 2 }\nrecover_mode=\nrecover_id=\nif [[ $app == recover ]]; then\n case ${2:-} in\n --plan)\n [[ $# -eq 2 ]] || { usage >&2; exit 2 }\n recover_mode=plan\n ;;\n --apply)\n [[ $# -eq 3 ]] && \"$RUBY\" -e 'exit(ARGV.fetch(0).match?(/\\A[0-9]{8}T[0-9]{6}Z-[a-z0-9_-]{1,32}-[0-9]+\\z/) ? 0 : 1)' \"$3\" || {\n print -u2 \"vps-deploy: recover --apply needs one exact run ID from --plan\"\n exit 2\n }\n recover_mode=apply\n recover_id=$3\n ;;\n *) usage >&2; exit 2 ;;\n esac\n unset VPS_DEPLOY_RUN_ID VPS_DEPLOY_RUN_TARGET VPS_DEPLOY_RUN_STARTED_AT\n unset VPS_DEPLOY_STARTED VPS_DEPLOY_LOCK_HELD\nelse\n [[ $# -eq 1 ]] || { usage >&2; exit 2 }\n # Restrict the log/path component to a safe identifier before inventory lookup.\n [[ $app != *[^A-Za-z0-9_-]* ]] || {\n print -u2 \"vps-deploy: invalid target identifier\"\n exit 2\n }\nfi\n# Until the pinned checkout is ready, the current inventory is only a\n# best-effort restore-order hint for services shed during preflight. These arrays\n# are refreshed from VPS_DEPLOY_SHA before target validation or deployment.\ndeploy_target_kind=unknown\ntypeset -a DEPLOY_ALL DEPLOY_ALL_DEGRADED DEPLOY_RAILS\nDEPLOY_ALL=(${(@f)\"$(inventory_names all 2>/dev/null || true)\"})\nDEPLOY_ALL_DEGRADED=(${(@f)\"$(inventory_names all 1 2>/dev/null || true)\"})\nDEPLOY_RAILS=(${(@f)\"$(inventory_names rails 2>/dev/null || true)\"})\n\n# Environment values below participate in shell arithmetic and bounded waits.\n# Validate their decimal spelling and limits before acquiring a lock or touching\n# a service. In particular, a zero poll interval otherwise creates an endless\n# wait_up loop during a failed-deploy restore.\nvalidate_deploy_setting() {\n local name=$1 value=$2 minimum=$3 maximum=$4\n [[ $value == <-> && ( $value == 0 || $value != 0* ) ]] || {\n print -u2 \"vps-deploy: invalid $name; expected a decimal integer\"\n return 2\n }\n (( ${#value} <= 10 )) || {\n print -u2 \"vps-deploy: invalid $name; value is out of range\"\n return 2\n }\n (( value >= minimum && value <= maximum )) || {\n print -u2 \"vps-deploy: invalid $name; allowed range is $minimum..$maximum\"\n return 2\n }\n}\n\nvalidate_deploy_setting VPS_DEPLOY_LOCK_TTL \"${VPS_DEPLOY_LOCK_TTL:-7200}\" 1 604800 || exit 2\nvalidate_deploy_setting VPS_DEPLOY_RESTORE_WAIT \"${VPS_DEPLOY_RESTORE_WAIT:-240}\" 1 3600 || exit 2\nvalidate_deploy_setting VPS_DEPLOY_POLL \"${VPS_DEPLOY_POLL:-3}\" 1 60 || exit 2\nvalidate_deploy_setting VPS_DEPLOY_SETTLE_WAIT \"${VPS_DEPLOY_SETTLE_WAIT:-240}\" 1 3600 || exit 2\nvalidate_deploy_setting VPS_DEPLOY_MASTER_MIN_AVAIL \"${VPS_DEPLOY_MASTER_MIN_AVAIL:-10}\" 0 100 || exit 2\nvalidate_deploy_setting VPS_DEPLOY_MIN_FREE_KB \"${VPS_DEPLOY_MIN_FREE_KB:-1048576}\" 0 1000000000 || exit 2\n: \"${PUB5_ADMIN:=/usr/local/sbin/pub5-admin}\"\n[[ -f \"$PUB5_ADMIN\" && -x \"$PUB5_ADMIN\" && ! -L \"$PUB5_ADMIN\" ]] || {\n print -u2 \"vps-deploy: pub5-admin missing; run reviewed OPENBSD/OPERATOR.sh first\"\n exit 1\n}\n\n# One host, one deploy. The parent all/rails process keeps this lock while its\n# re-entrant child processes deploy each app in order; a second top-level deploy\n# is refused rather than running two pulls, precompiles and restarts together.\nlock_dir=${VPS_DEPLOY_LOCK_DIR:-$DEPLOY_HOME/.pub5-vps-deploy.lock}\nlock_ttl=${VPS_DEPLOY_LOCK_TTL:-7200}\nlock_owner=0\n\n# The lock is a directory, because mkdir is the one atomic test-and-set the\n# base system gives a shell, and the directory holds an `owner` file: pid, host\n# and epoch seconds. A bare directory could not tell a running deploy from the\n# remains of one killed by a dropped session or a reboot, and an operator had to\n# guess before removing it. Now a lock is reaped here, with the reason printed,\n# when its same-host owner PID is gone. Age alone never steals a lock from a\n# live local deployment: a slow or stalled deploy requires operator inspection,\n# not a second deploy running beside it. A remote-host or unreadable lock may\n# be reaped only after the recorded TTL, because this checkout cannot inspect\n# the other host's process.\n# BSD date formats an epoch with -r; GNU date reads -r as a file and needs\n# -d @epoch. The reaper prints the owner's start time on both dev hosts and vm23.\nepoch_date() {\n date -r \"$1\" '+%Y-%m-%d %H:%M:%S' 2>/dev/null || date -d \"@$1\" '+%Y-%m-%d %H:%M:%S' 2>/dev/null\n}\n\n# BSD stat formats with -f; GNU stat reads -f as filesystem status and needs\n# -c. The reaper pins the observed directory by device and inode on both hosts.\nfile_identity() {\n local out\n out=$(stat -f '%d:%i' \"$1\" 2>/dev/null)\n if [[ $out == <->:* ]]; then\n print -r -- $out\n return 0\n fi\n stat -c '%d:%i' \"$1\" 2>/dev/null\n}\n\n# A PID can be reused after its original deploy exits. Pair it with the\n# process start time and a checksum of its command line, so an unrelated process\n# cannot keep an abandoned lock alive forever. OpenBSD ps(1) exposes lstart and\n# command; if either cannot be read, callers fail safe and keep the lock.\nlock_process_identity() {\n local pid=$1 started command digest\n started=$(LC_ALL=C ps -p \"$pid\" -o lstart= 2>/dev/null) || return 1\n command=$(ps -ww -p \"$pid\" -o command= 2>/dev/null) || return 1\n [[ -n $started && -n $command ]] || return 1\n started=${started//[^A-Za-z0-9]/_}\n digest=$(print -rn -- \"$command\" | cksum) || return 1\n digest=${digest// /_}\n print -r -- \"${started}_${digest}\"\n}\n\nlock_read_owner() {\n lock_pid= lock_host= lock_epoch= lock_identity=\n [[ -r $lock_dir/owner ]] || return 1\n read -r lock_pid lock_host lock_epoch lock_identity < \"$lock_dir/owner\" || true\n [[ $lock_pid == <-> && $lock_epoch == <-> && $lock_identity != *[^A-Za-z0-9_]* ]]\n}\n\nlock_stale_reason() {\n local now=$(date +%s) current_identity\n if lock_read_owner; then\n if [[ $lock_host == $(hostname -s) ]]; then\n if ! ps -p \"$lock_pid\" >/dev/null 2>&1; then\n print -r -- \"owner pid $lock_pid is not running on $lock_host\"\n return 0\n fi\n if [[ -n $lock_identity ]]; then\n current_identity=$(lock_process_identity \"$lock_pid\") || {\n # A visible PID whose full identity cannot be measured is ambiguous,\n # not stale. Never steal its lock based on a partial observation.\n return 1\n }\n if [[ $current_identity != $lock_identity ]]; then\n print -r -- \"owner pid $lock_pid process identity changed on $lock_host (PID reused)\"\n return 0\n fi\n fi\n # Legacy three-field owners have no fingerprint: keep them protected while\n # their PID exists. New owners are distinguished even after PID reuse.\n # A live same-host owner is authoritative even after the TTL.\n return 1\n fi\n if (( now - lock_epoch > lock_ttl )); then\n print -r -- \"held $(( (now - lock_epoch) / 60 )) minutes by pid $lock_pid on $lock_host, past the ${lock_ttl}s TTL\"\n return 0\n fi\n return 1\n fi\n # mkdir has run but the owner file is not written yet, or never was. Judge\n # such a lock by the directory's own age so a deploy mid-start is left alone.\n local -a aged\n aged=( ${lock_dir}(N/mm+$(( lock_ttl / 60 ))) )\n (( $#aged )) || return 1\n print -r -- \"no readable owner and older than the ${lock_ttl}s TTL\"\n}\n\nrelease_deploy_lock() {\n (( lock_owner )) || return 0\n rm -f -- \"$lock_dir/owner\"\n rmdir \"$lock_dir\" 2>/dev/null || true\n}\n\nreap_stale_lock() {\n local reason before_identity after_identity retired_lock\n\n # Do not enter recovery for a live lock. The second check below closes the\n # race between this observation and claiming the recovery marker.\n reason=$(lock_stale_reason) || return 1\n before_identity=$(file_identity \"$lock_dir\") || return 1\n\n # The marker is inside the exact directory observed above, so mkdir is an\n # atomic claim. Inode identity catches a stale directory being replaced by a\n # fresh lock between the first observation and this claim.\n if ! mkdir \"$lock_dir/reaping\" 2>/dev/null; then\n return 1\n fi\n after_identity=$(file_identity \"$lock_dir\") || {\n rmdir \"$lock_dir/reaping\" 2>/dev/null || true\n return 1\n }\n if [[ $after_identity != $before_identity ]]; then\n rmdir \"$lock_dir/reaping\" 2>/dev/null || true\n return 1\n fi\n\n # Creating the marker updates the parent directory's mtime. Recheck owners,\n # but retain the pre-marker age decision for a directory that was already\n # ownerless; no other process can write its owner file while it exists.\n if [[ -r \"$lock_dir/owner\" ]]; then\n reason=$(lock_stale_reason) || {\n rmdir \"$lock_dir/reaping\" 2>/dev/null || true\n return 1\n }\n fi\n\n retired_lock=\"${lock_dir}.stale.$$\"\n if [[ -e \"$retired_lock\" ]]; then\n print -u2 \"vps-deploy: stale-lock quarantine exists at $retired_lock\"\n rmdir \"$lock_dir/reaping\" 2>/dev/null || true\n return 1\n fi\n\n print -u2 \"vps-deploy: reaping stale lock $lock_dir - $reason\"\n # Rename the whole claimed directory, marker included. The old lock path\n # becomes free atomically; cleanup touches only this unique retired path.\n if ! mv \"$lock_dir\" \"$retired_lock\" 2>/dev/null; then\n rmdir \"$lock_dir/reaping\" 2>/dev/null || true\n return 1\n fi\n rm -f -- \"$retired_lock/owner\"\n # A process killed before the atomic owner rename can leave only this temp.\n # Remove it with the quarantined lock so crash recovery does not leak dirs.\n rm -f -- \"$retired_lock\"/owner.tmp.*(N)\n rmdir \"$retired_lock/reaping\" 2>/dev/null || true\n rmdir \"$retired_lock\" 2>/dev/null || true\n return 0\n}\n\nacquire_deploy_lock() {\n local -i attempt\n for attempt in 1 2; do\n if mkdir \"$lock_dir\" 2>/dev/null; then\n local identity owner_tmp\n identity=$(lock_process_identity \"${$}\") || {\n print -u2 \"vps-deploy: cannot identify the lock owner process; refusing an unverifiable deploy\"\n rmdir \"$lock_dir\" 2>/dev/null || true\n exit 1\n }\n owner_tmp=\"$lock_dir/owner.tmp.${$}\"\n if ! print -r -- \"${$} $(hostname -s) $(date +%s) $identity\" > \"$owner_tmp\" ||\n ! mv \"$owner_tmp\" \"$lock_dir/owner\"; then\n print -u2 \"vps-deploy: cannot write the deploy lock owner record\"\n rm -f -- \"$owner_tmp\"\n rmdir \"$lock_dir\" 2>/dev/null || true\n exit 1\n fi\n export VPS_DEPLOY_LOCK_HELD=1\n export VPS_DEPLOY_LOCK_PID=${$}\n export VPS_DEPLOY_LOCK_IDENTITY=$identity\n lock_owner=1\n return 0\n fi\n if reap_stale_lock; then\n continue\n fi\n break\n done\n if lock_read_owner; then\n print -u2 \"vps-deploy: refused - another deploy owns $lock_dir (pid $lock_pid on $lock_host since $(epoch_date $lock_epoch))\"\n else\n print -u2 \"vps-deploy: refused - another deploy owns $lock_dir\"\n fi\n exit 75\n}\n\n# Re-entrant children must prove the inherited lock belongs to their live\n# parent. An environment flag alone must never bypass top-level serialization.\nif [[ -n ${VPS_DEPLOY_LOCK_HELD:-} ]]; then\n lock_read_owner || {\n print -u2 \"vps-deploy: refused - child invocation has no readable parent lock owner\"\n exit 75\n }\n [[ $lock_host == \"$(hostname -s)\" &&\n $lock_pid == ${VPS_DEPLOY_LOCK_PID:-} &&\n $lock_identity == ${VPS_DEPLOY_LOCK_IDENTITY:-} ]] || {\n print -u2 \"vps-deploy: refused - child invocation does not match the parent lock record\"\n exit 75\n }\n ps -p \"$lock_pid\" >/dev/null 2>&1 || {\n print -u2 \"vps-deploy: refused - parent deploy process is no longer running\"\n exit 75\n }\n current_identity=$(lock_process_identity \"$lock_pid\") || {\n print -u2 \"vps-deploy: refused - cannot verify the parent deploy process\"\n exit 75\n }\n [[ $current_identity == \"$lock_identity\" ]] || {\n print -u2 \"vps-deploy: refused - parent deploy PID identity changed\"\n exit 75\n }\nelse\n acquire_deploy_lock\nfi\n\n# Durable deploy logging.\n#\n# A full deploy outlives the ssh session that starts it. When that session is\n# cut the only surviving record is the log file. We keep:\n# - a rolling aggregate log (VPS_DEPLOY_LOG / /var/log/pub5/vps-deploy.log)\n# - a per-run log under /var/log/pub5/runs/ with the pid and app in the name\n# - a stable symlink at $DEPLOY_HOME/vps-deploy.log for operators who still\n# tail the home path\n# Once per run, not once per app: each group re-invokes one child per app.\n# VPS_DEPLOY_LOGGING is inherited so child processes do not double-tee.\ndeploy_log_dir=${VPS_DEPLOY_LOG_DIR:-/var/log/pub5}\ndeploy_log=${VPS_DEPLOY_LOG:-$deploy_log_dir/vps-deploy.log}\ndeploy_run_log=${VPS_DEPLOY_RUN_LOG:-}\n\nlog_ts() { date -u '+%Y-%m-%dT%H:%M:%SZ'; }\n\n# Structured line: timestamp host pid app stage message. Always goes to stdout\n# (which is already teed into the log files).\nlog() {\n print -r -- \"$(log_ts) host=$(hostname -s) pid=$$ app=${app:-?} stage=${deploy_stage:-?} $*\"\n}\n\n# The fleet receipt is written only through the fixed root broker. Ruby's JSON\n# encoder keeps service names, paths and metadata out of shell-generated JSON.\nwrite_deploy_run_receipt() {\n [[ -n ${VPS_DEPLOY_RUN_ID:-} ]] || return 0\n local payload\n payload=$(\"$RUBY\" -rjson -e '\n run_id, target, host, pid, owner_identity, status, recovery_state, stage,\n started_at, pinned_sha, previous_sha, services, ports_csv, order_csv,\n run_log, aggregate_log, snapshot_recorded = ARGV\n ports = {}\n ports_csv.split(\",\").reject(&:empty?).each do |pair|\n name, value = pair.split(\"=\", 2)\n abort \"invalid port pair\" unless name && value && value.match?(/\\A[0-9]+\\z/)\n ports[name] = Integer(value, 10)\n end\n record = {\n schema: 1, run_id: run_id, target: target, host: host, pid: Integer(pid, 10),\n owner_identity: owner_identity, status: status, recovery_state: recovery_state,\n stage: stage, started_at: started_at,\n pinned_sha: pinned_sha.empty? ? nil : pinned_sha,\n previous_sha: previous_sha.empty? ? nil : previous_sha,\n started_services: services.empty? ? [] : services.split(\" \"),\n ports: ports,\n restore_order: order_csv.empty? ? [] : order_csv.split(\",\"),\n run_log: run_log, aggregate_log: aggregate_log,\n snapshot_recorded: snapshot_recorded == \"1\"\n }\n puts JSON.generate(record)\n ' \"${VPS_DEPLOY_RUN_ID}\" \"${VPS_DEPLOY_RUN_TARGET}\" \"$(hostname -s)\" \\\n \"${VPS_DEPLOY_LOCK_PID}\" \"${VPS_DEPLOY_LOCK_IDENTITY}\" \\\n \"${VPS_DEPLOY_RECEIPT_STATUS:-running}\" \"${VPS_DEPLOY_RECEIPT_RECOVERY:-none}\" \\\n \"${deploy_stage:-start}\" \"${VPS_DEPLOY_RUN_STARTED_AT}\" \\\n \"${VPS_DEPLOY_RECEIPT_PINNED_SHA:-}\" \"${VPS_DEPLOY_RECEIPT_PREVIOUS_SHA:-}\" \\\n \"${VPS_DEPLOY_RECEIPT_SERVICES:-}\" \"${VPS_DEPLOY_RECEIPT_PORTS:-}\" \\\n \"${VPS_DEPLOY_RECEIPT_ORDER:-}\" \\\n \"${VPS_DEPLOY_RUN_LOG}\" \"${VPS_DEPLOY_LOG}\" \\\n \"${VPS_DEPLOY_RECEIPT_SNAPSHOT_RECORDED:-0}\") || return 1\n \"$PUB5_ADMIN\" deploy-run-write \"$VPS_DEPLOY_RUN_ID\" \"$payload\" >/dev/null\n}\n\n# Stage transitions persist the latest recovery breadcrumb before the next\n# stage may stop, replace, migrate or restart a production service.\nset_stage() {\n deploy_stage=$1\n log \"enter\"\n write_deploy_run_receipt || {\n print -u2 \"vps-deploy: failed to persist recovery receipt at stage $deploy_stage\"\n return 1\n }\n}\n# Retry a command with exponential backoff. Use only for transient failures\n# (network, service boot races). Do not wrap git merge conflicts or policy\n# refusals. Logs each attempt. Returns the last exit status.\n#\n# retry 3 2 curl -fsS http://127.0.0.1:port/up\n# attempts=3 initial_delay=2s, doubles each time, capped at 30s\nretry() {\n local max=${1:?} delay=${2:?}\n shift 2\n local attempt=1 rc=0\n while (( attempt <= max )); do\n if \"$@\"; then\n (( attempt > 1 )) && log \"retry ok attempt=$attempt cmd=$*\"\n return 0\n fi\n rc=$?\n if (( attempt == max )); then\n log \"retry exhausted attempts=$max rc=$rc cmd=$*\"\n return $rc\n fi\n log \"retry fail attempt=$attempt/$max rc=$rc sleep=${delay}s cmd=$*\"\n sleep $delay\n delay=$(( delay * 2 ))\n (( delay > 30 )) && delay=30\n attempt=$(( attempt + 1 ))\n done\n return $rc\n}\n\n# admin_service with retry for start/restart only (not check/list).\nadmin_service_retry() {\n local action=$1\n shift || true\n case $action in\n start|restart)\n retry \"${VPS_DEPLOY_SERVICE_RETRIES:-3}\" \"${VPS_DEPLOY_SERVICE_RETRY_DELAY:-2}\" \\\n admin_service \"$action\" \"$@\"\n ;;\n *)\n admin_service \"$action\" \"$@\"\n ;;\n esac\n}\n\nrotate_deploy_log() {\n # Safety net only. Canonical rotation is newsyslog (OPENBSD/mirror/etc/newsyslog.conf).\n # This fires if the log grew past the size limit between newsyslog runs.\n local max_bytes=${VPS_DEPLOY_LOG_MAX_BYTES:-10485760}\n [[ -f $deploy_log ]] || return 0\n local size\n size=$(stat -f %z \"$deploy_log\" 2>/dev/null) || return 0\n (( size >= max_bytes )) || return 0\n log \"rotate aggregate log size=$size max=$max_bytes\"\n mv -f -- \"$deploy_log\" \"$deploy_log.1\" 2>/dev/null || true\n}\n\nif [[ -z ${VPS_DEPLOY_LOGGING:-} ]]; then\n export VPS_DEPLOY_LOGGING=1\n mkdir -p -- \"$deploy_log_dir/runs\" 2>/dev/null || {\n # Fall back to the deploy user's home when /var/log/pub5 is not writable\n # yet (broker/OPERATOR not installed). Still durable across SSH drops.\n deploy_log_dir=$DEPLOY_HOME\n deploy_log=${VPS_DEPLOY_LOG:-$DEPLOY_HOME/vps-deploy.log}\n mkdir -p -- \"$deploy_log_dir/runs\" 2>/dev/null || true\n }\n rotate_deploy_log\n if [[ -z $deploy_run_log ]]; then\n deploy_run_log=\"$deploy_log_dir/runs/deploy-$(date -u '+%Y%m%d-%H%M%S')-${app}-$$.log\"\n fi\n export VPS_DEPLOY_LOG=$deploy_log\n export VPS_DEPLOY_RUN_LOG=$deploy_run_log\n export VPS_DEPLOY_LOG_DIR=$deploy_log_dir\n # Stable path operators already know about.\n if [[ -d $DEPLOY_HOME ]]; then\n ln -sfn \"$deploy_log\" \"$DEPLOY_HOME/vps-deploy.log\" 2>/dev/null || true\n ln -sfn \"$deploy_run_log\" \"$DEPLOY_HOME/vps-deploy-latest.log\" 2>/dev/null || true\n fi\n # Recovery depends on the per-run log retaining the pinned SHA, initial\n # service snapshot and last entered stage. Do not begin a deploy when that\n # durable record cannot be opened as a regular file.\n if ! : >> \"$deploy_run_log\" || [[ ! -f \"$deploy_run_log\" || -L \"$deploy_run_log\" ]]; then\n print -u2 \"vps-deploy: cannot create a regular per-run recovery log at $deploy_run_log; refusing before any service is touched\"\n exit 1\n fi\n if ! : >> \"$deploy_log\" || [[ ! -f \"$deploy_log\" || -L \"$deploy_log\" ]]; then\n print -u2 \"vps-deploy: cannot create a regular aggregate log at $deploy_log; refusing before any service is touched\"\n exit 1\n fi\n # Tee stdout/stderr into both the aggregate and the per-run log. process\n # substitution keeps a single stream so line order stays honest.\n exec > >(tee -a \"$deploy_log\" \"$deploy_run_log\") 2>&1\n set_stage start\n log \"start target=$app uid=$(id -u) user=$(id -un) log=$deploy_log run_log=$deploy_run_log\"\n log \"argv: $*\"\nfi\n\n# Transitional adapter: use the fixed root-owned broker once installed. The\n# fallback remains until the operator sync has installed it on vm23.\nadmin_service() {\n local action=${1:-}\n shift || true\n if [[ $action == list-started ]]; then\n \"$PUB5_ADMIN\" service list-started\n else\n \"$PUB5_ADMIN\" service \"$action\" \"$@\"\n fi\n}\n\nadmin_request() {\n [[ -x \"$PUB5_ADMIN\" ]] || {\n print -u2 \"vps-deploy: pub5-admin is not installed; run the reviewed OPERATOR sync first\"\n return 127\n }\n \"$PUB5_ADMIN\" \"$@\"\n}\n\n# The guard's deploy flag, held for the whole deploy and not only the restart.\n#\n# rc.d/ touches .deploying- from rc_pre to the end of its /up wait and\n# removes it there. CI, migrate, precompile and the checks all run outside that\n# window at the box's highest load, and resource_guard counts every one of those\n# ticks as a strike. So the counter already stands past GUARD_SHED_STRIKES when\n# the restart's flag comes off, and the next tick sheds the first OPTIONAL\n# service. bsdports leads that list, which is why its own deploy exits 0 and\n# vps-state then reads bsdports(failed) with :47312 closed.\n#\n# Taken after the preflight guard run, which may still shed to free memory for\n# CI. A tick that sees the flag resets the counter, so the guard needs two fresh\n# breaching ticks after this deploy ends before it sheds anything. The name is\n# not .deploying-, because rc.d removes that one mid-deploy; it is touched\n# again at each long stage because the guard ignores a flag older than 30 minutes.\ndeploy_flag=\"$repo/.deploying-vps-deploy-${app}\"\nhold_deploy_flag() { touch \"$deploy_flag\" 2>/dev/null || true }\n\n# The stage touches alone did not hold it: brgen's CI ran 34 minutes between two of\n# them on 2026-10-27, the flag aged out at 30, and the guard's emergency stop fired\n# mid-deploy. A keeper touches it every 4 minutes for as long as this process lives;\n# finish() stops it before removing the flag, and it exits on its own if we die.\nflag_keeper=\nkeep_deploy_flag() {\n hold_deploy_flag\n ( while kill -0 $$ 2>/dev/null; do touch \"$deploy_flag\" 2>/dev/null; sleep 240; done ) &!\n flag_keeper=$!\n}\n\n# The terminal result, recorded however the script ends.\n#\n# A failure past `rcctl stop` used to leave the app stopped under a stamp that\n# still said ok, and an ssh disconnect cut the tee pipe before the last line\n# reached the log. Every exit now lands here: it stamps failed (unless a stage\n# already did), names the stage and the services left down, prints the safe next\n# command, and appends the verdict straight to the log file rather than through\n# tee. Each step is `|| true`: a trap that fails would hide the failure it reports.\ndeploy_stage=start\napp_stopped=0\nstamp_written=0\nhalted_app=\npreflight_active=0\npreflight_back_to=\ntypeset -a preflight_back_flag own_stops\n\n# Services the deploy needs to put back, and how it waits for them.\n#\n# On 2026-10-12 a halted `all` pass left brgen, amber and bsdports down for minutes\n# until the operator typed rcctl start by hand: the restore sat at the end of a pass\n# that never got there. The snapshot of started services is taken before anything is\n# touched (VPS_DEPLOY_STARTED, exported so the children see it), and the exit trap\n# of the top-level process puts back whatever is missing, one service at a time.\n# Falcon needs 30-40 s to boot on this box, and two booting at once is what shed\n# the apps in the first place, so each service answers /up before the next starts.\ntypeset -A deploy_ports\nload_deploy_ports() {\n (( $#deploy_ports )) && return 0\n local line\n local -a rows\n rows=(${(f)\"$(\"$RUBY\" -r\"$repo/OPENBSD/lib/deploy_inventory\" \\\n -e 'Deploy::Inventory.new(root: ARGV[0]).apps.each { |a| puts \"\#{a.name} \#{a.port}\" }' \"$repo\" 2>/dev/null)\"})\n for line in $rows; do\n deploy_ports[${line%% *}]=${line##* }\n done\n return 0\n}\n\n# Bounded wait for http://127.0.0.1:PORT/up. Returns 1 when the limit passes.\nwait_up() {\n local port=$1 limit=${2:-${VPS_DEPLOY_RESTORE_WAIT:-240}} step=${VPS_DEPLOY_POLL:-3} waited=0\n while (( waited < limit )); do\n if curl -fsS --max-time 5 \"http://127.0.0.1:${port}/up\" >/dev/null 2>&1; then\n (( waited > 0 )) && log \"wait_up ok port=$port after=${waited}s\"\n return 0\n fi\n sleep $step\n waited=$(( waited + step ))\n done\n log \"wait_up timeout port=$port limit=${limit}s\"\n return 1\n}\n\n# Every service that was started and is running must answer /up before the next\n# memory-heavy step (CI, bundle, precompile). A service that is down is not booting\n# and is not waited for; one that never answers is reported, not waited on forever.\nsettle_boots() {\n local svc port\n load_deploy_ports\n for svc in ${=VPS_DEPLOY_STARTED:-}; do\n port=${deploy_ports[$svc]:-}\n [[ -n $port ]] || continue\n admin_service check \"$svc\" >/dev/null 2>&1 || continue\n wait_up \"$port\" \"${VPS_DEPLOY_SETTLE_WAIT:-240}\" \\\n || print -u2 \"vps-deploy: WARNING - $svc is running but :${port}/up did not answer within ${VPS_DEPLOY_SETTLE_WAIT:-240}s\"\n done\n return 0\n}\n\n# Who stopped it: this deploy, resource_guard (its shed ledger), or neither.\nstop_origin() {\n local svc=$1 shed_file=${VPS_DEPLOY_SHED_FILE:-/var/db/resource_guard_shed}\n if [[ $svc == $halted_app ]] || (( ${own_stops[(Ie)$svc]} )); then\n print \"stopped by this deploy\"\n elif [[ -r $shed_file ]] && (( ${${(f)\"$(<$shed_file)\"}[(Ie)$svc]} )); then\n print \"shed by resource_guard\"\n else\n print \"down for another reason (crashed or stopped outside the deploy)\"\n fi\n}\n\n# Starts, one at a time and in serving order, every service that was started when\n# the deploy began and is not running now. Returns 1 if any would not come back.\nrestore_started() {\n [[ -n ${VPS_DEPLOY_STARTED:-} ]] || return 0\n if [[ ${VPS_DEPLOY_RESTORE_PORTS_FROM_RECEIPT:-0} != 1 ]]; then\n load_deploy_ports\n fi\n local -a want order failed restored\n local svc port origin\n want=( ${=VPS_DEPLOY_STARTED} )\n for svc in $DEPLOY_ALL_DEGRADED; do (( ${want[(Ie)$svc]} )) && order+=($svc); done\n for svc in $want; do (( ${order[(Ie)$svc]} )) || order+=($svc); done\n for svc in $order; do\n admin_service check \"$svc\" >/dev/null 2>&1 && continue\n origin=$(stop_origin \"$svc\")\n print \"vps-deploy: restore $svc ($origin)\"\n port=${deploy_ports[$svc]:-}\n # restart, not start: rc.d/master runs its relayd restart, TTS warmup and flag\n # removal only on restart, and restart on a stopped service is a start.\n admin_service_retry restart \"$svc\" >/dev/null 2>&1 || true\n if [[ -n $port ]]; then\n wait_up \"$port\" || { admin_service_retry restart \"$svc\" >/dev/null 2>&1 || true; wait_up \"$port\"; } || {\n print -u2 \"vps-deploy: restore $svc FAILED - :${port}/up not ready, read /var/log/daemon\"\n failed+=($svc); continue\n }\n else\n sleep 3\n admin_service check \"$svc\" >/dev/null 2>&1 || { print -u2 \"vps-deploy: restore $svc FAILED - rcctl check\"; failed+=($svc); continue }\n fi\n print \"vps-deploy: restore $svc ok\"\n restored+=($svc)\n done\n (( $#restored )) && print \"vps-deploy: restored ${restored}\"\n (( $#failed )) && print -u2 \"vps-deploy: STILL DOWN after restore: ${failed} (/usr/local/sbin/pub5-admin service restart , one at a time)\"\n (( $#failed == 0 ))\n}\n\n# A signal or shell exit can arrive while the box is detached for preflight.\n# Keep the original ref as explicit recovery state until it is restored.\nrestore_preflight_checkout() {\n (( preflight_active )) || return 0\n [[ -n $preflight_back_to ]] || {\n print -u2 \"vps-deploy: preflight checkout recovery has no saved ref\"\n return 1\n }\n git_as_deploy_user switch -q \"${preflight_back_flag[@]}\" \"$preflight_back_to\" || {\n print -u2 \"vps-deploy: FAILED to restore checkout ref $preflight_back_to after preflight\"\n return 1\n }\n preflight_active=0\n log \"preflight checkout restored ref=$preflight_back_to\"\n}\n\n# Vendorables are not tracked in git (TREE_CONTRACTS.md, 8 November 2026):\n# master's smart-turn runtime bytes are fetched when absent, with pinned\n# versions and sha256-verified digests. Failure leaves ?smart_turn=1\n# unavailable — the feature is default-off — and the deploy continues.\nensure_smart_turn_assets() {\n local vendor=\"$repo/MASTER/web/public/vendor/onnxruntime\"\n local model=\"$repo/MASTER/web/public/models/smart-turn-v3.2-cpu.onnx\"\n [[ -f $vendor/ort-wasm-simd-threaded.wasm && -f $vendor/ort.wasm.min.mjs && -f $model ]] && return 0\n print \"vps-deploy: smart_turn assets absent; fetching pinned, checksummed copies\"\n if ! run_as_deploy_user \"cd $(printf '%q' \"$repo/MASTER/web\") && sh script/build_smart_turn.sh\"; then\n print -u2 \"vps-deploy: smart_turn fetch failed; ?smart_turn=1 stays unavailable (default-off) until MASTER/web/script/build_smart_turn.sh succeeds\"\n fi\n}\n\nfinish() {\n local rc=$?\n trap - EXIT\n if (( preflight_active )); then\n restore_preflight_checkout || {\n print -u2 \"vps-deploy: checkout remains in preflight recovery state; restore ref $preflight_back_to before another deploy\"\n (( rc == 0 )) && rc=1\n }\n fi\n if (( rc != 0 )) && [[ $deploy_target_kind == all || $deploy_target_kind == group ]]; then\n print -u2 \"vps-deploy: $app FAILED at stage \\\"$deploy_stage\\\" (exit $rc)\"\n fi\n if (( rc != 0 )) && [[ $deploy_target_kind != all && $deploy_target_kind != group && $deploy_target_kind != unknown ]]; then\n if (( ! stamp_written )) && (( $+functions[write_stamp] )); then\n write_stamp \"$app\" failed || true\n fi\n print -u2 \"vps-deploy: $app FAILED at stage \\\"$deploy_stage\\\" (exit $rc)\"\n if (( app_stopped )); then\n print -u2 \"vps-deploy: $app was stopped for this deploy and is DOWN - read /var/log/daemon, then\"\n print -u2 \" /usr/local/sbin/pub5-admin service start $app (or rerun: doas zsh OPENBSD/bin/vps-deploy $app)\"\n else\n print -u2 \"vps-deploy: $app was not stopped; the previous release is still serving. Fix the cause and rerun:\"\n print -u2 \" doas zsh OPENBSD/bin/vps-deploy $app\"\n fi\n local other\n for other in $DEPLOY_RAILS; do\n [[ $other == \"$app\" ]] && continue\n admin_service get \"$other\" >/dev/null 2>&1 || continue\n admin_service check \"$other\" >/dev/null 2>&1 \\\n || print -u2 \"vps-deploy: $other is DOWN (shed or stopped) - /usr/local/sbin/pub5-admin service restart $other\"\n done\n fi\n # Only the process that owns the lock restores: the children of `all` fail into\n # their parent, which knows the whole pass. Each restore result is printed.\n local restore_ok=1\n if (( rc != 0 )) && (( lock_owner )) && [[ ${VPS_DEPLOY_RESTORE_ATTEMPTED:-0} != 1 ]]; then\n restore_started || {\n restore_ok=0\n print -u2 \"vps-deploy: production is NOT fully restored - see the lines above\"\n }\n fi\n # A normal EXIT/TERM/HUP reaches this terminal update; SIGKILL/power loss\n # leaves the last durable receipt pending for an explicit operator recovery.\n if (( lock_owner )) && [[ -n ${VPS_DEPLOY_RUN_ID:-} ]]; then\n VPS_DEPLOY_RECEIPT_STATUS=failed\n (( rc == 0 )) && VPS_DEPLOY_RECEIPT_STATUS=ok\n if [[ -n ${VPS_DEPLOY_STARTED:-} ]]; then\n if (( rc == 0 || restore_ok )); then\n VPS_DEPLOY_RECEIPT_RECOVERY=complete\n else\n VPS_DEPLOY_RECEIPT_RECOVERY=pending\n fi\n else\n VPS_DEPLOY_RECEIPT_RECOVERY=none\n fi\n export VPS_DEPLOY_RECEIPT_STATUS VPS_DEPLOY_RECEIPT_RECOVERY\n write_deploy_run_receipt || {\n print -u2 \"vps-deploy: FAILED to persist terminal recovery receipt\"\n (( rc == 0 )) && rc=1\n }\n fi\n # Keep resource_guard excluded until every service restored by this failure\n # has answered, including slow boots; only then stop the heartbeat and remove\n # this process's deploy flag.\n [[ -n $flag_keeper ]] && kill $flag_keeper 2>/dev/null\n rm -f -- \"$deploy_flag\"\n log \"finished exit=$rc\"\n # Write the verdict straight to the files as well: if the tee pipe died with\n # the ssh session the last structured line can still land on disk.\n {\n print -r -- \"$(log_ts) host=$(hostname -s) pid=$$ app=$app stage=$deploy_stage finished exit=$rc\"\n } >> \"${deploy_log:-/var/log/pub5/vps-deploy.log}\" 2>/dev/null || true\n if [[ -n ${deploy_run_log:-${VPS_DEPLOY_RUN_LOG:-}} ]]; then\n print -r -- \"$(log_ts) host=$(hostname -s) pid=$$ app=$app stage=$deploy_stage finished exit=$rc\" \\\n >> \"${deploy_run_log:-$VPS_DEPLOY_RUN_LOG}\" 2>/dev/null || true\n fi\n release_deploy_lock\n exit $rc\n}\n# zsh runs the EXIT trap for HUP but not for TERM or INT, so name all three: a\n# dropped ssh session or a kill is a failure with a recorded stage, not a silent end.\ntrap finish EXIT\ntrap 'exit 129' HUP\ntrap 'exit 130' INT\ntrap 'exit 143' TERM\n\n# One durable receipt belongs to the lock-owning parent. Child app deploys\n# inherit the ID, snapshot and owner identity and update only the current stage.\nif (( lock_owner )) && [[ $app != recover ]]; then\n unset VPS_DEPLOY_RUN_ID VPS_DEPLOY_RUN_TARGET VPS_DEPLOY_RUN_STARTED_AT\n unset VPS_DEPLOY_STARTED\n unset VPS_DEPLOY_RECEIPT_STATUS VPS_DEPLOY_RECEIPT_RECOVERY\n unset VPS_DEPLOY_RECEIPT_PINNED_SHA VPS_DEPLOY_RECEIPT_PREVIOUS_SHA\n unset VPS_DEPLOY_RECEIPT_SERVICES VPS_DEPLOY_RECEIPT_PORTS VPS_DEPLOY_RECEIPT_ORDER\n unset VPS_DEPLOY_RECEIPT_SNAPSHOT_RECORDED\n VPS_DEPLOY_RUN_ID=\"$(date -u '+%Y%m%dT%H%M%SZ')-$app-${$}\"\n VPS_DEPLOY_RUN_TARGET=$app\n VPS_DEPLOY_RUN_STARTED_AT=\"$(date -u '+%Y-%m-%dT%H:%M:%SZ')\"\n VPS_DEPLOY_RECEIPT_STATUS=running\n VPS_DEPLOY_RECEIPT_RECOVERY=none\n VPS_DEPLOY_RECEIPT_PINNED_SHA=\n VPS_DEPLOY_RECEIPT_PREVIOUS_SHA=\n VPS_DEPLOY_RECEIPT_SERVICES=\n VPS_DEPLOY_RECEIPT_PORTS=\n VPS_DEPLOY_RECEIPT_ORDER=\n VPS_DEPLOY_RECEIPT_SNAPSHOT_RECORDED=0\n export VPS_DEPLOY_RECEIPT_SNAPSHOT_RECORDED\n export VPS_DEPLOY_RUN_ID VPS_DEPLOY_RUN_TARGET VPS_DEPLOY_RUN_STARTED_AT\n export VPS_DEPLOY_RECEIPT_STATUS VPS_DEPLOY_RECEIPT_RECOVERY\n export VPS_DEPLOY_RECEIPT_PINNED_SHA VPS_DEPLOY_RECEIPT_PREVIOUS_SHA\n export VPS_DEPLOY_RECEIPT_SERVICES VPS_DEPLOY_RECEIPT_PORTS VPS_DEPLOY_RECEIPT_ORDER\n write_deploy_run_receipt || {\n print -u2 \"vps-deploy: cannot create the durable recovery receipt; refusing before any service is touched\"\n exit 1\n }\nfi\n\n# A parent can be killed while a child app deploy is still unwinding. The\n# receipt identifies the parent, so also refuse while another deployer process\n# is visible. Exclude this recovery command by its own PID.\nother_deploy_processes() {\n local pid exclude_pid=${1:-} matches rc\n local -a candidates\n if matches=$(pgrep -f 'OPENBSD/bin/vps-deploy' 2>/dev/null); then\n :\n else\n rc=$?\n (( rc == 1 )) || return 1\n fi\n candidates=(${(f)matches})\n for pid in $candidates; do\n [[ $pid == \"${$}\" || ( -n $exclude_pid && $pid == \"$exclude_pid\" ) ]] || print -r -- \"$pid\"\n done\n}\n\n# Operator-driven recovery is intentionally separate from deploy. It never pulls,\n# checks out code, runs migrations or replays a deploy; it restores only the\n# service set recorded as running before the interrupted run.\nif [[ $app == recover ]]; then\n runs_json=$(admin_request deploy-run-list) || {\n print -u2 \"vps-deploy: cannot list recovery receipts; install and verify the pub5-admin mirror first\"\n exit 1\n }\n if [[ $recover_mode == plan ]]; then\n report=$(\"$RUBY\" -rjson -e '\n rows = JSON.parse(STDIN.read)\n rows.select { |r| r[\"recovery_state\"] == \"pending\" && %w[running failed].include?(r[\"status\"]) }.each do |r|\n puts [r[\"run_id\"], r[\"target\"], r[\"pinned_sha\"] || \"not-pinned\",\n r[\"stage\"], r[\"status\"], r[\"pid\"], r[\"owner_identity\"],\n r[\"host\"], r[\"snapshot_recorded\"], r[\"run_log\"]].join(\"|\")\n end\n ' <<< \"$runs_json\") || exit 1\n if [[ -z $report ]]; then\n print \"vps-deploy: no pending interrupted-deploy recovery receipts\"\n exit 0\n fi\n while IFS='|' read -r run_id target sha stage status owner_pid owner_identity owner_host snapshot_recorded run_log; do\n [[ -n $run_id ]] || continue\n if [[ $owner_host != \"$(hostname -s)\" ]]; then\n print \"vps-deploy: FOREIGN $run_id host=$owner_host target=$target - recover on the recorded host\"\n continue\n fi\n if [[ $snapshot_recorded != true ]]; then\n print \"vps-deploy: INCOMPLETE $run_id target=$target stage=$stage - service snapshot is not complete; do not apply\"\n continue\n fi\n other_pids=$(other_deploy_processes \"$owner_pid\") || {\n print \"vps-deploy: AMBIGUOUS $run_id - cannot verify the process table; do not recover\"\n continue\n }\n if [[ -n $other_pids ]]; then\n print \"vps-deploy: AMBIGUOUS $run_id - another deployer process is still present; do not recover\"\n print -r -- \" other deployer PID(s): $other_pids\"\n continue\n fi\n if ps -p \"$owner_pid\" >/dev/null 2>&1; then\n current_identity=$(lock_process_identity \"$owner_pid\") || {\n print \"vps-deploy: AMBIGUOUS $run_id - owner PID exists but its identity cannot be checked\"\n continue\n }\n if [[ $current_identity == \"$owner_identity\" ]]; then\n print \"vps-deploy: LIVE $run_id target=$target sha=$sha stage=$stage pid=$owner_pid - do not recover\"\n continue\n fi\n fi\n print \"vps-deploy: RECOVERY AVAILABLE $run_id target=$target sha=$sha stage=$stage\"\n print \" log=$run_log\"\n done <<< \"$report\"\n print \"vps-deploy: inspect the receipt and its log before: zsh OPENBSD/bin/vps-deploy recover --apply \"\n exit 0\n fi\n\n receipt_json=$(admin_request deploy-run-read \"$recover_id\") || exit 1\n [[ $receipt_json != null ]] || {\n print -u2 \"vps-deploy: no recovery receipt found for $recover_id\"\n exit 1\n }\n details=$(\"$RUBY\" -rjson -e '\n r = JSON.parse(STDIN.read)\n abort \"receipt is not pending recovery\" unless\n r[\"recovery_state\"] == \"pending\" && %w[running failed].include?(r[\"status\"])\n abort \"receipt snapshot is incomplete\" unless r[\"snapshot_recorded\"] == true\n puts [\"META\", r.fetch(\"pid\"), r.fetch(\"owner_identity\"), r.fetch(\"stage\"),\n r.fetch(\"target\"), r[\"pinned_sha\"] || \"not-pinned\", r.fetch(\"host\")].join(\"|\")\n puts [\"SERVICES\", r.fetch(\"started_services\").join(\" \")].join(\"|\")\n puts [\"ORDER\", r.fetch(\"restore_order\").join(\" \")].join(\"|\")\n r.fetch(\"ports\").sort.each { |service, port| puts [\"PORT\", service, port].join(\"|\") }\n ' <<< \"$receipt_json\") || {\n print -u2 \"vps-deploy: receipt is corrupt, incomplete or not eligible for recovery\"\n exit 1\n }\n\n typeset -a recovery_lines\n recovery_lines=(${(f)details})\n IFS='|' read -r kind owner_pid owner_identity original_stage original_target original_sha owner_host <<< \"$recovery_lines[1]\"\n [[ $owner_host == \"$(hostname -s)\" ]] || {\n print -u2 \"vps-deploy: receipt belongs to $owner_host; recovery must run on that host\"\n exit 1\n }\n other_pids=$(other_deploy_processes \"$owner_pid\") || {\n print -u2 \"vps-deploy: cannot verify the process table; refusing recovery\"\n exit 75\n }\n [[ -z $other_pids ]] || {\n print -u2 \"vps-deploy: another deployer process is still present; refusing recovery\"\n print -u2 \" other deployer PID(s): $other_pids\"\n exit 75\n }\n if ps -p \"$owner_pid\" >/dev/null 2>&1; then\n current_identity=$(lock_process_identity \"$owner_pid\") || {\n print -u2 \"vps-deploy: owner PID exists but its identity cannot be checked; refusing recovery\"\n exit 75\n }\n if [[ $current_identity == \"$owner_identity\" ]]; then\n print -u2 \"vps-deploy: original deploy $recover_id is still running; refusing recovery\"\n exit 75\n fi\n fi\n\n VPS_DEPLOY_STARTED=\n DEPLOY_ALL_DEGRADED=()\n deploy_ports=()\n for ((i=2; i<=${#recovery_lines}; i++)); do\n IFS='|' read -r kind one two <<< \"$recovery_lines[$i]\"\n case $kind in\n SERVICES) VPS_DEPLOY_STARTED=$one ;;\n ORDER) DEPLOY_ALL_DEGRADED=(${=one}) ;;\n PORT) deploy_ports[$one]=$two ;;\n esac\n done\n [[ -n $VPS_DEPLOY_STARTED ]] || {\n print -u2 \"vps-deploy: receipt has no captured running services; refusing an inferred restore\"\n exit 1\n }\n export VPS_DEPLOY_STARTED\n export VPS_DEPLOY_RESTORE_PORTS_FROM_RECEIPT=1\n VPS_DEPLOY_RESTORE_ATTEMPTED=1\n\n print \"vps-deploy: recovery target=$original_target pinned=$original_sha failed_stage=$original_stage\"\n print \"vps-deploy: only the recorded service snapshot will be restored\"\n keep_deploy_flag\n recovery_payload=$(\"$RUBY\" -rjson -e 'puts JSON.generate(status: \"failed\", recovery_state: \"pending\", stage: \"recovery-restore\")')\n admin_request deploy-run-write \"$recover_id\" \"$recovery_payload\" || {\n print -u2 \"vps-deploy: cannot persist recovery-attempt stage; no services were changed\"\n exit 1\n }\n\n recovery_failed=0\n restore_started || recovery_failed=1\n if (( ! recovery_failed )); then\n restore_wait=${VPS_DEPLOY_RESTORE_WAIT:-240}\n for svc in ${=VPS_DEPLOY_STARTED}; do\n admin_service check \"$svc\" >/dev/null 2>&1 || {\n print -u2 \"vps-deploy: recovery verification failed for $svc (rcctl check)\"\n recovery_failed=1\n break\n }\n port=${deploy_ports[$svc]:-}\n if [[ -n $port ]] && ! wait_up \"$port\" \"$restore_wait\"; then\n print -u2 \"vps-deploy: recovery verification failed for $svc /up on port $port\"\n recovery_failed=1\n break\n fi\n done\n fi\n\n if (( recovery_failed )); then\n recovery_payload=$(\"$RUBY\" -rjson -e 'puts JSON.generate(status: \"failed\", recovery_state: \"pending\", stage: \"recovery-failed\")')\n admin_request deploy-run-write \"$recover_id\" \"$recovery_payload\" || \\\n print -u2 \"vps-deploy: recovery failed and the pending receipt could not be updated\"\n print -u2 \"vps-deploy: recovery remains pending; inspect /var/log/daemon before another attempt\"\n exit 1\n fi\n\n recovery_payload=$(\"$RUBY\" -rjson -e 'puts JSON.generate(status: \"recovered\", recovery_state: \"complete\", stage: \"recovered\", recovered_at: Time.now.utc.iso8601)')\n admin_request deploy-run-write \"$recover_id\" \"$recovery_payload\" || {\n print -u2 \"vps-deploy: services answered, but the recovery receipt could not be finalized\"\n exit 1\n }\n print \"vps-deploy: recovered $recover_id target=$original_target pinned=$original_sha (previous stage=$original_stage)\"\n exit 0\nfi\n\n# Pin the commit, prove it boots, record what is running.\n#\n# Done once, by the process that owns the lock, before any service is touched.\n#\n# The deploy used to pull origin/main at whatever instant each step reached its\n# pull, so a commit pushed mid-pass was deployed half-way through it. On 2026-10-12\n# the pulled tip carried a SyntaxError and later a law that raised at boot; master's\n# container died on the box and the pass halted with production already shed. Now\n# the target is one SHA, resolved here (VPS_DEPLOY_SHA, or origin/main fetched\n# once), and every later step checks HEAD against it instead of pulling.\n#\n# The checks (MASTER/lib/operator/canary.rb) ran on the operator's\n# machine when `bin/operator vps deploy --remote` started this run and said so with\n# VPS_DEPLOY_PREFLIGHT_SHA; a direct run on the box has no such word and runs the\n# MASTER-only subset itself on the pinned commit, detached, so a refusal leaves the\n# checkout where it was. VPS_DEPLOY_SKIP_PREFLIGHT=1 is break-glass and, like the\n# runtime-check skip, needs the human-only I_UNDERSTAND_FAST_DEPLOY=1.\ndeploy_refuse() { log \"refused: $1\"; print -u2 \"vps-deploy: refused - $1\"; print -u2 \" Nothing was stopped.\"; exit 1 }\nif (( lock_owner )); then\n set_stage pin\n retry \"${VPS_DEPLOY_GIT_RETRIES:-3}\" 2 git_as_deploy_user fetch -q origin main \\\n || deploy_refuse \"git fetch origin main failed (as $DEPLOY_USER) after retries\"\n pin=$(git_as_deploy_user rev-parse --verify -q \"${VPS_DEPLOY_SHA:-FETCH_HEAD}^{commit}\") \\\n || deploy_refuse \"${VPS_DEPLOY_SHA:-origin/main} is not a commit this checkout can see\"\n head_before=$(git_as_deploy_user rev-parse HEAD)\n git_as_deploy_user merge-base --is-ancestor \"$head_before\" \"$pin\" \\\n || deploy_refuse \"$pin is not a fast-forward of the box's HEAD $head_before\"\n export VPS_DEPLOY_SHA=$pin\n print \"vps-deploy: pinned $pin (box HEAD was $head_before; git as $DEPLOY_USER)\"\n # A CI attestation names one commit; tests that passed on another prove nothing here.\n if [[ -n ${VPS_DEPLOY_CI_SHA:-} ]]; then\n [[ $VPS_DEPLOY_CI_SHA == $pin ]] \\\n || deploy_refuse \"bin/ci ran on $VPS_DEPLOY_CI_SHA but this run would deploy $pin\"\n print \"vps-deploy: bin/ci passed on the operator's machine for $pin; the box skips its suite\"\n fi\n if [[ -n ${VPS_DEPLOY_PREFLIGHT_SHA:-} ]]; then\n [[ $VPS_DEPLOY_PREFLIGHT_SHA == $pin ]] \\\n || deploy_refuse \"preflight ran on $VPS_DEPLOY_PREFLIGHT_SHA but this run would deploy $pin\"\n print \"vps-deploy: preflight passed on the operator's machine for $pin\"\n elif [[ ${VPS_DEPLOY_SKIP_PREFLIGHT:-} == 1 ]]; then\n [[ ${I_UNDERSTAND_FAST_DEPLOY:-} == 1 ]] \\\n || deploy_refuse \"VPS_DEPLOY_SKIP_PREFLIGHT=1 needs the human-only I_UNDERSTAND_FAST_DEPLOY=1\"\n print \"vps-deploy: WARNING - preflight skipped (break-glass); $pin was not proven to boot\"\n else\n set_stage preflight-boot\n print \"vps-deploy: preflight on $pin (syntax, law files, boot smoke)\"\n preflight_back_to=$(git_as_deploy_user symbolic-ref -q --short HEAD || print -r -- \"$head_before\")\n # A box left detached comes back to its commit, not to a guessed branch.\n preflight_back_flag=()\n git_as_deploy_user symbolic-ref -q HEAD >/dev/null || preflight_back_flag=(--detach)\n # Set recovery state before switching: the EXIT/HUP/INT/TERM path now owns\n # restoration even if the signal lands immediately after the detach.\n preflight_active=1\n git_as_deploy_user switch -q --detach \"$pin\" || deploy_refuse \"cannot check out $pin in $repo (local changes?)\"\n if ! \"$RUBY\" \"$repo/MASTER/lib/operator/canary.rb\" --no-rehearse; then\n restore_preflight_checkout || deploy_refuse \"preflight failed on $pin and checkout restoration failed; restore $preflight_back_to before retrying\"\n deploy_refuse \"preflight failed on $pin; the checkout is back on $preflight_back_to and the previous release is still serving\"\n fi\n restore_preflight_checkout || deploy_refuse \"cannot return to $preflight_back_to after preflight\"\n fi\n git_as_deploy_user merge -q --ff-only \"$pin\" || deploy_refuse \"cannot fast-forward $repo to $pin\"\n VPS_DEPLOY_STARTED=\"$(admin_service list-started 2>/dev/null)\"\n VPS_DEPLOY_STARTED=${VPS_DEPLOY_STARTED//$'\\n'/ }\n export VPS_DEPLOY_STARTED\n VPS_DEPLOY_RECEIPT_PREVIOUS_SHA=$head_before\n VPS_DEPLOY_RECEIPT_PINNED_SHA=$pin\n VPS_DEPLOY_RECEIPT_SERVICES=$VPS_DEPLOY_STARTED\n VPS_DEPLOY_RECEIPT_RECOVERY=none\n [[ -z $VPS_DEPLOY_STARTED ]] || VPS_DEPLOY_RECEIPT_RECOVERY=pending\n export VPS_DEPLOY_RECEIPT_PREVIOUS_SHA VPS_DEPLOY_RECEIPT_PINNED_SHA\n export VPS_DEPLOY_RECEIPT_SERVICES VPS_DEPLOY_RECEIPT_RECOVERY\n print \"vps-deploy: started before this deploy: $VPS_DEPLOY_STARTED\"\n set_stage snapshot\nfi\n\n # Resolve against the inventory in the checkout after it has been fast-forwarded\n # to VPS_DEPLOY_SHA. A stale box checkout must not define target membership.\n deploy_target_kind=$(inventory_target_kind \"$app\") || deploy_refuse \"could not read the pinned deployment inventory for $app\"\n [[ $deploy_target_kind != unknown ]] || deploy_refuse \"unknown app or group $app in the pinned RAILS/apps.yml\"\n DEPLOY_ALL=(${(@f)$(inventory_names all)})\n DEPLOY_ALL_DEGRADED=(${(@f)$(inventory_names all 1)})\n DEPLOY_RAILS=(${(@f)$(inventory_names rails)})\n load_deploy_ports\n typeset -a receipt_ports\n for svc in ${(k)deploy_ports}; do\n receipt_ports+=(\"$svc=${deploy_ports[$svc]}\")\n done\n VPS_DEPLOY_RECEIPT_PORTS=\"${(j:,:)receipt_ports}\"\n VPS_DEPLOY_RECEIPT_ORDER=\"${(j:,:)DEPLOY_ALL_DEGRADED}\"\n VPS_DEPLOY_RECEIPT_SNAPSHOT_RECORDED=1\n export VPS_DEPLOY_RECEIPT_PORTS VPS_DEPLOY_RECEIPT_ORDER\n export VPS_DEPLOY_RECEIPT_SNAPSHOT_RECORDED\n set_stage inventory-ready\n (( ${#DEPLOY_ALL} )) || deploy_refuse \"the pinned deployment inventory has no active apps\"\n deploy_app_group=$(inventory_app_group \"$app\")\n if [[ $deploy_target_kind == all || $deploy_app_group == master ]]; then\n ensure_smart_turn_assets\n fi\n\n# The deployment sequence is data in RAILS/apps.yml. Degraded mode\n# moves the MASTER namespace last; all other relative order stays unchanged.\nchoose_deploy_order() {\n local svc port avail why=\n load_deploy_ports\n avail=$(\"$RUBY\" -e 'print File.readlines(ARGV[0]).last.to_s[/mem_avail=(\\d+)/, 1].to_s' \\\n \"${VPS_DEPLOY_GUARD_LOG:-/var/log/resource_guard_history.log}\" 2>/dev/null)\n for svc in $DEPLOY_RAILS; do\n (( ${${=VPS_DEPLOY_STARTED:-}[(Ie)$svc]} )) || continue\n port=${deploy_ports[$svc]:-}\n [[ -n $port ]] && wait_up \"$port\" 5 || why=\"$svc is not answering /up\"\n done\n if [[ -z $why ]]; then\n [[ $avail == <-> ]] || why=\"memory availability unreadable\"\n fi\n if [[ -z $why ]] && (( avail < ${VPS_DEPLOY_MASTER_MIN_AVAIL:-10} )); then\n why=\"memory availability ${avail}% is under ${VPS_DEPLOY_MASTER_MIN_AVAIL:-10}%\"\n fi\n if [[ -z $why ]]; then\n deploy_order=($DEPLOY_ALL)\n else\n print \"vps-deploy: all - MASTER moved to the end: $why\"\n deploy_order=($DEPLOY_ALL_DEGRADED)\n fi\n}\n\nif [[ $deploy_target_kind == all || $deploy_target_kind == group ]]; then\n # The parent spans child deploys and cooldowns. Its own flag keeps the\n # resource guard out until the whole batch (or its failure recovery) ends.\n keep_deploy_flag\n typeset -i first=1\n typeset -a deploy_order\n if [[ $deploy_target_kind == all ]]; then\n choose_deploy_order\n else\n deploy_order=(${(@f)$(inventory_names \"$app\")})\n fi\n (( ${#deploy_order} )) || { print -u2 \"vps-deploy: deployment group $app is empty\"; exit 2 }\n print \"vps-deploy: $app order: ${deploy_order}\"\n if [[ $deploy_target_kind == all ]]; then\n set_stage openbsd-mirror\n print \"vps-deploy: applying tracked OpenBSD filesystem mirror\"\n zsh \"$repo/OPENBSD/OPERATOR.sh\" --apply || deploy_refuse \"OpenBSD filesystem mirror apply failed; no application deployment was started\"\n fi\n for one in $deploy_order; do\n # Let the previous app's deploy drain before starting the next one's CI.\n # Serial builds protect this one-core host from overlapping load peaks.\n if (( ! first )); then\n print \"vps-deploy: $app - settling for ${PUB5_DEPLOY_COOLDOWN:-60}s before $one\"\n sleep ${PUB5_DEPLOY_COOLDOWN:-60}\n fi\n first=0\n print \"vps-deploy: $app -> $one\"\n # Each child executes the complete per-app CI, sync, migration, restart,\n # health-check and deploy-stamp path. A failure halts this group.\n set_stage \"$app:$one\"\n zsh \"$0\" \"$one\" || { halted_app=$one; print -u2 \"vps-deploy: $app halted at $one\"; exit 1 }\n done\n print \"vps-deploy: $app ok (${deploy_order})\"\n exit 0\nfi\n\n# SHA and time are captured at write_stamp, not here. vps_deploy_master.sh\n# pulls origin/main; reading HEAD before that call stamped the pre-pull SHA\n# (76f1a65fa on 2026-08-12 while /health correctly said 5eff4d87a).\n\nwrite_stamp() {\n local name=$1 stamp_status=$2\n local sha\n stamp_written=1\n sha=$(git_as_deploy_user rev-parse --short HEAD)\n admin_request stamp \"$name\" \"$sha\" \"$stamp_status\"\n}\n\nset_stage \"$deploy_app_group\"\nif [[ $deploy_app_group == master ]]; then\n print \"vps-deploy: master web\"\n # master's bundle, native gem builds and precompile run as long as an app's CI,\n # and this branch never reached the keeper below: on 2026-10-27 the guard read\n # deploy=0 through a 30-minute master deploy and shed amber and bsdports twice.\n keep_deploy_flag\n settle_boots\n if zsh \"$repo/OPENBSD/bin/vps_deploy_master.sh\"; then\n # The loopback journeys that need no browser, for this app only.\n set_stage master-flows\n hold_deploy_flag\n if ! FLOW_APP=\"$app\" MASTER_STRICT=deploy CHECK_AUTOFIX=0 CHECK_REQUIRE_LIVE=1 \"$RUBY\" \"$repo/MASTER/checks/runner.rb\" flow_journey; then\n write_stamp \"$app\" failed\n print -u2 \"vps-deploy: master flow journeys failed - see the per-flow lines above\"\n exit 1\n fi\n set_stage master-canary\n if ! \"$RUBY\" \"$repo/OPENBSD/bin/canary\" --local --app \"$app\" --no-flows; then\n write_stamp \"$app\" failed\n print -u2 \"vps-deploy: master local canary failed - see the failed checks above\"\n exit 1\n fi\n set_stage restore-shed\n if restore_started; then\n restore_ok=1\n else\n restore_ok=0\n fi\n if (( ! restore_ok )); then\n write_stamp \"$app\" failed\n print -u2 \"vps-deploy: master is not complete; a service that was running before is still down\"\n exit 1\n fi\n write_stamp \"$app\" ok\n exit 0\n fi\n # A failed run stamps failed, or last_deploy_master.json keeps saying ok\n # from the last good run and /health reports a live SHA while :53187 is\n # closed (measured 2026-08-13).\n write_stamp \"$app\" failed\n exit 1\nfi\n\n[[ -d /home/${app}/app ]] || { print -u2 \"missing /home/${app}/app\"; exit 1 }\n\n# Rails apps do not pull inside vps_ci.sh, so without this step a deploy CIs and\n# syncs whatever the VPS checkout already holds, which can lag origin/main by\n# days. master pulls inside vps_deploy_master.sh; this is the matching step for\n# the three copy-tree apps.\nset_stage pull\n# No pull here: the top-level process pinned one SHA and moved the checkout to it.\n# A checkout that has moved off it since means another hand touched the box.\nprint \"vps-deploy: $app (checkout must be ${VPS_DEPLOY_SHA:?no pinned SHA, run through the top-level vps-deploy})\"\n[[ $(git_as_deploy_user rev-parse HEAD) == \"$VPS_DEPLOY_SHA\" ]] \\\n || { print -u2 \"vps-deploy: $app refused - HEAD is $(git_as_deploy_user rev-parse HEAD), not the pinned $VPS_DEPLOY_SHA\"; exit 1 }\n\n# Free space, before anything is stopped. A bundle install, an asset precompile or\n# a CI mirror that runs out of disk dies halfway, after the app was stopped for the\n# migrate; refusing here costs nothing and leaves the previous release serving.\n# /home holds the checkout, the app trees, bundles and storage (3.9 GB free of\n# 17 GB on 2026-10-12), so it is the filesystem that fills. The floor is a\n# variable because it is a measurement of this box, not a law.\nset_stage preflight-disk\nmin_free_kb=${VPS_DEPLOY_MIN_FREE_KB:-1048576}\nfor mount_point in /home /var; do\n free_kb=${${(z)${${(f)\"$(${VPS_DEPLOY_DF:-df} -Pk $mount_point 2>/dev/null)\"}[2]}}[4]:-}\n if [[ $free_kb != <-> ]]; then\n print -u2 \"vps-deploy: $app refused - free-space measurement for $mount_point is unavailable.\"\n print -u2 \" Nothing was stopped. Check the filesystem and measurement command, then rerun:\"\n print -u2 \" doas zsh OPENBSD/bin/vps-deploy $app\"\n exit 1\n fi\n if (( free_kb < min_free_kb )); then\n print -u2 \"vps-deploy: $app refused - $mount_point has $(( free_kb / 1024 )) MB free, the floor is $(( min_free_kb / 1024 )) MB.\"\n print -u2 \" Nothing was stopped. Inspect usage with: doas /usr/local/sbin/pub5-admin disk-usage $mount_point ; clear old\"\n print -u2 \" logs or bundles, then rerun: doas zsh OPENBSD/bin/vps-deploy $app\"\n exit 1\n fi\ndone\n\nset_stage preflight\nprint \"vps-deploy: $app (preflight load)\"\nadmin_request resource-guard || true\n\nkeep_deploy_flag\n\n# What is about to be deployed, against what is actually running.\n#\n# The check compares OPENBSD/mirror/etc byte-for-byte with live /etc and root's crontab\n# with data/root.crontab. A deploy is exactly when that matters — it is the moment\n# the repo becomes the box.\n#\n# Advisory, not blocking, and deliberately so. Measured 2026-09-10 with\n# `SSH_HOST=dev@brgen.no ruby OPENBSD/checks/config_drift_check.rb --remote`: eight files\n# differ and two more are absent from the box entirely — emergency_cpu.sh, which\n# is resource_guard's crisis tier, and vps_weekly_integrity.sh, which the crontab\n# has scheduled for weeks. A hard check here would refuse every deploy until an\n# OPERATOR.sh run nobody has scheduled, and one of the eight is rc.d/master, which\n# starts a production service. Make it blocking the first time a deploy runs with\n# this clean; a warning that names a number is still the difference between\n# knowing and not.\nif [[ -x /usr/local/bin/config_drift_check.rb ]]; then\n if ! \"$RUBY\" /usr/local/bin/config_drift_check.rb > /tmp/vps-deploy-drift.out 2>&1; then\n # ruby, not grep: coreutils are banned in committed scripts here, and the\n # check's own summary line already carries the count in the shape we want.\n drift_count=$(\"$RUBY\" -e \\\n 'print File.read(\"/tmp/vps-deploy-drift.out\").scan(/: DRIFT -/).size' 2>/dev/null)\n print \"vps-deploy: $app WARNING - ${drift_count:-?} config(s) on the box differ from the repo being deployed\"\n print \"vps-deploy: $app see /tmp/vps-deploy-drift.out\"\n # Blocking is opt-in until the box has run clean once; the default stays the\n # warning above so an unscheduled OPERATOR.sh run cannot freeze every deploy.\n if [[ ${VPS_DEPLOY_DRIFT:-warn} == fail ]]; then\n print -u2 \"vps-deploy: $app refused - VPS_DEPLOY_DRIFT=fail and the box has drifted\"\n exit 1\n fi\n fi\nfi\n\nif [[ ${SKIP_CI:-} == 1 ]]; then\n print \"vps-deploy: $app (sync via ${app}.sh, SKIP_CI=1)\"\n # The app entrypoints refuse root (line 5 of every one of them: checkout\n # code is operator-owned, privileged actions go through dev's doas and the\n # pub5_rcctl broker), so the sync runs as the deploy user, not as this\n # script's root. First exercised on vm23 2026-11-08: as root, brgen.sh\n # refused and the app stage died before stop, leaving the release serving.\n run_as_deploy_user \"cd $(printf '%q' \"$repo\") && zsh $(printf '%q' \"$repo/RAILS/${app}/${app}.sh\")\"\nelse\n # The app's resident Solid Queue worker stands down for the CI run.\n #\n # vm23 is 1 GB. brgen_jobs holds ~380 MB resident and brgen's suite is 896\n # tests: on 2026-08-23 the runner was OOM-killed mid-suite with swap at 898 MB\n # of 1264, and the failure is silent in the worst way — 155 dots, no summary,\n # no failure block, and vps_ci reporting \"Tests: Rails failed\". Dots then\n # nothing in a CI test section means the runner was killed, not that a test\n # failed. Stopped and restarted here so the knowledge lives in the deploy\n # rather than in whoever remembers to type it.\n # No other app may still be booting when CI starts: a Falcon cold boot and a test\n # suite on one core is how the apps were shed.\n settle_boots\n jobs_service=\"${app}_jobs\"\n jobs_was_up=0\n if admin_service get \"$jobs_service\" >/dev/null 2>&1 && admin_service check \"$jobs_service\" >/dev/null 2>&1; then\n print \"vps-deploy: $app (stand down $jobs_service for the CI run)\"\n admin_service stop \"$jobs_service\" >/dev/null 2>&1 && { jobs_was_up=1; own_stops+=(\"$jobs_service\") }\n fi\n\n # Swap floor, measured after the stand-down above has freed what it frees.\n # CI, bundle and asset work on a 1 GB box with swap nearly gone is how every\n # app was shed: on 2026-11-03 a gem fetch at ~200 MB free took all sites down\n # for 17 minutes, and on 2026-11-04 a MASTER run at 217 MB free timed brgen out.\n set_stage preflight-swap\n min_swap_mb=${VPS_DEPLOY_MIN_FREE_SWAP_MB:-400}\n swap_blocks=${${(M)${(z)\"$(${VPS_DEPLOY_SWAPCTL:-swapctl} -s 2>/dev/null)\"}:#<->}[3]:-}\n if [[ $swap_blocks == <-> ]] && (( swap_blocks * 512 / 1048576 < min_swap_mb )); then\n (( jobs_was_up )) && admin_service start \"$jobs_service\" >/dev/null 2>&1\n free_swap_mb=$(( swap_blocks * 512 / 1048576 ))\n print -u2 \"vps-deploy: $app refused - $free_swap_mb MB of swap free, the floor is $min_swap_mb MB.\"\n print -u2 \" The app's jobs service is back up; nothing else was stopped. Free memory (stop another\"\n print -u2 \" app's *_jobs service), then rerun: doas zsh OPENBSD/bin/vps-deploy $app\"\n exit 1\n fi\n\n set_stage ci+sync\n print \"vps-deploy: $app (ci + sync)\"\n # `|| ci_status=$?` rather than a bare call and `ci_status=$?` after it. This\n # script runs under `set -e`, so a bare failing CI run exits here and the\n # restart below — the whole reason the stand-down records jobs_was_up — never\n # runs, leaving the worker stopped with nothing 5xx'ing and the queue simply not\n # draining. A command followed by `||` is exempt from errexit, which is what\n # makes the status capture reachable at all.\n ci_status=0\n zsh \"$repo/OPENBSD/bin/vps_ci.sh\" \"$app\" || ci_status=$?\n\n if (( jobs_was_up )); then\n admin_service start \"$jobs_service\" >/dev/null 2>&1 \\\n || print -u2 \"vps-deploy: $jobs_service did not come back - start it\"\n fi\n (( ci_status == 0 )) || exit $ci_status\nfi\n\n# Primary database, then the cache/queue/cable schemas.\n#\n# db_create_migrate_as_app runs db:prepare, which is idempotent — it creates the\n# database when absent, loads the schema when empty, and otherwise runs only\n# pending migrations — and then does the secondary schemas itself. Loading the\n# secondaries alone leaves the primary unmigrated, so a merged migration never\n# reaches production: on 2026-07-29 brgen shipped a counter-cache column its\n# views already read, and every feed render 500'd until an operator ran the\n# three pending migrations by hand.\nset_stage migrate\nprint \"vps-deploy: $app (primary db migrate + secondary schemas)\"\n. \"$repo/RAILS/_core.sh\"\n. \"$repo/RAILS/_bundle.sh\"\n. \"$repo/RAILS/_runtime_check.sh\"\n. \"$repo/RAILS/_database.sh\"\n. \"$repo/RAILS/_assets.sh\"\n# Same reason RAILS/_deploy.sh stops first: db:prepare hangs on the write\n# lock a live Falcon worker already holds on production.sqlite3.\nprint \"vps-deploy: $app (stop for sqlite migrate)\"\nadmin_service stop \"$app\" 2>/dev/null || true\napp_stopped=1\nown_stops+=(\"$app\")\ndb_create_migrate_as_app \"$app\" \"/home/${app}/app\"\n\n# The SKIP_CI=1 path (${app}.sh -> deploy_tracked_app -> rails_runtime_check)\n# already precompiles as part of its check. The default ci+sync path\n# (vps_ci.sh) only runs tests against a throwaway test DB and never\n# touches production assets -- without this, public/assets/ is simply\n# never created for a plain `vps-deploy $app` run, so Propshaft 404s\n# every asset_path() and the page renders with no CSS at all. Safe to\n# call unconditionally: Propshaft precompile is idempotent.\nset_stage precompile\nprint \"vps-deploy: $app (assets precompile)\"\nsettle_boots\nrails_assets_precompile_as_app \"$app\" \"/home/${app}/app\" || exit 1\n\nset_stage restart\nprint \"vps-deploy: $app (restart)\"\nhold_deploy_flag\nif ! admin_service_retry restart \"$app\"; then\n log \"restart failed after retries\"\n write_stamp \"$app\" failed\n exit 1\nfi\nif ! retry 5 2 admin_service check \"$app\"; then\n log \"service check failed after restart\"\n write_stamp \"$app\" failed\n exit 1\nfi\napp_stopped=0\nset_stage health\nport=$(\"$RUBY\" -ryaml -e 'puts YAML.safe_load(File.read(\"'\"$repo\"'/RAILS/apps.yml\")).dig(\"apps\",\"'\"$app\"'\",\"port\")')\nup_ok=0\nfor i in {1..150}; do\n curl -fsS \"http://127.0.0.1:${port}/up\" >/dev/null 2>&1 && { up_ok=1; break; }\n sleep 2\ndone\n[[ $up_ok -eq 1 ]] || { write_stamp \"$app\" failed; print -u2 \"vps-deploy: $app /up not ready on :${port}\"; exit 1; }\n\n# /up answers before Propshaft is involved, so it says nothing about whether the\n# page a visitor gets is styled. On 2026-08-14 brgen served every page with a\n# to a stylesheet that 404'd — public/assets held zero files — and /up,\n# rcctl check, the port probe and the public TLS check all passed. The site was\n# plainly broken and the only thing that noticed was a person looking at it.\n#\n# So ask for the stylesheet the page actually links, through the Host that owns\n# it: these apps are domain-constrained and a bare-IP request is a 403, which\n# would make this check pass by never finding a link to test.\ndomain=$(\"$RUBY\" -ryaml -e 'puts YAML.safe_load(File.read(\"'\"$repo\"'/RAILS/apps.yml\")).dig(\"apps\",\"'\"$app\"'\",\"domain\")')\n# The page and the link are two reads, because under set -e and pipefail one\n# pipeline that finds no link fails the assignment and ends the deploy with no\n# stamp and no message. A home page that does not answer is a failed deploy; a\n# home page that links no stylesheet is the else branch below.\nhome_page=$(curl -fsS --max-time 30 -H \"Host: ${domain}\" \"http://127.0.0.1:${port}/\" 2>/dev/null) || {\n write_stamp \"$app\" failed\n print -u2 \"vps-deploy: $app / did not answer on :${port} with Host ${domain}\"\n exit 1\n}\ncss_href=$(print -r -- \"$home_page\" | \"$RUBY\" -e 'print $stdin.binmode.read[%r{/assets/[^\"]+\\.css}n].to_s')\nif [[ -n $css_href ]]; then\n if curl -fsS -o /dev/null --max-time 30 -H \"Host: ${domain}\" \"http://127.0.0.1:${port}${css_href}\" 2>/dev/null; then\n print \"vps-deploy: $app stylesheet ok (${css_href})\"\n else\n write_stamp \"$app\" failed\n print -u2 \"vps-deploy: $app serves ${css_href} as 404 - precompiled assets missing, the site is unstyled\"\n exit 1\n fi\nelse\n print \"vps-deploy: $app no /assets stylesheet linked on / - nothing to verify\"\nfi\n\nset_stage solid_queue\nqueue_schema=\"$repo/RAILS/${app}/db/queue_schema.rb\"\n# An app whose production config runs jobs in-process has no worker to prove;\n# demanding one, or an hourly drain, would be meaningless.\n# UTF-8 explicitly: the box runs this with no LANG, and production.rb has em dashes.\nqueue_async=0\n\"$RUBY\" -e 'exit File.read(ARGV[0], encoding: \"UTF-8\").match?(/queue_adapter\\s*=\\s*:async\\b/)' \\\n \"$repo/RAILS/${app}/config/environments/production.rb\" 2>/dev/null && queue_async=1\nqueue_empty=0\n[[ -f $queue_schema ]] && \"$RUBY\" -e 'exit File.read(ARGV[0]).include?(\"define(version: 0)\")' \\\n \"$queue_schema\" && queue_empty=1\nif (( ! queue_async && ! queue_empty )) && [[ -f $queue_schema ]]; then\n print \"vps-deploy: $app (solid_queue proof)\"\n admin_request solid-queue-proof \"$app\" \\\n || { write_stamp \"$app\" failed; print -u2 \"vps-deploy: solid_queue proof failed for $app\"; exit 1; }\nfi\n\n# Restore whatever the guard shed while we were deploying.\n#\n# resource_guard sheds its OPTIONAL list — bsdports, then amber — under\n# sustained pressure, and a deploy is exactly that. relayd keeps answering TLS\n# with their ports closed, so the outage reads as a hang rather than a 5xx and\n# nothing reports it: on 2026-08-23 amber and bsdports were restarted by hand\n# three times in one session, each time discovered by a person running curl.\n#\n# `vps-deploy all` folds the restore into its ordering; this pass gives a\n# single-app deploy, the common case, the same restore. Reported, not silent — a\n# service that needed restoring is a fact about the box worth seeing.\nset_stage restore-shed\nrestore_failed=0\nrestore_started || restore_failed=1\n\n# The checks, on the deployed thing.\n#\n# Outside this pass the checks run only when a person types a command, so the\n# half that can only be measured against a running app would be measured on a\n# laptop where it means less. A user journey is real only here: production\n# answers 403 to a request without the right Host header, which development\n# never does.\n#\n# This set is the cheap half: source checks plus the live probes against the\n# loopback. It runs in about forty seconds on this one core with the fleet up,\n# which is what makes it affordable on every deploy.\n#\n# The browser half — rendered_suite, layout_suite, visual_contract — is NOT here\n# and is opt-in through PUB5_DEPLOY_BROWSER_CHECKS=1. Chrome is installed, but\n# this box is 1GB with four apps resident and resource_guard sheds amber and\n# bsdports under exactly that pressure. A deploy that takes the site down to\n# check the site has failed at the only thing it was for.\n#\n# CHECK_AUTOFIX=0 because a check must not rewrite the tree it is judging, least\n# of all the one that is deployed.\n#\n# CHECK_REQUIRE_LIVE=1 because this is the one host where a live check has no\n# excuse. It turns \"port closed, skipping\" into a failure, and half this set —\n# flow_journey, page_simulation, port_inventory — measures nothing without a\n# listening app. The fleet was just restarted and the shed apps restored a few\n# lines above, so a closed port here is a deploy that did not come up, not a\n# laptop. bin/check-rails takes the same position on the box through\n# Environment.on_vps?, and a second caller on the same host must not quietly\n# accept a skip.\ncheck_names=(production port_inventory generated_asset schema_migration\n phantom_foreign_keys frontend_production stimulus_components\n flow_journey page_simulation)\nif [[ -n ${PUB5_DEPLOY_BROWSER_CHECKS:-} ]]; then\n check_names+=(rendered_suite layout_suite)\nfi\nset_stage checks\nprint \"vps-deploy: $app (checks: ${check_names})\"\nhold_deploy_flag\nif ! FLOW_APP=\"$app\" MASTER_STRICT=deploy CHECK_AUTOFIX=0 \"$RUBY\" \"$repo/MASTER/checks/runner.rb\" \"${check_names[@]}\"; then\n write_stamp \"$app\" failed\n print -u2 \"vps-deploy: $app checks failed - the app is deployed and serving, and something\"\n print -u2 \" it is now serving does not hold. Fix or revert; do not deploy over it.\"\n exit 1\nfi\n\nset_stage final-check\n# The app deployed last has to be the app still running when the deploy says ok.\n# Every check above ran minutes ago, and a deploy whose own app is down on exit\n# is the one outcome the stamp must never call ok.\nif ! admin_service check \"$app\" >/dev/null 2>&1; then\n write_stamp \"$app\" failed\n print -u2 \"vps-deploy: $app is down at the end of its own deploy - /usr/local/sbin/pub5-admin service restart $app, then read /var/log/daemon\"\n exit 1\nfi\n\n# The local canary is a release gate, not an advisory log line.\nif ! \"$RUBY\" \"$repo/OPENBSD/bin/canary\" --local --app \"$app\"; then\n write_stamp \"$app\" failed\n print -u2 \"vps-deploy: $app local canary failed - see the failed checks above\"\n exit 1\nfi\n\nif (( restore_failed )); then\n write_stamp \"$app\" failed\n print -u2 \"vps-deploy: $app is not complete; a service that was running before is still down\"\n exit 1\nfi\n\nprint \"vps-deploy: $app ok\"\nwrite_stamp \"$app\" ok\nexit 0\n" to include "def other_deploy_processes". 2) Failure: VpsDeployContractTest#test_operator_installs_pub5_broker_and_its_solid_queue_helper [test/test_vps_deploy_contract.rb:214]: Expected "#!/usr/bin/env zsh\n# Copy-ready OpenBSD filesystem payload. No Ruby, rendering, templating or deletion.\n# OPENBSD/mirror/ installs to /. Operator tools live in OPENBSD/bin and lib.\nset -euo pipefail\nsetopt extended_glob null_glob\n\nSCRIPT_DIR=${0:A:h}\nREPO_ROOT=${SCRIPT_DIR:h}\nMIRROR=${SCRIPT_DIR}/mirror\nDATA=${SCRIPT_DIR}/data\nBACKUP_ROOT=${OPENBSD_BACKUP_ROOT:-/var/backups/openbsd_setup}\nACTION=plan\nINSTALL_DOAS=0\nDOAS_NEEDED=0\nRUN_ID=\nLOCK_DIR=${OPENBSD_APPLY_LOCK_DIR:-/var/run/pub5-openbsd-apply.lock}\nLOCK_HELD=0\ntypeset -a FILES CHANGED SIGNED_BEFORE\ntypeset -A BACKUPS\ntypeset -i failures=0 CRONTAB_APPLIED=0\n\nusage() {\n print -r -- \"usage: doas zsh OPENBSD/OPERATOR.sh [--plan|--apply|--verify] [--doas]\"\n print -r -- \" --plan compare tracked mirror files with vm23 (default, read-only)\"\n print -r -- \" --apply back up and install changed files, then validate/reload affected services\"\n print -r -- \" --verify require the installed filesystem and managed root cron to match\"\n print -r -- \" --doas explicitly install etc/doas.conf through validate_doas.ksh with its canary/rollback\"\n print -r -- \" sshd_config is always protected from automatic replacement.\"\n}\n\nfor arg in \"$@\"; do\n case \"$arg\" in\n --plan) ACTION=plan ;;\n --apply) ACTION=apply ;;\n --verify) ACTION=verify ;;\n --doas) INSTALL_DOAS=1 ;;\n -h|--help) usage; exit 0 ;;\n *) print -u2 -- \"OPERATOR: unknown argument: $arg\"; usage >&2; exit 64 ;;\n esac\ndone\n\n[[ $(uname -s) == OpenBSD ]] || {\n print -u2 -- \"OPERATOR: BLOCKED - this installer compares and changes the local OpenBSD host; current host is $(uname -s)\"\n exit 78\n}\nif [[ $ACTION != plan && $(id -u) != 0 ]]; then\n print -u2 -- \"OPERATOR: BLOCKED - --$ACTION must run as root (use doas)\"\n exit 78\nfi\nif (( INSTALL_DOAS )) && [[ $ACTION != apply ]]; then\n print -u2 -- \"OPERATOR: --doas is valid only with --apply\"\n exit 64\nfi\n[[ -d $MIRROR && -d $DATA ]] || {\n print -u2 -- \"OPERATOR: required OPENBSD/mirror or OPENBSD/data directory is missing\"\n exit 1\n}\n\n# Only Git-tracked regular files under the four filesystem roots are eligible.\nwhile IFS= read -r -d '' tracked; do\n rel=${tracked#OPENBSD/mirror/}\n case \"$rel\" in\n etc/*|home/*|usr/*|var/*) ;;\n *) print -u2 -- \"OPERATOR: invalid mirror path: $rel\"; exit 1 ;;\n esac\n source_file=$MIRROR/$rel\n [[ -f $source_file && ! -L $source_file ]] || {\n print -u2 -- \"OPERATOR: BLOCKED - mirror source is missing, non-regular or symlinked: $rel\"\n exit 1\n }\n FILES+=(\"$rel\")\ndone < <(git -c safe.directory=\"$REPO_ROOT\" -C \"$REPO_ROOT\" ls-files -z -- OPENBSD/mirror)\n\n(( ${#FILES} > 0 )) || {\n print -u2 -- \"OPERATOR: BLOCKED - no tracked filesystem payload found under OPENBSD/mirror\"\n exit 1\n}\n\nrelease_apply_lock() {\n (( LOCK_HELD )) || return 0\n rm -f \"$LOCK_DIR/owner\"\n rmdir \"$LOCK_DIR\" 2>/dev/null || print -u2 -- \"OPERATOR: stale lock directory remains: $LOCK_DIR\"\n LOCK_HELD=0\n}\n\nacquire_apply_lock() {\n if ! mkdir -m 700 \"$LOCK_DIR\" 2>/dev/null; then\n print -u2 -- \"OPERATOR: BLOCKED - another OpenBSD apply owns $LOCK_DIR (inspect it; do not remove a live lock)\"\n return 1\n fi\n LOCK_HELD=1\n print -r -- \"pid=$$ host=$(hostname) started=$(date -u +%Y-%m-%dT%H:%M:%SZ)\" > \"$LOCK_DIR/owner\" || {\n release_apply_lock\n return 1\n }\n trap release_apply_lock EXIT\n trap 'exit 129' HUP\n trap 'exit 130' INT\n trap 'exit 143' TERM\n}\n\nfile_mode() {\n local rel=$1 source_file=$MIRROR/$1\n case \"$rel\" in\n etc/rc.d/master|etc/rc.d/brgen|etc/rc.d/amber|etc/rc.d/bsdports|etc/rc.d/mrdamascy|usr/local/sbin/*) print 555 ;;\n etc/doas.conf|etc/ssh/sshd_config) print 600 ;;\n etc/rc.d/*) print 755 ;;\n etc/daily.local|etc/weekly.local|usr/local/bin/*|usr/local/libexec/*) print 755 ;;\n home/johann/.muttrc|home/johann/.mailcap) print 600 ;;\n home/johann/bin/*) print 755 ;;\n var/nsd/zones/master/*) print 640 ;;\n *) if [[ -x $source_file ]]; then print 755; else print 644; fi ;;\n esac\n}\n\nfile_owner() {\n case \"$1\" in\n home/johann/*) print johann:johann ;;\n var/nsd/zones/master/*) print _nsd:_nsd ;;\n *) print root:wheel ;;\n esac\n}\n\nfile_matches() {\n local rel=$1 source_file=$MIRROR/$1 target=/$1 attrs owner_group owner group mode\n [[ -f $target && ! -L $target ]] || return 1\n cmp -s \"$source_file\" \"$target\" || return 1\n mode=$(file_mode \"$rel\")\n owner_group=$(file_owner \"$rel\")\n owner=${owner_group%%:*}\n group=${owner_group##*:}\n attrs=$(stat -f '%Lp:%Su:%Sg' \"$target\") || return 1\n [[ $attrs == \"$mode:$owner:$group\" ]]\n}\n\nprint_plan() {\n local rel\n for rel in $FILES; do\n if [[ $rel == etc/ssh/sshd_config ]]; then\n if file_matches \"$rel\"; then\n print \"SAME /$rel (protected)\"\n else\n print -u2 -- \"PROTECTED /$rel (SSH configuration requires an out-of-band recovery path)\"\n failures=1\n fi\n elif [[ $rel == etc/doas.conf ]]; then\n if file_matches \"$rel\"; then\n print \"SAME /$rel (protected)\"\n elif (( INSTALL_DOAS )); then\n print \"GUARDED /$rel (dedicated doas canary and rollback)\"\n DOAS_NEEDED=1\n else\n print -u2 -- \"PROTECTED /$rel (pass --doas to use validate_doas.ksh)\"\n failures=1\n fi\n elif file_matches \"$rel\"; then\n print \"SAME /$rel\"\n else\n print \"CHANGE /$rel\"\n CHANGED+=(\"$rel\")\n fi\n done\n print \"SPECIAL root crontab: OPENBSD/data/root.crontab (merged through crontab(1), never copied to /etc)\"\n}\n\nvalidate_payload() {\n [[ ! -f $MIRROR/etc/pf.conf ]] || /sbin/pfctl -nf \"$MIRROR/etc/pf.conf\"\n [[ ! -f $MIRROR/etc/relayd.conf ]] || /usr/sbin/relayd -n -f \"$MIRROR/etc/relayd.conf\"\n [[ ! -f $MIRROR/etc/httpd.conf ]] || /usr/sbin/httpd -n -f \"$MIRROR/etc/httpd.conf\"\n [[ ! -f $MIRROR/etc/mail/smtpd.conf ]] || /usr/sbin/smtpd -n -f \"$MIRROR/etc/mail/smtpd.conf\"\n [[ ! -f $MIRROR/etc/ssh/sshd_config ]] || /usr/sbin/sshd -t -f \"$MIRROR/etc/ssh/sshd_config\"\n if [[ -f $MIRROR/var/nsd/etc/nsd.conf ]]; then\n /usr/local/sbin/nsd-checkconf \"$MIRROR/var/nsd/etc/nsd.conf\" || return 1\n local zone\n for zone in \"$MIRROR\"/var/nsd/zones/master/*.zone(N); do\n /usr/local/sbin/nsd-checkzone \"${zone:t:r}\" \"$zone\" || {\n print -u2 -- \"OPERATOR: NSD zone validation failed: $zone\"\n return 1\n }\n done\n fi\n}\n\nno_symlink_path() {\n local path=$1 current=/ component\n local -a pieces\n pieces=(${(s:/:)path})\n for component in $pieces; do\n [[ -n $component ]] || continue\n current=${current%/}/$component\n [[ ! -L $current ]] || return 1\n done\n return 0\n}\n\nbackup_signed_zones() {\n local dir=$BACKUP_ROOT/$RUN_ID/nsd-signed zone_file\n install -d -o root -g wheel -m 700 \"$dir\" || return 1\n SIGNED_BEFORE=(/var/nsd/zones/master/*.zone.signed(N))\n for zone_file in $SIGNED_BEFORE; do\n cp -p \"$zone_file\" \"$dir/${zone_file:t}\" || return 1\n done\n}\n\nrestore_signed_zones() {\n local zone_file base dir=$BACKUP_ROOT/$RUN_ID/nsd-signed\n for zone_file in /var/nsd/zones/master/*.zone.signed(N); do\n (( ${SIGNED_BEFORE[(Ie)$zone_file]} )) || rm -f \"$zone_file\" || return 1\n done\n for zone_file in $SIGNED_BEFORE; do\n base=${zone_file:t}\n [[ -f $dir/$base ]] && cp -p \"$dir/$base\" \"$zone_file\" || return 1\n done\n /usr/local/sbin/nsd-control reload\n}\n\nbackup_file() {\n local rel=$1 target=/$1 backup=$BACKUP_ROOT/$RUN_ID/$1\n install -d -o root -g wheel -m 700 \"$backup:h\"\n if [[ -e $target ]]; then\n [[ -f $target && ! -L $target ]] || {\n print -u2 -- \"OPERATOR: refusing non-regular destination: $target\"\n return 1\n }\n cp -p \"$target\" \"$backup\" || return 1\n cmp -s \"$target\" \"$backup\" || return 1\n BACKUPS[$rel]=$backup\n else\n BACKUPS[$rel]=MISSING\n fi\n}\n\nrestore_files() {\n local rel target backup restore_failed=0 tmp\n for rel in $CHANGED; do\n target=/$rel\n backup=${BACKUPS[$rel]:-}\n [[ -n $backup ]] || continue\n if [[ $backup == MISSING ]]; then\n rm -f \"$target\" || restore_failed=1\n elif [[ -f $backup ]]; then\n tmp=$(mktemp \"$target:h/.pub5-restore.XXXXXXXX\") || { restore_failed=1; continue; }\n cp -p \"$backup\" \"$tmp\" && mv -f \"$tmp\" \"$target\" || {\n rm -f \"$tmp\"\n restore_failed=1\n }\n else\n restore_failed=1\n fi\n done\n return $restore_failed\n}\n\ninstall_one() {\n local rel=$1 source_file=$MIRROR/$1 target=/$1 owner_group owner group mode tmp\n owner_group=$(file_owner \"$rel\")\n owner=${owner_group%%:*}\n group=${owner_group##*:}\n mode=$(file_mode \"$rel\")\n no_symlink_path \"$target\" || {\n print -u2 -- \"OPERATOR: refusing symlinked destination path: $target\"\n return 1\n }\n if [[ $rel == home/johann/* ]] && ! id johann >/dev/null 2>&1; then\n print -u2 -- \"OPERATOR: account johann is absent; refusing to create a home implicitly\"\n return 1\n fi\n if [[ ! -d $target:h ]]; then\n install -d -o \"$owner\" -g \"$group\" -m 755 \"$target:h\" || return 1\n fi\n tmp=$(mktemp \"$target:h/.pub5-install.XXXXXXXX\") || return 1\n if ! install -o \"$owner\" -g \"$group\" -m \"$mode\" \"$source_file\" \"$tmp\" ||\n ! mv -f \"$tmp\" \"$target\"; then\n rm -f \"$tmp\"\n return 1\n fi\n print \"INSTALLED /$rel ($owner:$group $mode)\"\n}\n\nbackup_root_crontab() {\n local backup=$BACKUP_ROOT/$RUN_ID/root.crontab error_file=$BACKUP_ROOT/$RUN_ID/root.crontab.err detail\n if crontab -l -u root > \"$backup\" 2> \"$error_file\"; then\n rm -f \"$error_file\"\n return 0\n fi\n detail=$(<\"$error_file\")\n rm -f \"$error_file\"\n if [[ $detail == *\"no crontab for root\"* ]]; then\n : > \"$backup\"\n return 0\n fi\n print -u2 -- \"OPERATOR: cannot read root crontab safely: $detail\"\n rm -f \"$backup\"\n return 1\n}\n\nrestore_root_crontab() {\n local backup=$BACKUP_ROOT/$RUN_ID/root.crontab\n [[ -f $backup ]] || return 1\n crontab -u root \"$backup\" || return 1\n CRONTAB_APPLIED=0\n print \"RESTORED root crontab\"\n}\n\ninstall_root_crontab() {\n local tracked=$DATA/root.crontab current=$BACKUP_ROOT/$RUN_ID/root.crontab merged path_line line cmdpath index work\n [[ -f $tracked && -f $current ]] || {\n print -u2 -- \"OPERATOR: root crontab source or verified backup is missing\"\n return 1\n }\n work=$(mktemp -d /var/db/pub5-crontab.XXXXXXXX) || return 1\n if ! chown root:wheel \"$work\" || ! chmod 700 \"$work\"; then\n rm -rf \"$work\"\n return 1\n fi\n merged=$work/merged\n path_line=\n while IFS= read -r line; do\n [[ $line == PATH=* ]] && { path_line=$line; break; }\n done < \"$tracked\"\n if [[ -n $path_line ]]; then print -r -- \"$path_line\" > \"$merged\"; else : > \"$merged\"; fi\n while IFS= read -r line; do\n [[ $line == PATH=* ]] && continue\n print -r -- \"$line\" >> \"$merged\"\n done < \"$current\"\n\n while IFS= read -r line; do\n [[ -z $line || $line == \\#* || $line == PATH=* ]] && continue\n local -a fields\n fields=(${=line})\n if [[ ${fields[1]} == @* ]]; then\n (( ${#fields} >= 2 )) || {\n print -u2 -- \"OPERATOR: malformed tracked cron entry: $line\"\n rm -rf \"$work\"\n return 1\n }\n index=2\n else\n (( ${#fields} >= 6 )) || {\n print -u2 -- \"OPERATOR: malformed tracked cron entry: $line\"\n rm -rf \"$work\"\n return 1\n }\n index=6\n fi\n cmdpath=\n while (( index <= ${#fields} )); do\n if [[ ${fields[index]} == /* ]]; then cmdpath=${fields[index]}; break; fi\n (( index++ ))\n done\n [[ -n $cmdpath && -x $cmdpath ]] || {\n print -u2 -- \"OPERATOR: tracked cron command is missing or not executable: $line\"\n rm -rf \"$work\"\n return 1\n }\n cron_has_command \"$cmdpath\" \"$merged\" || print -r -- \"$line\" >> \"$merged\"\n done < \"$tracked\"\n\n if ! crontab -u root \"$merged\"; then\n rm -rf \"$work\"\n return 1\n fi\n CRONTAB_APPLIED=1\n rm -rf \"$work\"\n}\n\ncron_has_command() {\n local command=$1 file=$2\n awk -v command=\"$command\" '\n /^([[:space:]]*#|[[:space:]]*$)/ { next }\n {\n start = ($1 ~ /^@/) ? 2 : 6\n for (i = start; i <= NF; i++) if ($i == command) found = 1\n }\n END { exit(found ? 0 : 1) }\n ' \"$file\"\n}\n\nverify_root_crontab() {\n local tracked=$DATA/root.crontab live line cmdpath index expected_path live_path\n [[ -f $tracked ]] || return 1\n live=$(mktemp /tmp/pub5-crontab.XXXXXXXX) || return 1\n local error_file\n error_file=$(mktemp /tmp/pub5-crontab-error.XXXXXXXX) || { rm -f \"$live\"; return 1; }\n if ! crontab -l -u root > \"$live\" 2>\"$error_file\"; then\n local detail\n detail=$(<\"$error_file\")\n if [[ $detail == *\"no crontab for root\"* ]]; then\n : > \"$live\"\n else\n print -u2 -- \"DRIFT cannot read root crontab: $detail\"\n rm -f \"$error_file\" \"$live\"\n return 1\n fi\n fi\n rm -f \"$error_file\"\n expected_path=\n while IFS= read -r line; do\n [[ $line == PATH=* ]] && { expected_path=$line; break; }\n done < \"$tracked\"\n live_path=\n while IFS= read -r line; do\n [[ $line == PATH=* ]] && { live_path=$line; break; }\n done < \"$live\"\n if [[ -n $expected_path && $expected_path != $live_path ]]; then\n print -u2 -- \"DRIFT root crontab PATH assignment differs\"\n rm -f \"$live\"\n return 1\n fi\n while IFS= read -r line; do\n [[ -z $line || $line == \\#* || $line == PATH=* ]] && continue\n local -a fields\n fields=(${=line})\n index=6\n [[ ${fields[1]} == @* ]] && index=2\n cmdpath=\n while (( index <= ${#fields} )); do\n if [[ ${fields[index]} == /* ]]; then cmdpath=${fields[index]}; break; fi\n (( index++ ))\n done\n if [[ -z $cmdpath ]] || ! cron_has_command \"$cmdpath\" \"$live\"; then\n print -u2 -- \"DRIFT root crontab missing $cmdpath\"\n rm -f \"$live\"\n return 1\n fi\n done < \"$tracked\"\n rm -f \"$live\"\n print \"SAME managed root-crontab entries\"\n}\n\ncase \"$ACTION\" in\n plan)\n [[ ! -d $LOCK_DIR ]] || { print -u2 -- \"OPERATOR: BLOCKED - apply lock is active: $LOCK_DIR\"; exit 75; }\n print \"OPENBSD mirror plan (read-only; source paths map directly to /)\"\n print_plan\n (( failures == 0 )) || exit 78\n ;;\n verify)\n [[ ! -d $LOCK_DIR ]] || { print -u2 -- \"OPERATOR: BLOCKED - apply lock is active: $LOCK_DIR\"; exit 75; }\n print \"OPENBSD mirror verification\"\n print_plan\n verify_root_crontab || failures=1\n (( failures == 0 )) || exit 1\n ;;\n apply)\n print \"OPENBSD mirror apply\"\n print_plan\n if (( DOAS_NEEDED && ${#CHANGED} )); then\n print -u2 -- \"OPERATOR: BLOCKED - --doas may only change doas.conf by itself; apply other mirror changes separately\"\n exit 78\n fi\n (( failures == 0 )) || {\n print -u2 -- \"OPERATOR: BLOCKED - protected or invalid mirror differences must be handled separately\"\n exit 78\n }\n validate_payload || {\n print -u2 -- \"OPERATOR: BLOCKED - native OpenBSD configuration validation failed; nothing was copied\"\n exit 1\n }\n acquire_apply_lock || exit 75\n RUN_ID=$(date -u +%Y%m%dT%H%M%SZ)-$$\n install -d -o root -g wheel -m 700 \"$BACKUP_ROOT/$RUN_ID\"\n # Every recovery input is captured and verified before the first live write.\n for rel in $CHANGED; do\n backup_file \"$rel\" || {\n print -u2 -- \"OPERATOR: backup failed for /$rel; nothing was installed\"\n exit 1\n }\n done\n backup_root_crontab || {\n print -u2 -- \"OPERATOR: root crontab backup failed; nothing was installed\"\n exit 1\n }\n for rel in $CHANGED; do\n install_one \"$rel\" || {\n print -u2 -- \"OPERATOR: install failed for /$rel\"\n restore_files || print -u2 -- \"DELIVERY_FAILED: automatic file restoration failed\"\n exit 1\n }\n done\n for rel in $CHANGED; do\n file_matches \"$rel\" || {\n print -u2 -- \"OPERATOR: installed file did not verify: /$rel\"\n restore_files || print -u2 -- \"DELIVERY_FAILED: automatic file restoration failed\"\n exit 1\n }\n done\n install_root_crontab || {\n print -u2 -- \"OPERATOR: root crontab merge failed\"\n if (( CRONTAB_APPLIED )); then\n restore_root_crontab || { print -u2 -- \"DELIVERY_FAILED: root crontab restoration failed\"; exit 1; }\n fi\n restore_files || print -u2 -- \"DELIVERY_FAILED: automatic file restoration failed\"\n exit 1\n }\n verify_root_crontab || {\n print -u2 -- \"OPERATOR: root crontab read-back verification failed\"\n restore_root_crontab || { print -u2 -- \"DELIVERY_FAILED: root crontab restoration failed\"; exit 1; }\n restore_files || print -u2 -- \"DELIVERY_FAILED: automatic file restoration failed\"\n exit 1\n }\n if (( ${CHANGED[(Ie)etc/pf.conf]} )); then\n /sbin/pfctl -nf /etc/pf.conf && /sbin/pfctl -f /etc/pf.conf || failures=1\n fi\n if (( failures == 0 && ${CHANGED[(Ie)etc/relayd.conf]} )); then\n /usr/sbin/rcctl restart relayd || failures=1\n fi\n if (( failures == 0 && ${CHANGED[(Ie)etc/httpd.conf]} )); then\n /usr/sbin/rcctl restart httpd || failures=1\n fi\n if (( failures == 0 && ${CHANGED[(Ie)etc/mail/smtpd.conf]} )); then\n /usr/sbin/rcctl restart smtpd || failures=1\n fi\n if (( failures == 0 && ${CHANGED[(Ie)var/nsd/etc/nsd.conf]} )); then\n /usr/sbin/rcctl restart nsd || failures=1\n fi\n if (( failures == 0 && ${CHANGED[(I)var/nsd/zones/master/*.zone]} )); then\n if backup_signed_zones; then\n /usr/local/bin/nsd-resign || failures=1\n else\n failures=1\n fi\n fi\n if (( failures )); then\n print -u2 -- \"OPERATOR: activation failed; restoring the previous tracked files\"\n if (( CRONTAB_APPLIED )); then\n restore_root_crontab || { print -u2 -- \"DELIVERY_FAILED: root crontab restoration failed\"; exit 1; }\n fi\n restore_files || { print -u2 -- \"DELIVERY_FAILED: automatic file restoration failed\"; exit 1; }\n if (( ${#SIGNED_BEFORE} )) || [[ -d $BACKUP_ROOT/$RUN_ID/nsd-signed ]]; then\n restore_signed_zones || { print -u2 -- \"DELIVERY_FAILED: DNSSEC signature restoration failed\"; exit 1; }\n fi\n typeset -i recovery_failed=0\n if (( ${CHANGED[(Ie)etc/pf.conf]} )); then\n /sbin/pfctl -nf /etc/pf.conf && /sbin/pfctl -f /etc/pf.conf || recovery_failed=1\n fi\n if (( ${CHANGED[(Ie)etc/relayd.conf]} )); then\n /usr/sbin/rcctl restart relayd && /usr/sbin/rcctl check relayd || recovery_failed=1\n fi\n if (( ${CHANGED[(Ie)etc/httpd.conf]} )); then\n /usr/sbin/rcctl restart httpd && /usr/sbin/rcctl check httpd || recovery_failed=1\n fi\n if (( ${CHANGED[(Ie)etc/mail/smtpd.conf]} )); then\n /usr/sbin/rcctl restart smtpd && /usr/sbin/rcctl check smtpd || recovery_failed=1\n fi\n if (( ${CHANGED[(Ie)var/nsd/etc/nsd.conf]} )); then\n /usr/sbin/rcctl restart nsd && /usr/sbin/rcctl check nsd || recovery_failed=1\n fi\n if (( recovery_failed )); then\n print -u2 -- \"DELIVERY_FAILED: prior files were restored, but one or more services did not recover; inspect the backup and daemon logs\"\n exit 1\n fi\n print \"RECOVERED prior tracked files and affected services\"\n exit 1\n fi\n if (( DOAS_NEEDED )); then\n if ! \"$SCRIPT_DIR/bin/validate_doas.ksh\" install \"$MIRROR/etc/doas.conf\" \"pub5-openbsd\"; then\n print -u2 -- \"OPERATOR: guarded doas configuration change failed (helper restored the prior policy)\"\n restore_root_crontab || { print -u2 -- \"DELIVERY_FAILED: root crontab restoration failed\"; exit 1; }\n exit 1\n fi\n fi\n print \"PASS OPENBSD filesystem mirror applied\"\n print \"BACKUP $BACKUP_ROOT/$RUN_ID\"\n ;;\nesac\n" to include "usr/local/sbin/pub5-admin". 3) Failure: VpsDeployContractTest#test_a_matching_live_process_fingerprint_keeps_its_lock [test/test_vps_deploy_contract.rb:612]: Expected: 75 Actual: 0 4) Failure: VpsDeployContractTest#test_fleet_recovery_receipt_is_created_and_snapshot_precedes_service_stops [test/test_vps_deploy_contract.rb:1066]: Expected "#!/usr/bin/env zsh\n# Serial VPS deploy entrypoint — sync, CI, restart, health, deploy stamp.\n# Usage: zsh OPENBSD/bin/vps-deploy all|GROUP|APP\nset -euo pipefail\n\n# SSH executes this as a non-login shell on vm23; pin the OpenBSD package path.\nexport PATH=/usr/local/bin:/usr/local/sbin:/usr/bin:/usr/sbin:/bin:/sbin\n\nusage() {\n print -r -- \"usage: zsh OPENBSD/bin/vps-deploy all|GROUP|APP (runs as root; re-execs via doas)\n\n all every active registered surface, including MASTER web\n GROUP every registered app whose deploy_root belongs to that namespace\n APP one exact app ID from RAILS/apps.yml\n recover --plan inspect interrupted-deploy recovery receipts\n recover --apply restore only services recorded as up before that run\n VPS_DEPLOY_SHA= deploy exactly this commit (default: origin/main, fetched once)\n VPS_DEPLOY_PREFLIGHT_SHA= the operator's machine already proved this commit boots\n VPS_DEPLOY_CI_SHA= the operator's machine ran the app's bin/ci green on this commit; the box skips its suite\n VPS_DEPLOY_SKIP_PREFLIGHT=1 break-glass, needs I_UNDERSTAND_FAST_DEPLOY=1\n VPS_DEPLOY_SKIP_TESTS=1 break-glass: no test suite anywhere, needs I_UNDERSTAND_FAST_DEPLOY=1\n SKIP_CI=1 skip vps_ci.sh; \\${app}.sh and its runtime check still run\n VPS_DEPLOY_DRIFT=fail refuse to deploy when the box's configs differ from the repo\n PUB5_DEPLOY_BROWSER_CHECKS=1 add the browser checks after restart\n VPS_DEPLOY_LOG=/path aggregate log (default /var/log/pub5/vps-deploy.log)\n VPS_DEPLOY_LOG_DIR=/path log directory (default /var/log/pub5)\n VPS_DEPLOY_RUN_LOG=/path per-run log (default auto under LOG_DIR/runs/)\n\n Git fetch/checkout always runs as user dev so the /home/dev/pub5 tree and the\n GitHub deploy key stay owned and reachable. Service control uses pub5-admin\n directly when already root.\"\n}\ncase ${1:-} in\n -h|--help) usage; exit 0 ;;\nesac\n\n# Prefer root. Passwordless doas (dev in wheel / permit nopass) re-execs so the\n# operator can still type the command as dev. Git never runs as root: root has\n# no GitHub host key by design, and a root checkout would leave root-owned files\n# in the dev tree. Service control calls the broker directly when uid 0.\n#\n# doas strips the caller environment (the dev rule has no keepenv — intentional,\n# so RUBYOPT and friends cannot cross the privilege boundary). Re-exec through\n# /usr/bin/env with a prefix allowlist (VPS_DEPLOY_*, PUB5_DEPLOY_*, plus PUB5_ROOT,\n# SKIP_CI, I_UNDERSTAND_FAST_DEPLOY) so pin, preflight, canary and logging\n# knobs survive. Not keepenv: RUBYOPT and friends still cannot cross.\nif [[ $(id -u) -ne 0 ]]; then\n if command -v doas >/dev/null 2>&1; then\n self=$0\n if [[ $self != /* ]]; then\n self=\"$(cd -- \"$(dirname -- \"$self\")\" && pwd)/$(basename -- \"$self\")\"\n fi\n typeset -a deploy_env\n # Prefix allowlist (not keepenv): every VPS_DEPLOY_* and PUB5_DEPLOY_* that\n # is set in the caller crosses doas via /usr/bin/env. Plus a few one-offs\n # the deploy reads. New knobs named under those prefixes need no edit here.\n for _name in ${(k)parameters}; do\n case $_name in\n VPS_DEPLOY_*|PUB5_DEPLOY_*|PUB5_ROOT|SKIP_CI|I_UNDERSTAND_FAST_DEPLOY)\n deploy_env+=(\"$_name=${(P)_name}\")\n ;;\n esac\n done\n exec doas -n /usr/bin/env \"${deploy_env[@]}\" zsh \"$self\" \"$@\"\n fi\n print -u2 \"vps-deploy: must run as root (or with passwordless doas)\"\n print -u2 \" try: doas zsh OPENBSD/bin/vps-deploy ${1:-all}\"\n exit 2\nfi\n\n# Unprivileged owner of the checkout and GitHub credentials on this box.\nDEPLOY_USER=${VPS_DEPLOY_USER:-dev}\nDEPLOY_HOME=${VPS_DEPLOY_HOME:-/home/$DEPLOY_USER}\nrepo=${PUB5_ROOT:-/home/dev/pub5}\nsource \"$repo/OPENBSD/lib/ruby_select.sh\"\ninventory_target_kind() {\n \"$RUBY\" -r\"$repo/OPENBSD/lib/deploy_inventory\" -e \\\n 'print Deploy::Inventory.new(root: ARGV[0]).target_kind(ARGV[1])' \"$repo\" \"$1\"\n}\n\ninventory_app_group() {\n \"$RUBY\" -r\"$repo/OPENBSD/lib/deploy_inventory\" -e '\n app = Deploy::Inventory.new(root: ARGV[0]).apps.find { |row| row.name == ARGV[1] }\n print app&.deploy_group\n ' \"$repo\" \"$1\"\n}\n\ninventory_names() {\n local target=$1 degraded=${2:-0}\n \"$RUBY\" -r\"$repo/OPENBSD/lib/deploy_inventory\" -e '\n inventory = Deploy::Inventory.new(root: ARGV[0])\n rows = ARGV[1] == \"all\" ? inventory.ordered_apps(degraded: ARGV[2] == \"1\") :\n inventory.ordered_apps(group: ARGV[1])\n puts rows.map(&:name)\n ' \"$repo\" \"$target\" \"$degraded\"\n}\n\n# Run a shell snippet as DEPLOY_USER without pulling root's environment into\n# the checkout. Always su: root may change to DEPLOY_USER without a doas rule.\nrun_as_deploy_user() {\n # Root may su to DEPLOY_USER without a doas rule. Avoid doas -u: the box\n # only permits dev→root and root→root by default.\n local snippet=$1\n su -s /bin/sh \"$DEPLOY_USER\" -c \"$snippet\"\n}\n\ngit_as_deploy_user() {\n # All git mutations against the production checkout run as DEPLOY_USER.\n local args_q\n args_q=$(printf '%q ' \"$@\")\n run_as_deploy_user \"cd $(printf '%q' \"$repo\") && git $args_q\"\n}\n\napp=${1:-}\n[[ -n $app ]] || { usage >&2; exit 2 }\nrecover_mode=\nrecover_id=\nif [[ $app == recover ]]; then\n case ${2:-} in\n --plan)\n [[ $# -eq 2 ]] || { usage >&2; exit 2 }\n recover_mode=plan\n ;;\n --apply)\n [[ $# -eq 3 ]] && \"$RUBY\" -e 'exit(ARGV.fetch(0).match?(/\\A[0-9]{8}T[0-9]{6}Z-[a-z0-9_-]{1,32}-[0-9]+\\z/) ? 0 : 1)' \"$3\" || {\n print -u2 \"vps-deploy: recover --apply needs one exact run ID from --plan\"\n exit 2\n }\n recover_mode=apply\n recover_id=$3\n ;;\n *) usage >&2; exit 2 ;;\n esac\n unset VPS_DEPLOY_RUN_ID VPS_DEPLOY_RUN_TARGET VPS_DEPLOY_RUN_STARTED_AT\n unset VPS_DEPLOY_STARTED VPS_DEPLOY_LOCK_HELD\nelse\n [[ $# -eq 1 ]] || { usage >&2; exit 2 }\n # Restrict the log/path component to a safe identifier before inventory lookup.\n [[ $app != *[^A-Za-z0-9_-]* ]] || {\n print -u2 \"vps-deploy: invalid target identifier\"\n exit 2\n }\nfi\n# Until the pinned checkout is ready, the current inventory is only a\n# best-effort restore-order hint for services shed during preflight. These arrays\n# are refreshed from VPS_DEPLOY_SHA before target validation or deployment.\ndeploy_target_kind=unknown\ntypeset -a DEPLOY_ALL DEPLOY_ALL_DEGRADED DEPLOY_RAILS\nDEPLOY_ALL=(${(@f)\"$(inventory_names all 2>/dev/null || true)\"})\nDEPLOY_ALL_DEGRADED=(${(@f)\"$(inventory_names all 1 2>/dev/null || true)\"})\nDEPLOY_RAILS=(${(@f)\"$(inventory_names rails 2>/dev/null || true)\"})\n\n# Environment values below participate in shell arithmetic and bounded waits.\n# Validate their decimal spelling and limits before acquiring a lock or touching\n# a service. In particular, a zero poll interval otherwise creates an endless\n# wait_up loop during a failed-deploy restore.\nvalidate_deploy_setting() {\n local name=$1 value=$2 minimum=$3 maximum=$4\n [[ $value == <-> && ( $value == 0 || $value != 0* ) ]] || {\n print -u2 \"vps-deploy: invalid $name; expected a decimal integer\"\n return 2\n }\n (( ${#value} <= 10 )) || {\n print -u2 \"vps-deploy: invalid $name; value is out of range\"\n return 2\n }\n (( value >= minimum && value <= maximum )) || {\n print -u2 \"vps-deploy: invalid $name; allowed range is $minimum..$maximum\"\n return 2\n }\n}\n\nvalidate_deploy_setting VPS_DEPLOY_LOCK_TTL \"${VPS_DEPLOY_LOCK_TTL:-7200}\" 1 604800 || exit 2\nvalidate_deploy_setting VPS_DEPLOY_RESTORE_WAIT \"${VPS_DEPLOY_RESTORE_WAIT:-240}\" 1 3600 || exit 2\nvalidate_deploy_setting VPS_DEPLOY_POLL \"${VPS_DEPLOY_POLL:-3}\" 1 60 || exit 2\nvalidate_deploy_setting VPS_DEPLOY_SETTLE_WAIT \"${VPS_DEPLOY_SETTLE_WAIT:-240}\" 1 3600 || exit 2\nvalidate_deploy_setting VPS_DEPLOY_MASTER_MIN_AVAIL \"${VPS_DEPLOY_MASTER_MIN_AVAIL:-10}\" 0 100 || exit 2\nvalidate_deploy_setting VPS_DEPLOY_MIN_FREE_KB \"${VPS_DEPLOY_MIN_FREE_KB:-1048576}\" 0 1000000000 || exit 2\n: \"${PUB5_ADMIN:=/usr/local/sbin/pub5-admin}\"\n[[ -f \"$PUB5_ADMIN\" && -x \"$PUB5_ADMIN\" && ! -L \"$PUB5_ADMIN\" ]] || {\n print -u2 \"vps-deploy: pub5-admin missing; run reviewed OPENBSD/OPERATOR.sh first\"\n exit 1\n}\n\n# One host, one deploy. The parent all/rails process keeps this lock while its\n# re-entrant child processes deploy each app in order; a second top-level deploy\n# is refused rather than running two pulls, precompiles and restarts together.\nlock_dir=${VPS_DEPLOY_LOCK_DIR:-$DEPLOY_HOME/.pub5-vps-deploy.lock}\nlock_ttl=${VPS_DEPLOY_LOCK_TTL:-7200}\nlock_owner=0\n\n# The lock is a directory, because mkdir is the one atomic test-and-set the\n# base system gives a shell, and the directory holds an `owner` file: pid, host\n# and epoch seconds. A bare directory could not tell a running deploy from the\n# remains of one killed by a dropped session or a reboot, and an operator had to\n# guess before removing it. Now a lock is reaped here, with the reason printed,\n# when its same-host owner PID is gone. Age alone never steals a lock from a\n# live local deployment: a slow or stalled deploy requires operator inspection,\n# not a second deploy running beside it. A remote-host or unreadable lock may\n# be reaped only after the recorded TTL, because this checkout cannot inspect\n# the other host's process.\n# BSD date formats an epoch with -r; GNU date reads -r as a file and needs\n# -d @epoch. The reaper prints the owner's start time on both dev hosts and vm23.\nepoch_date() {\n date -r \"$1\" '+%Y-%m-%d %H:%M:%S' 2>/dev/null || date -d \"@$1\" '+%Y-%m-%d %H:%M:%S' 2>/dev/null\n}\n\n# BSD stat formats with -f; GNU stat reads -f as filesystem status and needs\n# -c. The reaper pins the observed directory by device and inode on both hosts.\nfile_identity() {\n local out\n out=$(stat -f '%d:%i' \"$1\" 2>/dev/null)\n if [[ $out == <->:* ]]; then\n print -r -- $out\n return 0\n fi\n stat -c '%d:%i' \"$1\" 2>/dev/null\n}\n\n# A PID can be reused after its original deploy exits. Pair it with the\n# process start time and a checksum of its command line, so an unrelated process\n# cannot keep an abandoned lock alive forever. OpenBSD ps(1) exposes lstart and\n# command; if either cannot be read, callers fail safe and keep the lock.\nlock_process_identity() {\n local pid=$1 started command digest\n started=$(LC_ALL=C ps -p \"$pid\" -o lstart= 2>/dev/null) || return 1\n command=$(ps -ww -p \"$pid\" -o command= 2>/dev/null) || return 1\n [[ -n $started && -n $command ]] || return 1\n started=${started//[^A-Za-z0-9]/_}\n digest=$(print -rn -- \"$command\" | cksum) || return 1\n digest=${digest// /_}\n print -r -- \"${started}_${digest}\"\n}\n\nlock_read_owner() {\n lock_pid= lock_host= lock_epoch= lock_identity=\n [[ -r $lock_dir/owner ]] || return 1\n read -r lock_pid lock_host lock_epoch lock_identity < \"$lock_dir/owner\" || true\n [[ $lock_pid == <-> && $lock_epoch == <-> && $lock_identity != *[^A-Za-z0-9_]* ]]\n}\n\nlock_stale_reason() {\n local now=$(date +%s) current_identity\n if lock_read_owner; then\n if [[ $lock_host == $(hostname -s) ]]; then\n if ! ps -p \"$lock_pid\" >/dev/null 2>&1; then\n print -r -- \"owner pid $lock_pid is not running on $lock_host\"\n return 0\n fi\n if [[ -n $lock_identity ]]; then\n current_identity=$(lock_process_identity \"$lock_pid\") || {\n # A visible PID whose full identity cannot be measured is ambiguous,\n # not stale. Never steal its lock based on a partial observation.\n return 1\n }\n if [[ $current_identity != $lock_identity ]]; then\n print -r -- \"owner pid $lock_pid process identity changed on $lock_host (PID reused)\"\n return 0\n fi\n fi\n # Legacy three-field owners have no fingerprint: keep them protected while\n # their PID exists. New owners are distinguished even after PID reuse.\n # A live same-host owner is authoritative even after the TTL.\n return 1\n fi\n if (( now - lock_epoch > lock_ttl )); then\n print -r -- \"held $(( (now - lock_epoch) / 60 )) minutes by pid $lock_pid on $lock_host, past the ${lock_ttl}s TTL\"\n return 0\n fi\n return 1\n fi\n # mkdir has run but the owner file is not written yet, or never was. Judge\n # such a lock by the directory's own age so a deploy mid-start is left alone.\n local -a aged\n aged=( ${lock_dir}(N/mm+$(( lock_ttl / 60 ))) )\n (( $#aged )) || return 1\n print -r -- \"no readable owner and older than the ${lock_ttl}s TTL\"\n}\n\nrelease_deploy_lock() {\n (( lock_owner )) || return 0\n rm -f -- \"$lock_dir/owner\"\n rmdir \"$lock_dir\" 2>/dev/null || true\n}\n\nreap_stale_lock() {\n local reason before_identity after_identity retired_lock\n\n # Do not enter recovery for a live lock. The second check below closes the\n # race between this observation and claiming the recovery marker.\n reason=$(lock_stale_reason) || return 1\n before_identity=$(file_identity \"$lock_dir\") || return 1\n\n # The marker is inside the exact directory observed above, so mkdir is an\n # atomic claim. Inode identity catches a stale directory being replaced by a\n # fresh lock between the first observation and this claim.\n if ! mkdir \"$lock_dir/reaping\" 2>/dev/null; then\n return 1\n fi\n after_identity=$(file_identity \"$lock_dir\") || {\n rmdir \"$lock_dir/reaping\" 2>/dev/null || true\n return 1\n }\n if [[ $after_identity != $before_identity ]]; then\n rmdir \"$lock_dir/reaping\" 2>/dev/null || true\n return 1\n fi\n\n # Creating the marker updates the parent directory's mtime. Recheck owners,\n # but retain the pre-marker age decision for a directory that was already\n # ownerless; no other process can write its owner file while it exists.\n if [[ -r \"$lock_dir/owner\" ]]; then\n reason=$(lock_stale_reason) || {\n rmdir \"$lock_dir/reaping\" 2>/dev/null || true\n return 1\n }\n fi\n\n retired_lock=\"${lock_dir}.stale.$$\"\n if [[ -e \"$retired_lock\" ]]; then\n print -u2 \"vps-deploy: stale-lock quarantine exists at $retired_lock\"\n rmdir \"$lock_dir/reaping\" 2>/dev/null || true\n return 1\n fi\n\n print -u2 \"vps-deploy: reaping stale lock $lock_dir - $reason\"\n # Rename the whole claimed directory, marker included. The old lock path\n # becomes free atomically; cleanup touches only this unique retired path.\n if ! mv \"$lock_dir\" \"$retired_lock\" 2>/dev/null; then\n rmdir \"$lock_dir/reaping\" 2>/dev/null || true\n return 1\n fi\n rm -f -- \"$retired_lock/owner\"\n # A process killed before the atomic owner rename can leave only this temp.\n # Remove it with the quarantined lock so crash recovery does not leak dirs.\n rm -f -- \"$retired_lock\"/owner.tmp.*(N)\n rmdir \"$retired_lock/reaping\" 2>/dev/null || true\n rmdir \"$retired_lock\" 2>/dev/null || true\n return 0\n}\n\nacquire_deploy_lock() {\n local -i attempt\n for attempt in 1 2; do\n if mkdir \"$lock_dir\" 2>/dev/null; then\n local identity owner_tmp\n identity=$(lock_process_identity \"${$}\") || {\n print -u2 \"vps-deploy: cannot identify the lock owner process; refusing an unverifiable deploy\"\n rmdir \"$lock_dir\" 2>/dev/null || true\n exit 1\n }\n owner_tmp=\"$lock_dir/owner.tmp.${$}\"\n if ! print -r -- \"${$} $(hostname -s) $(date +%s) $identity\" > \"$owner_tmp\" ||\n ! mv \"$owner_tmp\" \"$lock_dir/owner\"; then\n print -u2 \"vps-deploy: cannot write the deploy lock owner record\"\n rm -f -- \"$owner_tmp\"\n rmdir \"$lock_dir\" 2>/dev/null || true\n exit 1\n fi\n export VPS_DEPLOY_LOCK_HELD=1\n export VPS_DEPLOY_LOCK_PID=${$}\n export VPS_DEPLOY_LOCK_IDENTITY=$identity\n lock_owner=1\n return 0\n fi\n if reap_stale_lock; then\n continue\n fi\n break\n done\n if lock_read_owner; then\n print -u2 \"vps-deploy: refused - another deploy owns $lock_dir (pid $lock_pid on $lock_host since $(epoch_date $lock_epoch))\"\n else\n print -u2 \"vps-deploy: refused - another deploy owns $lock_dir\"\n fi\n exit 75\n}\n\n# Re-entrant children must prove the inherited lock belongs to their live\n# parent. An environment flag alone must never bypass top-level serialization.\nif [[ -n ${VPS_DEPLOY_LOCK_HELD:-} ]]; then\n lock_read_owner || {\n print -u2 \"vps-deploy: refused - child invocation has no readable parent lock owner\"\n exit 75\n }\n [[ $lock_host == \"$(hostname -s)\" &&\n $lock_pid == ${VPS_DEPLOY_LOCK_PID:-} &&\n $lock_identity == ${VPS_DEPLOY_LOCK_IDENTITY:-} ]] || {\n print -u2 \"vps-deploy: refused - child invocation does not match the parent lock record\"\n exit 75\n }\n ps -p \"$lock_pid\" >/dev/null 2>&1 || {\n print -u2 \"vps-deploy: refused - parent deploy process is no longer running\"\n exit 75\n }\n current_identity=$(lock_process_identity \"$lock_pid\") || {\n print -u2 \"vps-deploy: refused - cannot verify the parent deploy process\"\n exit 75\n }\n [[ $current_identity == \"$lock_identity\" ]] || {\n print -u2 \"vps-deploy: refused - parent deploy PID identity changed\"\n exit 75\n }\nelse\n acquire_deploy_lock\nfi\n\n# Durable deploy logging.\n#\n# A full deploy outlives the ssh session that starts it. When that session is\n# cut the only surviving record is the log file. We keep:\n# - a rolling aggregate log (VPS_DEPLOY_LOG / /var/log/pub5/vps-deploy.log)\n# - a per-run log under /var/log/pub5/runs/ with the pid and app in the name\n# - a stable symlink at $DEPLOY_HOME/vps-deploy.log for operators who still\n# tail the home path\n# Once per run, not once per app: each group re-invokes one child per app.\n# VPS_DEPLOY_LOGGING is inherited so child processes do not double-tee.\ndeploy_log_dir=${VPS_DEPLOY_LOG_DIR:-/var/log/pub5}\ndeploy_log=${VPS_DEPLOY_LOG:-$deploy_log_dir/vps-deploy.log}\ndeploy_run_log=${VPS_DEPLOY_RUN_LOG:-}\n\nlog_ts() { date -u '+%Y-%m-%dT%H:%M:%SZ'; }\n\n# Structured line: timestamp host pid app stage message. Always goes to stdout\n# (which is already teed into the log files).\nlog() {\n print -r -- \"$(log_ts) host=$(hostname -s) pid=$$ app=${app:-?} stage=${deploy_stage:-?} $*\"\n}\n\n# The fleet receipt is written only through the fixed root broker. Ruby's JSON\n# encoder keeps service names, paths and metadata out of shell-generated JSON.\nwrite_deploy_run_receipt() {\n [[ -n ${VPS_DEPLOY_RUN_ID:-} ]] || return 0\n local payload\n payload=$(\"$RUBY\" -rjson -e '\n run_id, target, host, pid, owner_identity, status, recovery_state, stage,\n started_at, pinned_sha, previous_sha, services, ports_csv, order_csv,\n run_log, aggregate_log, snapshot_recorded = ARGV\n ports = {}\n ports_csv.split(\",\").reject(&:empty?).each do |pair|\n name, value = pair.split(\"=\", 2)\n abort \"invalid port pair\" unless name && value && value.match?(/\\A[0-9]+\\z/)\n ports[name] = Integer(value, 10)\n end\n record = {\n schema: 1, run_id: run_id, target: target, host: host, pid: Integer(pid, 10),\n owner_identity: owner_identity, status: status, recovery_state: recovery_state,\n stage: stage, started_at: started_at,\n pinned_sha: pinned_sha.empty? ? nil : pinned_sha,\n previous_sha: previous_sha.empty? ? nil : previous_sha,\n started_services: services.empty? ? [] : services.split(\" \"),\n ports: ports,\n restore_order: order_csv.empty? ? [] : order_csv.split(\",\"),\n run_log: run_log, aggregate_log: aggregate_log,\n snapshot_recorded: snapshot_recorded == \"1\"\n }\n puts JSON.generate(record)\n ' \"${VPS_DEPLOY_RUN_ID}\" \"${VPS_DEPLOY_RUN_TARGET}\" \"$(hostname -s)\" \\\n \"${VPS_DEPLOY_LOCK_PID}\" \"${VPS_DEPLOY_LOCK_IDENTITY}\" \\\n \"${VPS_DEPLOY_RECEIPT_STATUS:-running}\" \"${VPS_DEPLOY_RECEIPT_RECOVERY:-none}\" \\\n \"${deploy_stage:-start}\" \"${VPS_DEPLOY_RUN_STARTED_AT}\" \\\n \"${VPS_DEPLOY_RECEIPT_PINNED_SHA:-}\" \"${VPS_DEPLOY_RECEIPT_PREVIOUS_SHA:-}\" \\\n \"${VPS_DEPLOY_RECEIPT_SERVICES:-}\" \"${VPS_DEPLOY_RECEIPT_PORTS:-}\" \\\n \"${VPS_DEPLOY_RECEIPT_ORDER:-}\" \\\n \"${VPS_DEPLOY_RUN_LOG}\" \"${VPS_DEPLOY_LOG}\" \\\n \"${VPS_DEPLOY_RECEIPT_SNAPSHOT_RECORDED:-0}\") || return 1\n \"$PUB5_ADMIN\" deploy-run-write \"$VPS_DEPLOY_RUN_ID\" \"$payload\" >/dev/null\n}\n\n# Stage transitions persist the latest recovery breadcrumb before the next\n# stage may stop, replace, migrate or restart a production service.\nset_stage() {\n deploy_stage=$1\n log \"enter\"\n write_deploy_run_receipt || {\n print -u2 \"vps-deploy: failed to persist recovery receipt at stage $deploy_stage\"\n return 1\n }\n}\n# Retry a command with exponential backoff. Use only for transient failures\n# (network, service boot races). Do not wrap git merge conflicts or policy\n# refusals. Logs each attempt. Returns the last exit status.\n#\n# retry 3 2 curl -fsS http://127.0.0.1:port/up\n# attempts=3 initial_delay=2s, doubles each time, capped at 30s\nretry() {\n local max=${1:?} delay=${2:?}\n shift 2\n local attempt=1 rc=0\n while (( attempt <= max )); do\n if \"$@\"; then\n (( attempt > 1 )) && log \"retry ok attempt=$attempt cmd=$*\"\n return 0\n fi\n rc=$?\n if (( attempt == max )); then\n log \"retry exhausted attempts=$max rc=$rc cmd=$*\"\n return $rc\n fi\n log \"retry fail attempt=$attempt/$max rc=$rc sleep=${delay}s cmd=$*\"\n sleep $delay\n delay=$(( delay * 2 ))\n (( delay > 30 )) && delay=30\n attempt=$(( attempt + 1 ))\n done\n return $rc\n}\n\n# admin_service with retry for start/restart only (not check/list).\nadmin_service_retry() {\n local action=$1\n shift || true\n case $action in\n start|restart)\n retry \"${VPS_DEPLOY_SERVICE_RETRIES:-3}\" \"${VPS_DEPLOY_SERVICE_RETRY_DELAY:-2}\" \\\n admin_service \"$action\" \"$@\"\n ;;\n *)\n admin_service \"$action\" \"$@\"\n ;;\n esac\n}\n\nrotate_deploy_log() {\n # Safety net only. Canonical rotation is newsyslog (OPENBSD/mirror/etc/newsyslog.conf).\n # This fires if the log grew past the size limit between newsyslog runs.\n local max_bytes=${VPS_DEPLOY_LOG_MAX_BYTES:-10485760}\n [[ -f $deploy_log ]] || return 0\n local size\n size=$(stat -f %z \"$deploy_log\" 2>/dev/null) || return 0\n (( size >= max_bytes )) || return 0\n log \"rotate aggregate log size=$size max=$max_bytes\"\n mv -f -- \"$deploy_log\" \"$deploy_log.1\" 2>/dev/null || true\n}\n\nif [[ -z ${VPS_DEPLOY_LOGGING:-} ]]; then\n export VPS_DEPLOY_LOGGING=1\n mkdir -p -- \"$deploy_log_dir/runs\" 2>/dev/null || {\n # Fall back to the deploy user's home when /var/log/pub5 is not writable\n # yet (broker/OPERATOR not installed). Still durable across SSH drops.\n deploy_log_dir=$DEPLOY_HOME\n deploy_log=${VPS_DEPLOY_LOG:-$DEPLOY_HOME/vps-deploy.log}\n mkdir -p -- \"$deploy_log_dir/runs\" 2>/dev/null || true\n }\n rotate_deploy_log\n if [[ -z $deploy_run_log ]]; then\n deploy_run_log=\"$deploy_log_dir/runs/deploy-$(date -u '+%Y%m%d-%H%M%S')-${app}-$$.log\"\n fi\n export VPS_DEPLOY_LOG=$deploy_log\n export VPS_DEPLOY_RUN_LOG=$deploy_run_log\n export VPS_DEPLOY_LOG_DIR=$deploy_log_dir\n # Stable path operators already know about.\n if [[ -d $DEPLOY_HOME ]]; then\n ln -sfn \"$deploy_log\" \"$DEPLOY_HOME/vps-deploy.log\" 2>/dev/null || true\n ln -sfn \"$deploy_run_log\" \"$DEPLOY_HOME/vps-deploy-latest.log\" 2>/dev/null || true\n fi\n # Recovery depends on the per-run log retaining the pinned SHA, initial\n # service snapshot and last entered stage. Do not begin a deploy when that\n # durable record cannot be opened as a regular file.\n if ! : >> \"$deploy_run_log\" || [[ ! -f \"$deploy_run_log\" || -L \"$deploy_run_log\" ]]; then\n print -u2 \"vps-deploy: cannot create a regular per-run recovery log at $deploy_run_log; refusing before any service is touched\"\n exit 1\n fi\n if ! : >> \"$deploy_log\" || [[ ! -f \"$deploy_log\" || -L \"$deploy_log\" ]]; then\n print -u2 \"vps-deploy: cannot create a regular aggregate log at $deploy_log; refusing before any service is touched\"\n exit 1\n fi\n # Tee stdout/stderr into both the aggregate and the per-run log. process\n # substitution keeps a single stream so line order stays honest.\n exec > >(tee -a \"$deploy_log\" \"$deploy_run_log\") 2>&1\n set_stage start\n log \"start target=$app uid=$(id -u) user=$(id -un) log=$deploy_log run_log=$deploy_run_log\"\n log \"argv: $*\"\nfi\n\n# Transitional adapter: use the fixed root-owned broker once installed. The\n# fallback remains until the operator sync has installed it on vm23.\nadmin_service() {\n local action=${1:-}\n shift || true\n if [[ $action == list-started ]]; then\n \"$PUB5_ADMIN\" service list-started\n else\n \"$PUB5_ADMIN\" service \"$action\" \"$@\"\n fi\n}\n\nadmin_request() {\n [[ -x \"$PUB5_ADMIN\" ]] || {\n print -u2 \"vps-deploy: pub5-admin is not installed; run the reviewed OPERATOR sync first\"\n return 127\n }\n \"$PUB5_ADMIN\" \"$@\"\n}\n\n# The guard's deploy flag, held for the whole deploy and not only the restart.\n#\n# rc.d/ touches .deploying- from rc_pre to the end of its /up wait and\n# removes it there. CI, migrate, precompile and the checks all run outside that\n# window at the box's highest load, and resource_guard counts every one of those\n# ticks as a strike. So the counter already stands past GUARD_SHED_STRIKES when\n# the restart's flag comes off, and the next tick sheds the first OPTIONAL\n# service. bsdports leads that list, which is why its own deploy exits 0 and\n# vps-state then reads bsdports(failed) with :47312 closed.\n#\n# Taken after the preflight guard run, which may still shed to free memory for\n# CI. A tick that sees the flag resets the counter, so the guard needs two fresh\n# breaching ticks after this deploy ends before it sheds anything. The name is\n# not .deploying-, because rc.d removes that one mid-deploy; it is touched\n# again at each long stage because the guard ignores a flag older than 30 minutes.\ndeploy_flag=\"$repo/.deploying-vps-deploy-${app}\"\nhold_deploy_flag() { touch \"$deploy_flag\" 2>/dev/null || true }\n\n# The stage touches alone did not hold it: brgen's CI ran 34 minutes between two of\n# them on 2026-10-27, the flag aged out at 30, and the guard's emergency stop fired\n# mid-deploy. A keeper touches it every 4 minutes for as long as this process lives;\n# finish() stops it before removing the flag, and it exits on its own if we die.\nflag_keeper=\nkeep_deploy_flag() {\n hold_deploy_flag\n ( while kill -0 $$ 2>/dev/null; do touch \"$deploy_flag\" 2>/dev/null; sleep 240; done ) &!\n flag_keeper=$!\n}\n\n# The terminal result, recorded however the script ends.\n#\n# A failure past `rcctl stop` used to leave the app stopped under a stamp that\n# still said ok, and an ssh disconnect cut the tee pipe before the last line\n# reached the log. Every exit now lands here: it stamps failed (unless a stage\n# already did), names the stage and the services left down, prints the safe next\n# command, and appends the verdict straight to the log file rather than through\n# tee. Each step is `|| true`: a trap that fails would hide the failure it reports.\ndeploy_stage=start\napp_stopped=0\nstamp_written=0\nhalted_app=\npreflight_active=0\npreflight_back_to=\ntypeset -a preflight_back_flag own_stops\n\n# Services the deploy needs to put back, and how it waits for them.\n#\n# On 2026-10-12 a halted `all` pass left brgen, amber and bsdports down for minutes\n# until the operator typed rcctl start by hand: the restore sat at the end of a pass\n# that never got there. The snapshot of started services is taken before anything is\n# touched (VPS_DEPLOY_STARTED, exported so the children see it), and the exit trap\n# of the top-level process puts back whatever is missing, one service at a time.\n# Falcon needs 30-40 s to boot on this box, and two booting at once is what shed\n# the apps in the first place, so each service answers /up before the next starts.\ntypeset -A deploy_ports\nload_deploy_ports() {\n (( $#deploy_ports )) && return 0\n local line\n local -a rows\n rows=(${(f)\"$(\"$RUBY\" -r\"$repo/OPENBSD/lib/deploy_inventory\" \\\n -e 'Deploy::Inventory.new(root: ARGV[0]).apps.each { |a| puts \"\#{a.name} \#{a.port}\" }' \"$repo\" 2>/dev/null)\"})\n for line in $rows; do\n deploy_ports[${line%% *}]=${line##* }\n done\n return 0\n}\n\n# Bounded wait for http://127.0.0.1:PORT/up. Returns 1 when the limit passes.\nwait_up() {\n local port=$1 limit=${2:-${VPS_DEPLOY_RESTORE_WAIT:-240}} step=${VPS_DEPLOY_POLL:-3} waited=0\n while (( waited < limit )); do\n if curl -fsS --max-time 5 \"http://127.0.0.1:${port}/up\" >/dev/null 2>&1; then\n (( waited > 0 )) && log \"wait_up ok port=$port after=${waited}s\"\n return 0\n fi\n sleep $step\n waited=$(( waited + step ))\n done\n log \"wait_up timeout port=$port limit=${limit}s\"\n return 1\n}\n\n# Every service that was started and is running must answer /up before the next\n# memory-heavy step (CI, bundle, precompile). A service that is down is not booting\n# and is not waited for; one that never answers is reported, not waited on forever.\nsettle_boots() {\n local svc port\n load_deploy_ports\n for svc in ${=VPS_DEPLOY_STARTED:-}; do\n port=${deploy_ports[$svc]:-}\n [[ -n $port ]] || continue\n admin_service check \"$svc\" >/dev/null 2>&1 || continue\n wait_up \"$port\" \"${VPS_DEPLOY_SETTLE_WAIT:-240}\" \\\n || print -u2 \"vps-deploy: WARNING - $svc is running but :${port}/up did not answer within ${VPS_DEPLOY_SETTLE_WAIT:-240}s\"\n done\n return 0\n}\n\n# Who stopped it: this deploy, resource_guard (its shed ledger), or neither.\nstop_origin() {\n local svc=$1 shed_file=${VPS_DEPLOY_SHED_FILE:-/var/db/resource_guard_shed}\n if [[ $svc == $halted_app ]] || (( ${own_stops[(Ie)$svc]} )); then\n print \"stopped by this deploy\"\n elif [[ -r $shed_file ]] && (( ${${(f)\"$(<$shed_file)\"}[(Ie)$svc]} )); then\n print \"shed by resource_guard\"\n else\n print \"down for another reason (crashed or stopped outside the deploy)\"\n fi\n}\n\n# Starts, one at a time and in serving order, every service that was started when\n# the deploy began and is not running now. Returns 1 if any would not come back.\nrestore_started() {\n [[ -n ${VPS_DEPLOY_STARTED:-} ]] || return 0\n if [[ ${VPS_DEPLOY_RESTORE_PORTS_FROM_RECEIPT:-0} != 1 ]]; then\n load_deploy_ports\n fi\n local -a want order failed restored\n local svc port origin\n want=( ${=VPS_DEPLOY_STARTED} )\n for svc in $DEPLOY_ALL_DEGRADED; do (( ${want[(Ie)$svc]} )) && order+=($svc); done\n for svc in $want; do (( ${order[(Ie)$svc]} )) || order+=($svc); done\n for svc in $order; do\n admin_service check \"$svc\" >/dev/null 2>&1 && continue\n origin=$(stop_origin \"$svc\")\n print \"vps-deploy: restore $svc ($origin)\"\n port=${deploy_ports[$svc]:-}\n # restart, not start: rc.d/master runs its relayd restart, TTS warmup and flag\n # removal only on restart, and restart on a stopped service is a start.\n admin_service_retry restart \"$svc\" >/dev/null 2>&1 || true\n if [[ -n $port ]]; then\n wait_up \"$port\" || { admin_service_retry restart \"$svc\" >/dev/null 2>&1 || true; wait_up \"$port\"; } || {\n print -u2 \"vps-deploy: restore $svc FAILED - :${port}/up not ready, read /var/log/daemon\"\n failed+=($svc); continue\n }\n else\n sleep 3\n admin_service check \"$svc\" >/dev/null 2>&1 || { print -u2 \"vps-deploy: restore $svc FAILED - rcctl check\"; failed+=($svc); continue }\n fi\n print \"vps-deploy: restore $svc ok\"\n restored+=($svc)\n done\n (( $#restored )) && print \"vps-deploy: restored ${restored}\"\n (( $#failed )) && print -u2 \"vps-deploy: STILL DOWN after restore: ${failed} (/usr/local/sbin/pub5-admin service restart , one at a time)\"\n (( $#failed == 0 ))\n}\n\n# A signal or shell exit can arrive while the box is detached for preflight.\n# Keep the original ref as explicit recovery state until it is restored.\nrestore_preflight_checkout() {\n (( preflight_active )) || return 0\n [[ -n $preflight_back_to ]] || {\n print -u2 \"vps-deploy: preflight checkout recovery has no saved ref\"\n return 1\n }\n git_as_deploy_user switch -q \"${preflight_back_flag[@]}\" \"$preflight_back_to\" || {\n print -u2 \"vps-deploy: FAILED to restore checkout ref $preflight_back_to after preflight\"\n return 1\n }\n preflight_active=0\n log \"preflight checkout restored ref=$preflight_back_to\"\n}\n\n# Vendorables are not tracked in git (TREE_CONTRACTS.md, 8 November 2026):\n# master's smart-turn runtime bytes are fetched when absent, with pinned\n# versions and sha256-verified digests. Failure leaves ?smart_turn=1\n# unavailable — the feature is default-off — and the deploy continues.\nensure_smart_turn_assets() {\n local vendor=\"$repo/MASTER/web/public/vendor/onnxruntime\"\n local model=\"$repo/MASTER/web/public/models/smart-turn-v3.2-cpu.onnx\"\n [[ -f $vendor/ort-wasm-simd-threaded.wasm && -f $vendor/ort.wasm.min.mjs && -f $model ]] && return 0\n print \"vps-deploy: smart_turn assets absent; fetching pinned, checksummed copies\"\n if ! run_as_deploy_user \"cd $(printf '%q' \"$repo/MASTER/web\") && sh script/build_smart_turn.sh\"; then\n print -u2 \"vps-deploy: smart_turn fetch failed; ?smart_turn=1 stays unavailable (default-off) until MASTER/web/script/build_smart_turn.sh succeeds\"\n fi\n}\n\nfinish() {\n local rc=$?\n trap - EXIT\n if (( preflight_active )); then\n restore_preflight_checkout || {\n print -u2 \"vps-deploy: checkout remains in preflight recovery state; restore ref $preflight_back_to before another deploy\"\n (( rc == 0 )) && rc=1\n }\n fi\n if (( rc != 0 )) && [[ $deploy_target_kind == all || $deploy_target_kind == group ]]; then\n print -u2 \"vps-deploy: $app FAILED at stage \\\"$deploy_stage\\\" (exit $rc)\"\n fi\n if (( rc != 0 )) && [[ $deploy_target_kind != all && $deploy_target_kind != group && $deploy_target_kind != unknown ]]; then\n if (( ! stamp_written )) && (( $+functions[write_stamp] )); then\n write_stamp \"$app\" failed || true\n fi\n print -u2 \"vps-deploy: $app FAILED at stage \\\"$deploy_stage\\\" (exit $rc)\"\n if (( app_stopped )); then\n print -u2 \"vps-deploy: $app was stopped for this deploy and is DOWN - read /var/log/daemon, then\"\n print -u2 \" /usr/local/sbin/pub5-admin service start $app (or rerun: doas zsh OPENBSD/bin/vps-deploy $app)\"\n else\n print -u2 \"vps-deploy: $app was not stopped; the previous release is still serving. Fix the cause and rerun:\"\n print -u2 \" doas zsh OPENBSD/bin/vps-deploy $app\"\n fi\n local other\n for other in $DEPLOY_RAILS; do\n [[ $other == \"$app\" ]] && continue\n admin_service get \"$other\" >/dev/null 2>&1 || continue\n admin_service check \"$other\" >/dev/null 2>&1 \\\n || print -u2 \"vps-deploy: $other is DOWN (shed or stopped) - /usr/local/sbin/pub5-admin service restart $other\"\n done\n fi\n # Only the process that owns the lock restores: the children of `all` fail into\n # their parent, which knows the whole pass. Each restore result is printed.\n local restore_ok=1\n if (( rc != 0 )) && (( lock_owner )) && [[ ${VPS_DEPLOY_RESTORE_ATTEMPTED:-0} != 1 ]]; then\n restore_started || {\n restore_ok=0\n print -u2 \"vps-deploy: production is NOT fully restored - see the lines above\"\n }\n fi\n # A normal EXIT/TERM/HUP reaches this terminal update; SIGKILL/power loss\n # leaves the last durable receipt pending for an explicit operator recovery.\n if (( lock_owner )) && [[ -n ${VPS_DEPLOY_RUN_ID:-} ]]; then\n VPS_DEPLOY_RECEIPT_STATUS=failed\n (( rc == 0 )) && VPS_DEPLOY_RECEIPT_STATUS=ok\n if [[ -n ${VPS_DEPLOY_STARTED:-} ]]; then\n if (( rc == 0 || restore_ok )); then\n VPS_DEPLOY_RECEIPT_RECOVERY=complete\n else\n VPS_DEPLOY_RECEIPT_RECOVERY=pending\n fi\n else\n VPS_DEPLOY_RECEIPT_RECOVERY=none\n fi\n export VPS_DEPLOY_RECEIPT_STATUS VPS_DEPLOY_RECEIPT_RECOVERY\n write_deploy_run_receipt || {\n print -u2 \"vps-deploy: FAILED to persist terminal recovery receipt\"\n (( rc == 0 )) && rc=1\n }\n fi\n # Keep resource_guard excluded until every service restored by this failure\n # has answered, including slow boots; only then stop the heartbeat and remove\n # this process's deploy flag.\n [[ -n $flag_keeper ]] && kill $flag_keeper 2>/dev/null\n rm -f -- \"$deploy_flag\"\n log \"finished exit=$rc\"\n # Write the verdict straight to the files as well: if the tee pipe died with\n # the ssh session the last structured line can still land on disk.\n {\n print -r -- \"$(log_ts) host=$(hostname -s) pid=$$ app=$app stage=$deploy_stage finished exit=$rc\"\n } >> \"${deploy_log:-/var/log/pub5/vps-deploy.log}\" 2>/dev/null || true\n if [[ -n ${deploy_run_log:-${VPS_DEPLOY_RUN_LOG:-}} ]]; then\n print -r -- \"$(log_ts) host=$(hostname -s) pid=$$ app=$app stage=$deploy_stage finished exit=$rc\" \\\n >> \"${deploy_run_log:-$VPS_DEPLOY_RUN_LOG}\" 2>/dev/null || true\n fi\n release_deploy_lock\n exit $rc\n}\n# zsh runs the EXIT trap for HUP but not for TERM or INT, so name all three: a\n# dropped ssh session or a kill is a failure with a recorded stage, not a silent end.\ntrap finish EXIT\ntrap 'exit 129' HUP\ntrap 'exit 130' INT\ntrap 'exit 143' TERM\n\n# One durable receipt belongs to the lock-owning parent. Child app deploys\n# inherit the ID, snapshot and owner identity and update only the current stage.\nif (( lock_owner )) && [[ $app != recover ]]; then\n unset VPS_DEPLOY_RUN_ID VPS_DEPLOY_RUN_TARGET VPS_DEPLOY_RUN_STARTED_AT\n unset VPS_DEPLOY_STARTED\n unset VPS_DEPLOY_RECEIPT_STATUS VPS_DEPLOY_RECEIPT_RECOVERY\n unset VPS_DEPLOY_RECEIPT_PINNED_SHA VPS_DEPLOY_RECEIPT_PREVIOUS_SHA\n unset VPS_DEPLOY_RECEIPT_SERVICES VPS_DEPLOY_RECEIPT_PORTS VPS_DEPLOY_RECEIPT_ORDER\n unset VPS_DEPLOY_RECEIPT_SNAPSHOT_RECORDED\n VPS_DEPLOY_RUN_ID=\"$(date -u '+%Y%m%dT%H%M%SZ')-$app-${$}\"\n VPS_DEPLOY_RUN_TARGET=$app\n VPS_DEPLOY_RUN_STARTED_AT=\"$(date -u '+%Y-%m-%dT%H:%M:%SZ')\"\n VPS_DEPLOY_RECEIPT_STATUS=running\n VPS_DEPLOY_RECEIPT_RECOVERY=none\n VPS_DEPLOY_RECEIPT_PINNED_SHA=\n VPS_DEPLOY_RECEIPT_PREVIOUS_SHA=\n VPS_DEPLOY_RECEIPT_SERVICES=\n VPS_DEPLOY_RECEIPT_PORTS=\n VPS_DEPLOY_RECEIPT_ORDER=\n VPS_DEPLOY_RECEIPT_SNAPSHOT_RECORDED=0\n export VPS_DEPLOY_RECEIPT_SNAPSHOT_RECORDED\n export VPS_DEPLOY_RUN_ID VPS_DEPLOY_RUN_TARGET VPS_DEPLOY_RUN_STARTED_AT\n export VPS_DEPLOY_RECEIPT_STATUS VPS_DEPLOY_RECEIPT_RECOVERY\n export VPS_DEPLOY_RECEIPT_PINNED_SHA VPS_DEPLOY_RECEIPT_PREVIOUS_SHA\n export VPS_DEPLOY_RECEIPT_SERVICES VPS_DEPLOY_RECEIPT_PORTS VPS_DEPLOY_RECEIPT_ORDER\n write_deploy_run_receipt || {\n print -u2 \"vps-deploy: cannot create the durable recovery receipt; refusing before any service is touched\"\n exit 1\n }\nfi\n\n# A parent can be killed while a child app deploy is still unwinding. The\n# receipt identifies the parent, so also refuse while another deployer process\n# is visible. Exclude this recovery command by its own PID.\nother_deploy_processes() {\n local pid exclude_pid=${1:-} matches rc\n local -a candidates\n if matches=$(pgrep -f 'OPENBSD/bin/vps-deploy' 2>/dev/null); then\n :\n else\n rc=$?\n (( rc == 1 )) || return 1\n fi\n candidates=(${(f)matches})\n for pid in $candidates; do\n [[ $pid == \"${$}\" || ( -n $exclude_pid && $pid == \"$exclude_pid\" ) ]] || print -r -- \"$pid\"\n done\n}\n\n# Operator-driven recovery is intentionally separate from deploy. It never pulls,\n# checks out code, runs migrations or replays a deploy; it restores only the\n# service set recorded as running before the interrupted run.\nif [[ $app == recover ]]; then\n runs_json=$(admin_request deploy-run-list) || {\n print -u2 \"vps-deploy: cannot list recovery receipts; install and verify the pub5-admin mirror first\"\n exit 1\n }\n if [[ $recover_mode == plan ]]; then\n report=$(\"$RUBY\" -rjson -e '\n rows = JSON.parse(STDIN.read)\n rows.select { |r| r[\"recovery_state\"] == \"pending\" && %w[running failed].include?(r[\"status\"]) }.each do |r|\n puts [r[\"run_id\"], r[\"target\"], r[\"pinned_sha\"] || \"not-pinned\",\n r[\"stage\"], r[\"status\"], r[\"pid\"], r[\"owner_identity\"],\n r[\"host\"], r[\"snapshot_recorded\"], r[\"run_log\"]].join(\"|\")\n end\n ' <<< \"$runs_json\") || exit 1\n if [[ -z $report ]]; then\n print \"vps-deploy: no pending interrupted-deploy recovery receipts\"\n exit 0\n fi\n while IFS='|' read -r run_id target sha stage status owner_pid owner_identity owner_host snapshot_recorded run_log; do\n [[ -n $run_id ]] || continue\n if [[ $owner_host != \"$(hostname -s)\" ]]; then\n print \"vps-deploy: FOREIGN $run_id host=$owner_host target=$target - recover on the recorded host\"\n continue\n fi\n if [[ $snapshot_recorded != true ]]; then\n print \"vps-deploy: INCOMPLETE $run_id target=$target stage=$stage - service snapshot is not complete; do not apply\"\n continue\n fi\n other_pids=$(other_deploy_processes \"$owner_pid\") || {\n print \"vps-deploy: AMBIGUOUS $run_id - cannot verify the process table; do not recover\"\n continue\n }\n if [[ -n $other_pids ]]; then\n print \"vps-deploy: AMBIGUOUS $run_id - another deployer process is still present; do not recover\"\n print -r -- \" other deployer PID(s): $other_pids\"\n continue\n fi\n if ps -p \"$owner_pid\" >/dev/null 2>&1; then\n current_identity=$(lock_process_identity \"$owner_pid\") || {\n print \"vps-deploy: AMBIGUOUS $run_id - owner PID exists but its identity cannot be checked\"\n continue\n }\n if [[ $current_identity == \"$owner_identity\" ]]; then\n print \"vps-deploy: LIVE $run_id target=$target sha=$sha stage=$stage pid=$owner_pid - do not recover\"\n continue\n fi\n fi\n print \"vps-deploy: RECOVERY AVAILABLE $run_id target=$target sha=$sha stage=$stage\"\n print \" log=$run_log\"\n done <<< \"$report\"\n print \"vps-deploy: inspect the receipt and its log before: zsh OPENBSD/bin/vps-deploy recover --apply \"\n exit 0\n fi\n\n receipt_json=$(admin_request deploy-run-read \"$recover_id\") || exit 1\n [[ $receipt_json != null ]] || {\n print -u2 \"vps-deploy: no recovery receipt found for $recover_id\"\n exit 1\n }\n details=$(\"$RUBY\" -rjson -e '\n r = JSON.parse(STDIN.read)\n abort \"receipt is not pending recovery\" unless\n r[\"recovery_state\"] == \"pending\" && %w[running failed].include?(r[\"status\"])\n abort \"receipt snapshot is incomplete\" unless r[\"snapshot_recorded\"] == true\n puts [\"META\", r.fetch(\"pid\"), r.fetch(\"owner_identity\"), r.fetch(\"stage\"),\n r.fetch(\"target\"), r[\"pinned_sha\"] || \"not-pinned\", r.fetch(\"host\")].join(\"|\")\n puts [\"SERVICES\", r.fetch(\"started_services\").join(\" \")].join(\"|\")\n puts [\"ORDER\", r.fetch(\"restore_order\").join(\" \")].join(\"|\")\n r.fetch(\"ports\").sort.each { |service, port| puts [\"PORT\", service, port].join(\"|\") }\n ' <<< \"$receipt_json\") || {\n print -u2 \"vps-deploy: receipt is corrupt, incomplete or not eligible for recovery\"\n exit 1\n }\n\n typeset -a recovery_lines\n recovery_lines=(${(f)details})\n IFS='|' read -r kind owner_pid owner_identity original_stage original_target original_sha owner_host <<< \"$recovery_lines[1]\"\n [[ $owner_host == \"$(hostname -s)\" ]] || {\n print -u2 \"vps-deploy: receipt belongs to $owner_host; recovery must run on that host\"\n exit 1\n }\n other_pids=$(other_deploy_processes \"$owner_pid\") || {\n print -u2 \"vps-deploy: cannot verify the process table; refusing recovery\"\n exit 75\n }\n [[ -z $other_pids ]] || {\n print -u2 \"vps-deploy: another deployer process is still present; refusing recovery\"\n print -u2 \" other deployer PID(s): $other_pids\"\n exit 75\n }\n if ps -p \"$owner_pid\" >/dev/null 2>&1; then\n current_identity=$(lock_process_identity \"$owner_pid\") || {\n print -u2 \"vps-deploy: owner PID exists but its identity cannot be checked; refusing recovery\"\n exit 75\n }\n if [[ $current_identity == \"$owner_identity\" ]]; then\n print -u2 \"vps-deploy: original deploy $recover_id is still running; refusing recovery\"\n exit 75\n fi\n fi\n\n VPS_DEPLOY_STARTED=\n DEPLOY_ALL_DEGRADED=()\n deploy_ports=()\n for ((i=2; i<=${#recovery_lines}; i++)); do\n IFS='|' read -r kind one two <<< \"$recovery_lines[$i]\"\n case $kind in\n SERVICES) VPS_DEPLOY_STARTED=$one ;;\n ORDER) DEPLOY_ALL_DEGRADED=(${=one}) ;;\n PORT) deploy_ports[$one]=$two ;;\n esac\n done\n [[ -n $VPS_DEPLOY_STARTED ]] || {\n print -u2 \"vps-deploy: receipt has no captured running services; refusing an inferred restore\"\n exit 1\n }\n export VPS_DEPLOY_STARTED\n export VPS_DEPLOY_RESTORE_PORTS_FROM_RECEIPT=1\n VPS_DEPLOY_RESTORE_ATTEMPTED=1\n\n print \"vps-deploy: recovery target=$original_target pinned=$original_sha failed_stage=$original_stage\"\n print \"vps-deploy: only the recorded service snapshot will be restored\"\n keep_deploy_flag\n recovery_payload=$(\"$RUBY\" -rjson -e 'puts JSON.generate(status: \"failed\", recovery_state: \"pending\", stage: \"recovery-restore\")')\n admin_request deploy-run-write \"$recover_id\" \"$recovery_payload\" || {\n print -u2 \"vps-deploy: cannot persist recovery-attempt stage; no services were changed\"\n exit 1\n }\n\n recovery_failed=0\n restore_started || recovery_failed=1\n if (( ! recovery_failed )); then\n restore_wait=${VPS_DEPLOY_RESTORE_WAIT:-240}\n for svc in ${=VPS_DEPLOY_STARTED}; do\n admin_service check \"$svc\" >/dev/null 2>&1 || {\n print -u2 \"vps-deploy: recovery verification failed for $svc (rcctl check)\"\n recovery_failed=1\n break\n }\n port=${deploy_ports[$svc]:-}\n if [[ -n $port ]] && ! wait_up \"$port\" \"$restore_wait\"; then\n print -u2 \"vps-deploy: recovery verification failed for $svc /up on port $port\"\n recovery_failed=1\n break\n fi\n done\n fi\n\n if (( recovery_failed )); then\n recovery_payload=$(\"$RUBY\" -rjson -e 'puts JSON.generate(status: \"failed\", recovery_state: \"pending\", stage: \"recovery-failed\")')\n admin_request deploy-run-write \"$recover_id\" \"$recovery_payload\" || \\\n print -u2 \"vps-deploy: recovery failed and the pending receipt could not be updated\"\n print -u2 \"vps-deploy: recovery remains pending; inspect /var/log/daemon before another attempt\"\n exit 1\n fi\n\n recovery_payload=$(\"$RUBY\" -rjson -e 'puts JSON.generate(status: \"recovered\", recovery_state: \"complete\", stage: \"recovered\", recovered_at: Time.now.utc.iso8601)')\n admin_request deploy-run-write \"$recover_id\" \"$recovery_payload\" || {\n print -u2 \"vps-deploy: services answered, but the recovery receipt could not be finalized\"\n exit 1\n }\n print \"vps-deploy: recovered $recover_id target=$original_target pinned=$original_sha (previous stage=$original_stage)\"\n exit 0\nfi\n\n# Pin the commit, prove it boots, record what is running.\n#\n# Done once, by the process that owns the lock, before any service is touched.\n#\n# The deploy used to pull origin/main at whatever instant each step reached its\n# pull, so a commit pushed mid-pass was deployed half-way through it. On 2026-10-12\n# the pulled tip carried a SyntaxError and later a law that raised at boot; master's\n# container died on the box and the pass halted with production already shed. Now\n# the target is one SHA, resolved here (VPS_DEPLOY_SHA, or origin/main fetched\n# once), and every later step checks HEAD against it instead of pulling.\n#\n# The checks (MASTER/lib/operator/canary.rb) ran on the operator's\n# machine when `bin/operator vps deploy --remote` started this run and said so with\n# VPS_DEPLOY_PREFLIGHT_SHA; a direct run on the box has no such word and runs the\n# MASTER-only subset itself on the pinned commit, detached, so a refusal leaves the\n# checkout where it was. VPS_DEPLOY_SKIP_PREFLIGHT=1 is break-glass and, like the\n# runtime-check skip, needs the human-only I_UNDERSTAND_FAST_DEPLOY=1.\ndeploy_refuse() { log \"refused: $1\"; print -u2 \"vps-deploy: refused - $1\"; print -u2 \" Nothing was stopped.\"; exit 1 }\nif (( lock_owner )); then\n set_stage pin\n retry \"${VPS_DEPLOY_GIT_RETRIES:-3}\" 2 git_as_deploy_user fetch -q origin main \\\n || deploy_refuse \"git fetch origin main failed (as $DEPLOY_USER) after retries\"\n pin=$(git_as_deploy_user rev-parse --verify -q \"${VPS_DEPLOY_SHA:-FETCH_HEAD}^{commit}\") \\\n || deploy_refuse \"${VPS_DEPLOY_SHA:-origin/main} is not a commit this checkout can see\"\n head_before=$(git_as_deploy_user rev-parse HEAD)\n git_as_deploy_user merge-base --is-ancestor \"$head_before\" \"$pin\" \\\n || deploy_refuse \"$pin is not a fast-forward of the box's HEAD $head_before\"\n export VPS_DEPLOY_SHA=$pin\n print \"vps-deploy: pinned $pin (box HEAD was $head_before; git as $DEPLOY_USER)\"\n # A CI attestation names one commit; tests that passed on another prove nothing here.\n if [[ -n ${VPS_DEPLOY_CI_SHA:-} ]]; then\n [[ $VPS_DEPLOY_CI_SHA == $pin ]] \\\n || deploy_refuse \"bin/ci ran on $VPS_DEPLOY_CI_SHA but this run would deploy $pin\"\n print \"vps-deploy: bin/ci passed on the operator's machine for $pin; the box skips its suite\"\n fi\n if [[ -n ${VPS_DEPLOY_PREFLIGHT_SHA:-} ]]; then\n [[ $VPS_DEPLOY_PREFLIGHT_SHA == $pin ]] \\\n || deploy_refuse \"preflight ran on $VPS_DEPLOY_PREFLIGHT_SHA but this run would deploy $pin\"\n print \"vps-deploy: preflight passed on the operator's machine for $pin\"\n elif [[ ${VPS_DEPLOY_SKIP_PREFLIGHT:-} == 1 ]]; then\n [[ ${I_UNDERSTAND_FAST_DEPLOY:-} == 1 ]] \\\n || deploy_refuse \"VPS_DEPLOY_SKIP_PREFLIGHT=1 needs the human-only I_UNDERSTAND_FAST_DEPLOY=1\"\n print \"vps-deploy: WARNING - preflight skipped (break-glass); $pin was not proven to boot\"\n else\n set_stage preflight-boot\n print \"vps-deploy: preflight on $pin (syntax, law files, boot smoke)\"\n preflight_back_to=$(git_as_deploy_user symbolic-ref -q --short HEAD || print -r -- \"$head_before\")\n # A box left detached comes back to its commit, not to a guessed branch.\n preflight_back_flag=()\n git_as_deploy_user symbolic-ref -q HEAD >/dev/null || preflight_back_flag=(--detach)\n # Set recovery state before switching: the EXIT/HUP/INT/TERM path now owns\n # restoration even if the signal lands immediately after the detach.\n preflight_active=1\n git_as_deploy_user switch -q --detach \"$pin\" || deploy_refuse \"cannot check out $pin in $repo (local changes?)\"\n if ! \"$RUBY\" \"$repo/MASTER/lib/operator/canary.rb\" --no-rehearse; then\n restore_preflight_checkout || deploy_refuse \"preflight failed on $pin and checkout restoration failed; restore $preflight_back_to before retrying\"\n deploy_refuse \"preflight failed on $pin; the checkout is back on $preflight_back_to and the previous release is still serving\"\n fi\n restore_preflight_checkout || deploy_refuse \"cannot return to $preflight_back_to after preflight\"\n fi\n git_as_deploy_user merge -q --ff-only \"$pin\" || deploy_refuse \"cannot fast-forward $repo to $pin\"\n VPS_DEPLOY_STARTED=\"$(admin_service list-started 2>/dev/null)\"\n VPS_DEPLOY_STARTED=${VPS_DEPLOY_STARTED//$'\\n'/ }\n export VPS_DEPLOY_STARTED\n VPS_DEPLOY_RECEIPT_PREVIOUS_SHA=$head_before\n VPS_DEPLOY_RECEIPT_PINNED_SHA=$pin\n VPS_DEPLOY_RECEIPT_SERVICES=$VPS_DEPLOY_STARTED\n VPS_DEPLOY_RECEIPT_RECOVERY=none\n [[ -z $VPS_DEPLOY_STARTED ]] || VPS_DEPLOY_RECEIPT_RECOVERY=pending\n export VPS_DEPLOY_RECEIPT_PREVIOUS_SHA VPS_DEPLOY_RECEIPT_PINNED_SHA\n export VPS_DEPLOY_RECEIPT_SERVICES VPS_DEPLOY_RECEIPT_RECOVERY\n print \"vps-deploy: started before this deploy: $VPS_DEPLOY_STARTED\"\n set_stage snapshot\nfi\n\n # Resolve against the inventory in the checkout after it has been fast-forwarded\n # to VPS_DEPLOY_SHA. A stale box checkout must not define target membership.\n deploy_target_kind=$(inventory_target_kind \"$app\") || deploy_refuse \"could not read the pinned deployment inventory for $app\"\n [[ $deploy_target_kind != unknown ]] || deploy_refuse \"unknown app or group $app in the pinned RAILS/apps.yml\"\n DEPLOY_ALL=(${(@f)$(inventory_names all)})\n DEPLOY_ALL_DEGRADED=(${(@f)$(inventory_names all 1)})\n DEPLOY_RAILS=(${(@f)$(inventory_names rails)})\n load_deploy_ports\n typeset -a receipt_ports\n for svc in ${(k)deploy_ports}; do\n receipt_ports+=(\"$svc=${deploy_ports[$svc]}\")\n done\n VPS_DEPLOY_RECEIPT_PORTS=\"${(j:,:)receipt_ports}\"\n VPS_DEPLOY_RECEIPT_ORDER=\"${(j:,:)DEPLOY_ALL_DEGRADED}\"\n VPS_DEPLOY_RECEIPT_SNAPSHOT_RECORDED=1\n export VPS_DEPLOY_RECEIPT_PORTS VPS_DEPLOY_RECEIPT_ORDER\n export VPS_DEPLOY_RECEIPT_SNAPSHOT_RECORDED\n set_stage inventory-ready\n (( ${#DEPLOY_ALL} )) || deploy_refuse \"the pinned deployment inventory has no active apps\"\n deploy_app_group=$(inventory_app_group \"$app\")\n if [[ $deploy_target_kind == all || $deploy_app_group == master ]]; then\n ensure_smart_turn_assets\n fi\n\n# The deployment sequence is data in RAILS/apps.yml. Degraded mode\n# moves the MASTER namespace last; all other relative order stays unchanged.\nchoose_deploy_order() {\n local svc port avail why=\n load_deploy_ports\n avail=$(\"$RUBY\" -e 'print File.readlines(ARGV[0]).last.to_s[/mem_avail=(\\d+)/, 1].to_s' \\\n \"${VPS_DEPLOY_GUARD_LOG:-/var/log/resource_guard_history.log}\" 2>/dev/null)\n for svc in $DEPLOY_RAILS; do\n (( ${${=VPS_DEPLOY_STARTED:-}[(Ie)$svc]} )) || continue\n port=${deploy_ports[$svc]:-}\n [[ -n $port ]] && wait_up \"$port\" 5 || why=\"$svc is not answering /up\"\n done\n if [[ -z $why ]]; then\n [[ $avail == <-> ]] || why=\"memory availability unreadable\"\n fi\n if [[ -z $why ]] && (( avail < ${VPS_DEPLOY_MASTER_MIN_AVAIL:-10} )); then\n why=\"memory availability ${avail}% is under ${VPS_DEPLOY_MASTER_MIN_AVAIL:-10}%\"\n fi\n if [[ -z $why ]]; then\n deploy_order=($DEPLOY_ALL)\n else\n print \"vps-deploy: all - MASTER moved to the end: $why\"\n deploy_order=($DEPLOY_ALL_DEGRADED)\n fi\n}\n\nif [[ $deploy_target_kind == all || $deploy_target_kind == group ]]; then\n # The parent spans child deploys and cooldowns. Its own flag keeps the\n # resource guard out until the whole batch (or its failure recovery) ends.\n keep_deploy_flag\n typeset -i first=1\n typeset -a deploy_order\n if [[ $deploy_target_kind == all ]]; then\n choose_deploy_order\n else\n deploy_order=(${(@f)$(inventory_names \"$app\")})\n fi\n (( ${#deploy_order} )) || { print -u2 \"vps-deploy: deployment group $app is empty\"; exit 2 }\n print \"vps-deploy: $app order: ${deploy_order}\"\n if [[ $deploy_target_kind == all ]]; then\n set_stage openbsd-mirror\n print \"vps-deploy: applying tracked OpenBSD filesystem mirror\"\n zsh \"$repo/OPENBSD/OPERATOR.sh\" --apply || deploy_refuse \"OpenBSD filesystem mirror apply failed; no application deployment was started\"\n fi\n for one in $deploy_order; do\n # Let the previous app's deploy drain before starting the next one's CI.\n # Serial builds protect this one-core host from overlapping load peaks.\n if (( ! first )); then\n print \"vps-deploy: $app - settling for ${PUB5_DEPLOY_COOLDOWN:-60}s before $one\"\n sleep ${PUB5_DEPLOY_COOLDOWN:-60}\n fi\n first=0\n print \"vps-deploy: $app -> $one\"\n # Each child executes the complete per-app CI, sync, migration, restart,\n # health-check and deploy-stamp path. A failure halts this group.\n set_stage \"$app:$one\"\n zsh \"$0\" \"$one\" || { halted_app=$one; print -u2 \"vps-deploy: $app halted at $one\"; exit 1 }\n done\n print \"vps-deploy: $app ok (${deploy_order})\"\n exit 0\nfi\n\n# SHA and time are captured at write_stamp, not here. vps_deploy_master.sh\n# pulls origin/main; reading HEAD before that call stamped the pre-pull SHA\n# (76f1a65fa on 2026-08-12 while /health correctly said 5eff4d87a).\n\nwrite_stamp() {\n local name=$1 stamp_status=$2\n local sha\n stamp_written=1\n sha=$(git_as_deploy_user rev-parse --short HEAD)\n admin_request stamp \"$name\" \"$sha\" \"$stamp_status\"\n}\n\nset_stage \"$deploy_app_group\"\nif [[ $deploy_app_group == master ]]; then\n print \"vps-deploy: master web\"\n # master's bundle, native gem builds and precompile run as long as an app's CI,\n # and this branch never reached the keeper below: on 2026-10-27 the guard read\n # deploy=0 through a 30-minute master deploy and shed amber and bsdports twice.\n keep_deploy_flag\n settle_boots\n if zsh \"$repo/OPENBSD/bin/vps_deploy_master.sh\"; then\n # The loopback journeys that need no browser, for this app only.\n set_stage master-flows\n hold_deploy_flag\n if ! FLOW_APP=\"$app\" MASTER_STRICT=deploy CHECK_AUTOFIX=0 CHECK_REQUIRE_LIVE=1 \"$RUBY\" \"$repo/MASTER/checks/runner.rb\" flow_journey; then\n write_stamp \"$app\" failed\n print -u2 \"vps-deploy: master flow journeys failed - see the per-flow lines above\"\n exit 1\n fi\n set_stage master-canary\n if ! \"$RUBY\" \"$repo/OPENBSD/bin/canary\" --local --app \"$app\" --no-flows; then\n write_stamp \"$app\" failed\n print -u2 \"vps-deploy: master local canary failed - see the failed checks above\"\n exit 1\n fi\n set_stage restore-shed\n if restore_started; then\n restore_ok=1\n else\n restore_ok=0\n fi\n if (( ! restore_ok )); then\n write_stamp \"$app\" failed\n print -u2 \"vps-deploy: master is not complete; a service that was running before is still down\"\n exit 1\n fi\n write_stamp \"$app\" ok\n exit 0\n fi\n # A failed run stamps failed, or last_deploy_master.json keeps saying ok\n # from the last good run and /health reports a live SHA while :53187 is\n # closed (measured 2026-08-13).\n write_stamp \"$app\" failed\n exit 1\nfi\n\n[[ -d /home/${app}/app ]] || { print -u2 \"missing /home/${app}/app\"; exit 1 }\n\n# Rails apps do not pull inside vps_ci.sh, so without this step a deploy CIs and\n# syncs whatever the VPS checkout already holds, which can lag origin/main by\n# days. master pulls inside vps_deploy_master.sh; this is the matching step for\n# the three copy-tree apps.\nset_stage pull\n# No pull here: the top-level process pinned one SHA and moved the checkout to it.\n# A checkout that has moved off it since means another hand touched the box.\nprint \"vps-deploy: $app (checkout must be ${VPS_DEPLOY_SHA:?no pinned SHA, run through the top-level vps-deploy})\"\n[[ $(git_as_deploy_user rev-parse HEAD) == \"$VPS_DEPLOY_SHA\" ]] \\\n || { print -u2 \"vps-deploy: $app refused - HEAD is $(git_as_deploy_user rev-parse HEAD), not the pinned $VPS_DEPLOY_SHA\"; exit 1 }\n\n# Free space, before anything is stopped. A bundle install, an asset precompile or\n# a CI mirror that runs out of disk dies halfway, after the app was stopped for the\n# migrate; refusing here costs nothing and leaves the previous release serving.\n# /home holds the checkout, the app trees, bundles and storage (3.9 GB free of\n# 17 GB on 2026-10-12), so it is the filesystem that fills. The floor is a\n# variable because it is a measurement of this box, not a law.\nset_stage preflight-disk\nmin_free_kb=${VPS_DEPLOY_MIN_FREE_KB:-1048576}\nfor mount_point in /home /var; do\n free_kb=${${(z)${${(f)\"$(${VPS_DEPLOY_DF:-df} -Pk $mount_point 2>/dev/null)\"}[2]}}[4]:-}\n if [[ $free_kb != <-> ]]; then\n print -u2 \"vps-deploy: $app refused - free-space measurement for $mount_point is unavailable.\"\n print -u2 \" Nothing was stopped. Check the filesystem and measurement command, then rerun:\"\n print -u2 \" doas zsh OPENBSD/bin/vps-deploy $app\"\n exit 1\n fi\n if (( free_kb < min_free_kb )); then\n print -u2 \"vps-deploy: $app refused - $mount_point has $(( free_kb / 1024 )) MB free, the floor is $(( min_free_kb / 1024 )) MB.\"\n print -u2 \" Nothing was stopped. Inspect usage with: doas /usr/local/sbin/pub5-admin disk-usage $mount_point ; clear old\"\n print -u2 \" logs or bundles, then rerun: doas zsh OPENBSD/bin/vps-deploy $app\"\n exit 1\n fi\ndone\n\nset_stage preflight\nprint \"vps-deploy: $app (preflight load)\"\nadmin_request resource-guard || true\n\nkeep_deploy_flag\n\n# What is about to be deployed, against what is actually running.\n#\n# The check compares OPENBSD/mirror/etc byte-for-byte with live /etc and root's crontab\n# with data/root.crontab. A deploy is exactly when that matters — it is the moment\n# the repo becomes the box.\n#\n# Advisory, not blocking, and deliberately so. Measured 2026-09-10 with\n# `SSH_HOST=dev@brgen.no ruby OPENBSD/checks/config_drift_check.rb --remote`: eight files\n# differ and two more are absent from the box entirely — emergency_cpu.sh, which\n# is resource_guard's crisis tier, and vps_weekly_integrity.sh, which the crontab\n# has scheduled for weeks. A hard check here would refuse every deploy until an\n# OPERATOR.sh run nobody has scheduled, and one of the eight is rc.d/master, which\n# starts a production service. Make it blocking the first time a deploy runs with\n# this clean; a warning that names a number is still the difference between\n# knowing and not.\nif [[ -x /usr/local/bin/config_drift_check.rb ]]; then\n if ! \"$RUBY\" /usr/local/bin/config_drift_check.rb > /tmp/vps-deploy-drift.out 2>&1; then\n # ruby, not grep: coreutils are banned in committed scripts here, and the\n # check's own summary line already carries the count in the shape we want.\n drift_count=$(\"$RUBY\" -e \\\n 'print File.read(\"/tmp/vps-deploy-drift.out\").scan(/: DRIFT -/).size' 2>/dev/null)\n print \"vps-deploy: $app WARNING - ${drift_count:-?} config(s) on the box differ from the repo being deployed\"\n print \"vps-deploy: $app see /tmp/vps-deploy-drift.out\"\n # Blocking is opt-in until the box has run clean once; the default stays the\n # warning above so an unscheduled OPERATOR.sh run cannot freeze every deploy.\n if [[ ${VPS_DEPLOY_DRIFT:-warn} == fail ]]; then\n print -u2 \"vps-deploy: $app refused - VPS_DEPLOY_DRIFT=fail and the box has drifted\"\n exit 1\n fi\n fi\nfi\n\nif [[ ${SKIP_CI:-} == 1 ]]; then\n print \"vps-deploy: $app (sync via ${app}.sh, SKIP_CI=1)\"\n # The app entrypoints refuse root (line 5 of every one of them: checkout\n # code is operator-owned, privileged actions go through dev's doas and the\n # pub5_rcctl broker), so the sync runs as the deploy user, not as this\n # script's root. First exercised on vm23 2026-11-08: as root, brgen.sh\n # refused and the app stage died before stop, leaving the release serving.\n run_as_deploy_user \"cd $(printf '%q' \"$repo\") && zsh $(printf '%q' \"$repo/RAILS/${app}/${app}.sh\")\"\nelse\n # The app's resident Solid Queue worker stands down for the CI run.\n #\n # vm23 is 1 GB. brgen_jobs holds ~380 MB resident and brgen's suite is 896\n # tests: on 2026-08-23 the runner was OOM-killed mid-suite with swap at 898 MB\n # of 1264, and the failure is silent in the worst way — 155 dots, no summary,\n # no failure block, and vps_ci reporting \"Tests: Rails failed\". Dots then\n # nothing in a CI test section means the runner was killed, not that a test\n # failed. Stopped and restarted here so the knowledge lives in the deploy\n # rather than in whoever remembers to type it.\n # No other app may still be booting when CI starts: a Falcon cold boot and a test\n # suite on one core is how the apps were shed.\n settle_boots\n jobs_service=\"${app}_jobs\"\n jobs_was_up=0\n if admin_service get \"$jobs_service\" >/dev/null 2>&1 && admin_service check \"$jobs_service\" >/dev/null 2>&1; then\n print \"vps-deploy: $app (stand down $jobs_service for the CI run)\"\n admin_service stop \"$jobs_service\" >/dev/null 2>&1 && { jobs_was_up=1; own_stops+=(\"$jobs_service\") }\n fi\n\n # Swap floor, measured after the stand-down above has freed what it frees.\n # CI, bundle and asset work on a 1 GB box with swap nearly gone is how every\n # app was shed: on 2026-11-03 a gem fetch at ~200 MB free took all sites down\n # for 17 minutes, and on 2026-11-04 a MASTER run at 217 MB free timed brgen out.\n set_stage preflight-swap\n min_swap_mb=${VPS_DEPLOY_MIN_FREE_SWAP_MB:-400}\n swap_blocks=${${(M)${(z)\"$(${VPS_DEPLOY_SWAPCTL:-swapctl} -s 2>/dev/null)\"}:#<->}[3]:-}\n if [[ $swap_blocks == <-> ]] && (( swap_blocks * 512 / 1048576 < min_swap_mb )); then\n (( jobs_was_up )) && admin_service start \"$jobs_service\" >/dev/null 2>&1\n free_swap_mb=$(( swap_blocks * 512 / 1048576 ))\n print -u2 \"vps-deploy: $app refused - $free_swap_mb MB of swap free, the floor is $min_swap_mb MB.\"\n print -u2 \" The app's jobs service is back up; nothing else was stopped. Free memory (stop another\"\n print -u2 \" app's *_jobs service), then rerun: doas zsh OPENBSD/bin/vps-deploy $app\"\n exit 1\n fi\n\n set_stage ci+sync\n print \"vps-deploy: $app (ci + sync)\"\n # `|| ci_status=$?` rather than a bare call and `ci_status=$?` after it. This\n # script runs under `set -e`, so a bare failing CI run exits here and the\n # restart below — the whole reason the stand-down records jobs_was_up — never\n # runs, leaving the worker stopped with nothing 5xx'ing and the queue simply not\n # draining. A command followed by `||` is exempt from errexit, which is what\n # makes the status capture reachable at all.\n ci_status=0\n zsh \"$repo/OPENBSD/bin/vps_ci.sh\" \"$app\" || ci_status=$?\n\n if (( jobs_was_up )); then\n admin_service start \"$jobs_service\" >/dev/null 2>&1 \\\n || print -u2 \"vps-deploy: $jobs_service did not come back - start it\"\n fi\n (( ci_status == 0 )) || exit $ci_status\nfi\n\n# Primary database, then the cache/queue/cable schemas.\n#\n# db_create_migrate_as_app runs db:prepare, which is idempotent — it creates the\n# database when absent, loads the schema when empty, and otherwise runs only\n# pending migrations — and then does the secondary schemas itself. Loading the\n# secondaries alone leaves the primary unmigrated, so a merged migration never\n# reaches production: on 2026-07-29 brgen shipped a counter-cache column its\n# views already read, and every feed render 500'd until an operator ran the\n# three pending migrations by hand.\nset_stage migrate\nprint \"vps-deploy: $app (primary db migrate + secondary schemas)\"\n. \"$repo/RAILS/_core.sh\"\n. \"$repo/RAILS/_bundle.sh\"\n. \"$repo/RAILS/_runtime_check.sh\"\n. \"$repo/RAILS/_database.sh\"\n. \"$repo/RAILS/_assets.sh\"\n# Same reason RAILS/_deploy.sh stops first: db:prepare hangs on the write\n# lock a live Falcon worker already holds on production.sqlite3.\nprint \"vps-deploy: $app (stop for sqlite migrate)\"\nadmin_service stop \"$app\" 2>/dev/null || true\napp_stopped=1\nown_stops+=(\"$app\")\ndb_create_migrate_as_app \"$app\" \"/home/${app}/app\"\n\n# The SKIP_CI=1 path (${app}.sh -> deploy_tracked_app -> rails_runtime_check)\n# already precompiles as part of its check. The default ci+sync path\n# (vps_ci.sh) only runs tests against a throwaway test DB and never\n# touches production assets -- without this, public/assets/ is simply\n# never created for a plain `vps-deploy $app` run, so Propshaft 404s\n# every asset_path() and the page renders with no CSS at all. Safe to\n# call unconditionally: Propshaft precompile is idempotent.\nset_stage precompile\nprint \"vps-deploy: $app (assets precompile)\"\nsettle_boots\nrails_assets_precompile_as_app \"$app\" \"/home/${app}/app\" || exit 1\n\nset_stage restart\nprint \"vps-deploy: $app (restart)\"\nhold_deploy_flag\nif ! admin_service_retry restart \"$app\"; then\n log \"restart failed after retries\"\n write_stamp \"$app\" failed\n exit 1\nfi\nif ! retry 5 2 admin_service check \"$app\"; then\n log \"service check failed after restart\"\n write_stamp \"$app\" failed\n exit 1\nfi\napp_stopped=0\nset_stage health\nport=$(\"$RUBY\" -ryaml -e 'puts YAML.safe_load(File.read(\"'\"$repo\"'/RAILS/apps.yml\")).dig(\"apps\",\"'\"$app\"'\",\"port\")')\nup_ok=0\nfor i in {1..150}; do\n curl -fsS \"http://127.0.0.1:${port}/up\" >/dev/null 2>&1 && { up_ok=1; break; }\n sleep 2\ndone\n[[ $up_ok -eq 1 ]] || { write_stamp \"$app\" failed; print -u2 \"vps-deploy: $app /up not ready on :${port}\"; exit 1; }\n\n# /up answers before Propshaft is involved, so it says nothing about whether the\n# page a visitor gets is styled. On 2026-08-14 brgen served every page with a\n# to a stylesheet that 404'd — public/assets held zero files — and /up,\n# rcctl check, the port probe and the public TLS check all passed. The site was\n# plainly broken and the only thing that noticed was a person looking at it.\n#\n# So ask for the stylesheet the page actually links, through the Host that owns\n# it: these apps are domain-constrained and a bare-IP request is a 403, which\n# would make this check pass by never finding a link to test.\ndomain=$(\"$RUBY\" -ryaml -e 'puts YAML.safe_load(File.read(\"'\"$repo\"'/RAILS/apps.yml\")).dig(\"apps\",\"'\"$app\"'\",\"domain\")')\n# The page and the link are two reads, because under set -e and pipefail one\n# pipeline that finds no link fails the assignment and ends the deploy with no\n# stamp and no message. A home page that does not answer is a failed deploy; a\n# home page that links no stylesheet is the else branch below.\nhome_page=$(curl -fsS --max-time 30 -H \"Host: ${domain}\" \"http://127.0.0.1:${port}/\" 2>/dev/null) || {\n write_stamp \"$app\" failed\n print -u2 \"vps-deploy: $app / did not answer on :${port} with Host ${domain}\"\n exit 1\n}\ncss_href=$(print -r -- \"$home_page\" | \"$RUBY\" -e 'print $stdin.binmode.read[%r{/assets/[^\"]+\\.css}n].to_s')\nif [[ -n $css_href ]]; then\n if curl -fsS -o /dev/null --max-time 30 -H \"Host: ${domain}\" \"http://127.0.0.1:${port}${css_href}\" 2>/dev/null; then\n print \"vps-deploy: $app stylesheet ok (${css_href})\"\n else\n write_stamp \"$app\" failed\n print -u2 \"vps-deploy: $app serves ${css_href} as 404 - precompiled assets missing, the site is unstyled\"\n exit 1\n fi\nelse\n print \"vps-deploy: $app no /assets stylesheet linked on / - nothing to verify\"\nfi\n\nset_stage solid_queue\nqueue_schema=\"$repo/RAILS/${app}/db/queue_schema.rb\"\n# An app whose production config runs jobs in-process has no worker to prove;\n# demanding one, or an hourly drain, would be meaningless.\n# UTF-8 explicitly: the box runs this with no LANG, and production.rb has em dashes.\nqueue_async=0\n\"$RUBY\" -e 'exit File.read(ARGV[0], encoding: \"UTF-8\").match?(/queue_adapter\\s*=\\s*:async\\b/)' \\\n \"$repo/RAILS/${app}/config/environments/production.rb\" 2>/dev/null && queue_async=1\nqueue_empty=0\n[[ -f $queue_schema ]] && \"$RUBY\" -e 'exit File.read(ARGV[0]).include?(\"define(version: 0)\")' \\\n \"$queue_schema\" && queue_empty=1\nif (( ! queue_async && ! queue_empty )) && [[ -f $queue_schema ]]; then\n print \"vps-deploy: $app (solid_queue proof)\"\n admin_request solid-queue-proof \"$app\" \\\n || { write_stamp \"$app\" failed; print -u2 \"vps-deploy: solid_queue proof failed for $app\"; exit 1; }\nfi\n\n# Restore whatever the guard shed while we were deploying.\n#\n# resource_guard sheds its OPTIONAL list — bsdports, then amber — under\n# sustained pressure, and a deploy is exactly that. relayd keeps answering TLS\n# with their ports closed, so the outage reads as a hang rather than a 5xx and\n# nothing reports it: on 2026-08-23 amber and bsdports were restarted by hand\n# three times in one session, each time discovered by a person running curl.\n#\n# `vps-deploy all` folds the restore into its ordering; this pass gives a\n# single-app deploy, the common case, the same restore. Reported, not silent — a\n# service that needed restoring is a fact about the box worth seeing.\nset_stage restore-shed\nrestore_failed=0\nrestore_started || restore_failed=1\n\n# The checks, on the deployed thing.\n#\n# Outside this pass the checks run only when a person types a command, so the\n# half that can only be measured against a running app would be measured on a\n# laptop where it means less. A user journey is real only here: production\n# answers 403 to a request without the right Host header, which development\n# never does.\n#\n# This set is the cheap half: source checks plus the live probes against the\n# loopback. It runs in about forty seconds on this one core with the fleet up,\n# which is what makes it affordable on every deploy.\n#\n# The browser half — rendered_suite, layout_suite, visual_contract — is NOT here\n# and is opt-in through PUB5_DEPLOY_BROWSER_CHECKS=1. Chrome is installed, but\n# this box is 1GB with four apps resident and resource_guard sheds amber and\n# bsdports under exactly that pressure. A deploy that takes the site down to\n# check the site has failed at the only thing it was for.\n#\n# CHECK_AUTOFIX=0 because a check must not rewrite the tree it is judging, least\n# of all the one that is deployed.\n#\n# CHECK_REQUIRE_LIVE=1 because this is the one host where a live check has no\n# excuse. It turns \"port closed, skipping\" into a failure, and half this set —\n# flow_journey, page_simulation, port_inventory — measures nothing without a\n# listening app. The fleet was just restarted and the shed apps restored a few\n# lines above, so a closed port here is a deploy that did not come up, not a\n# laptop. bin/check-rails takes the same position on the box through\n# Environment.on_vps?, and a second caller on the same host must not quietly\n# accept a skip.\ncheck_names=(production port_inventory generated_asset schema_migration\n phantom_foreign_keys frontend_production stimulus_components\n flow_journey page_simulation)\nif [[ -n ${PUB5_DEPLOY_BROWSER_CHECKS:-} ]]; then\n check_names+=(rendered_suite layout_suite)\nfi\nset_stage checks\nprint \"vps-deploy: $app (checks: ${check_names})\"\nhold_deploy_flag\nif ! FLOW_APP=\"$app\" MASTER_STRICT=deploy CHECK_AUTOFIX=0 \"$RUBY\" \"$repo/MASTER/checks/runner.rb\" \"${check_names[@]}\"; then\n write_stamp \"$app\" failed\n print -u2 \"vps-deploy: $app checks failed - the app is deployed and serving, and something\"\n print -u2 \" it is now serving does not hold. Fix or revert; do not deploy over it.\"\n exit 1\nfi\n\nset_stage final-check\n# The app deployed last has to be the app still running when the deploy says ok.\n# Every check above ran minutes ago, and a deploy whose own app is down on exit\n# is the one outcome the stamp must never call ok.\nif ! admin_service check \"$app\" >/dev/null 2>&1; then\n write_stamp \"$app\" failed\n print -u2 \"vps-deploy: $app is down at the end of its own deploy - /usr/local/sbin/pub5-admin service restart $app, then read /var/log/daemon\"\n exit 1\nfi\n\n# The local canary is a release gate, not an advisory log line.\nif ! \"$RUBY\" \"$repo/OPENBSD/bin/canary\" --local --app \"$app\"; then\n write_stamp \"$app\" failed\n print -u2 \"vps-deploy: $app local canary failed - see the failed checks above\"\n exit 1\nfi\n\nif (( restore_failed )); then\n write_stamp \"$app\" failed\n print -u2 \"vps-deploy: $app is not complete; a service that was running before is still down\"\n exit 1\nfi\n\nprint \"vps-deploy: $app ok\"\nwrite_stamp \"$app\" ok\nexit 0\n" to include "def write_deploy_run_receipt". 83 runs, 720 assertions, 4 failures, 0 errors, 0 skips ok 0.1s OPENBSD/test/test_ci_mirror_contract.rb ok 2.8s OPENBSD/test/test_ruby_select.rb ok 0.9s RAILS/test/require_relative_targets_test.rb ok 0.9s RAILS/test/no_top_level_test_calls_test.rb ok 0.5s RAILS/test/coverage_layout_test.rb ok 0.4s RAILS/test/erb_container_balance_test.rb ok 0.2s RAILS/test/test_path_idioms_test.rb ok 0.1s RAILS/test/typography_lint_test.rb rehearsal: 1 of 16 failed - fix these before vps-deploy --- rehearsal_exit=1 ---