Cairnbin

Untitled

text 3 views Created: November 17, 2026 at 18:17:39 EST
 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
ok     0.2s  MASTER/tools/rhythm_lint.rb
ok     0.7s  MASTER/tools/rails/operator/fallback_drift_lint.rb
ok     0.1s  MASTER/tools/empty_state_lint.rb
ok     0.1s  MASTER/tools/adhoc_empty_lint.rb
ok     0.4s  MASTER/tools/chrome_i18n_lint.rb
ok     0.4s  MASTER/tools/rails/operator/dialect_token_drift_check.rb
ok     0.4s  OPENBSD/test/test_deploy_paths_exist.rb
FAIL   4.3s  OPENBSD/test/test_vps_deploy_contract.rb
       Run options: --seed 2837
       
       # Running:
       
       F...Ffinish:log:66: too many arguments
       ........F...........................................................F.........
       
       Finished in 4.155771s, 19.9722 runs/s, 173.2530 assertions/s.
       
         1) Failure:
       VpsDeployContractTest#test_recovery_plan_is_explicit_and_never_deploys_or_checks_out_code [test/test_vps_deploy_contract.rb:1097]:
       Expected "#!/usr/bin/env zsh\n# Serial VPS deploy entrypoint — sync, CI, restart, health, deploy stamp.\n# Usage: zsh OPENBSD/bin/vps-deploy all|GROUP|APP\nset -euo pipefail\n\n# SSH executes this as a non-login shell on vm23; pin the OpenBSD package path.\nexport PATH=/usr/local/bin:/usr/local/sbin:/usr/bin:/usr/sbin:/bin:/sbin\n\nusage() {\n  print -r -- \"usage: zsh OPENBSD/bin/vps-deploy all|GROUP|APP   (runs as root; re-execs via doas)\n\n  all          every active registered surface, including MASTER web\n  GROUP        every registered app whose deploy_root belongs to that namespace\n  APP          one exact app ID from RAILS/apps.yml\n  recover --plan              inspect interrupted-deploy recovery receipts\n  recover --apply <run-id>     restore only services recorded as up before that run\n  VPS_DEPLOY_SHA=<sha>   deploy exactly this commit (default: origin/main, fetched once)\n  VPS_DEPLOY_PREFLIGHT_SHA=<sha>   the operator's machine already proved this commit boots\n  VPS_DEPLOY_CI_SHA=<sha>   the operator's machine ran the app's bin/ci green on this commit; the box skips its suite\n  VPS_DEPLOY_SKIP_PREFLIGHT=1      break-glass, needs I_UNDERSTAND_FAST_DEPLOY=1\n  VPS_DEPLOY_SKIP_TESTS=1          break-glass: no test suite anywhere, needs I_UNDERSTAND_FAST_DEPLOY=1\n  SKIP_CI=1    skip vps_ci.sh; \\${app}.sh and its runtime check still run\n  VPS_DEPLOY_DRIFT=fail   refuse to deploy when the box's configs differ from the repo\n  PUB5_DEPLOY_BROWSER_CHECKS=1   add the browser checks after restart\n  VPS_DEPLOY_LOG=/path            aggregate log (default /var/log/pub5/vps-deploy.log)\n  VPS_DEPLOY_LOG_DIR=/path        log directory (default /var/log/pub5)\n  VPS_DEPLOY_RUN_LOG=/path        per-run log (default auto under LOG_DIR/runs/)\n\n  Git fetch/checkout always runs as user dev so the /home/dev/pub5 tree and the\n  GitHub deploy key stay owned and reachable. Service control uses pub5-admin\n  directly when already root.\"\n}\ncase ${1:-} in\n  -h|--help) usage; exit 0 ;;\nesac\n\n# Prefer root. Passwordless doas (dev in wheel / permit nopass) re-execs so the\n# operator can still type the command as dev. Git never runs as root: root has\n# no GitHub host key by design, and a root checkout would leave root-owned files\n# in the dev tree. Service control calls the broker directly when uid 0.\n#\n# doas strips the caller environment (the dev rule has no keepenv — intentional,\n# so RUBYOPT and friends cannot cross the privilege boundary). Re-exec through\n# /usr/bin/env with a prefix allowlist (VPS_DEPLOY_*, PUB5_DEPLOY_*, plus PUB5_ROOT,\n# SKIP_CI, I_UNDERSTAND_FAST_DEPLOY) so pin, preflight, canary and logging\n# knobs survive. Not keepenv: RUBYOPT and friends still cannot cross.\nif [[ $(id -u) -ne 0 ]]; then\n  if command -v doas >/dev/null 2>&1; then\n    self=$0\n    if [[ $self != /* ]]; then\n      self=\"$(cd -- \"$(dirname -- \"$self\")\" && pwd)/$(basename -- \"$self\")\"\n    fi\n    typeset -a deploy_env\n    # Prefix allowlist (not keepenv): every VPS_DEPLOY_* and PUB5_DEPLOY_* that\n    # is set in the caller crosses doas via /usr/bin/env. Plus a few one-offs\n    # the deploy reads. New knobs named under those prefixes need no edit here.\n    for _name in ${(k)parameters}; do\n      case $_name in\n        VPS_DEPLOY_*|PUB5_DEPLOY_*|PUB5_ROOT|SKIP_CI|I_UNDERSTAND_FAST_DEPLOY)\n          deploy_env+=(\"$_name=${(P)_name}\")\n          ;;\n      esac\n    done\n    exec doas -n /usr/bin/env \"${deploy_env[@]}\" zsh \"$self\" \"$@\"\n  fi\n  print -u2 \"vps-deploy: must run as root (or with passwordless doas)\"\n  print -u2 \"            try: doas zsh OPENBSD/bin/vps-deploy ${1:-all}\"\n  exit 2\nfi\n\n# Unprivileged owner of the checkout and GitHub credentials on this box.\nDEPLOY_USER=${VPS_DEPLOY_USER:-dev}\nDEPLOY_HOME=${VPS_DEPLOY_HOME:-/home/$DEPLOY_USER}\nrepo=${PUB5_ROOT:-/home/dev/pub5}\nsource \"$repo/OPENBSD/lib/ruby_select.sh\"\ninventory_target_kind() {\n  \"$RUBY\" -r\"$repo/OPENBSD/lib/deploy_inventory\" -e \\\n    'print Deploy::Inventory.new(root: ARGV[0]).target_kind(ARGV[1])' \"$repo\" \"$1\"\n}\n\ninventory_app_group() {\n  \"$RUBY\" -r\"$repo/OPENBSD/lib/deploy_inventory\" -e '\n    app = Deploy::Inventory.new(root: ARGV[0]).apps.find { |row| row.name == ARGV[1] }\n    print app&.deploy_group\n  ' \"$repo\" \"$1\"\n}\n\ninventory_names() {\n  local target=$1 degraded=${2:-0}\n  \"$RUBY\" -r\"$repo/OPENBSD/lib/deploy_inventory\" -e '\n    inventory = Deploy::Inventory.new(root: ARGV[0])\n    rows = ARGV[1] == \"all\" ? inventory.ordered_apps(degraded: ARGV[2] == \"1\") :\n                              inventory.ordered_apps(group: ARGV[1])\n    puts rows.map(&:name)\n  ' \"$repo\" \"$target\" \"$degraded\"\n}\n\n# Run a shell snippet as DEPLOY_USER without pulling root's environment into\n# the checkout. Always su: root may change to DEPLOY_USER without a doas rule.\nrun_as_deploy_user() {\n  # Root may su to DEPLOY_USER without a doas rule. Avoid doas -u: the box\n  # only permits dev→root and root→root by default.\n  local snippet=$1\n  su -s /bin/sh \"$DEPLOY_USER\" -c \"$snippet\"\n}\n\ngit_as_deploy_user() {\n  # All git mutations against the production checkout run as DEPLOY_USER.\n  local args_q\n  args_q=$(printf '%q ' \"$@\")\n  run_as_deploy_user \"cd $(printf '%q' \"$repo\") && git $args_q\"\n}\n\napp=${1:-}\n[[ -n $app ]] || { usage >&2; exit 2 }\nrecover_mode=\nrecover_id=\nif [[ $app == recover ]]; then\n  case ${2:-} in\n    --plan)\n      [[ $# -eq 2 ]] || { usage >&2; exit 2 }\n      recover_mode=plan\n      ;;\n    --apply)\n      [[ $# -eq 3 ]] && \"$RUBY\" -e 'exit(ARGV.fetch(0).match?(/\\A[0-9]{8}T[0-9]{6}Z-[a-z0-9_-]{1,32}-[0-9]+\\z/) ? 0 : 1)' \"$3\" || {\n        print -u2 \"vps-deploy: recover --apply needs one exact run ID from --plan\"\n        exit 2\n      }\n      recover_mode=apply\n      recover_id=$3\n      ;;\n    *) usage >&2; exit 2 ;;\n  esac\n  unset VPS_DEPLOY_RUN_ID VPS_DEPLOY_RUN_TARGET VPS_DEPLOY_RUN_STARTED_AT\n  unset VPS_DEPLOY_STARTED VPS_DEPLOY_LOCK_HELD\nelse\n  [[ $# -eq 1 ]] || { usage >&2; exit 2 }\n  # Restrict the log/path component to a safe identifier before inventory lookup.\n  [[ $app != *[^A-Za-z0-9_-]* ]] || {\n    print -u2 \"vps-deploy: invalid target identifier\"\n    exit 2\n  }\nfi\n# Until the pinned checkout is ready, the current inventory is only a\n# best-effort restore-order hint for services shed during preflight. These arrays\n# are refreshed from VPS_DEPLOY_SHA before target validation or deployment.\ndeploy_target_kind=unknown\ntypeset -a DEPLOY_ALL DEPLOY_ALL_DEGRADED DEPLOY_RAILS\nDEPLOY_ALL=(${(@f)\"$(inventory_names all 2>/dev/null || true)\"})\nDEPLOY_ALL_DEGRADED=(${(@f)\"$(inventory_names all 1 2>/dev/null || true)\"})\nDEPLOY_RAILS=(${(@f)\"$(inventory_names rails 2>/dev/null || true)\"})\n\n# Environment values below participate in shell arithmetic and bounded waits.\n# Validate their decimal spelling and limits before acquiring a lock or touching\n# a service. In particular, a zero poll interval otherwise creates an endless\n# wait_up loop during a failed-deploy restore.\nvalidate_deploy_setting() {\n  local name=$1 value=$2 minimum=$3 maximum=$4\n  [[ $value == <-> && ( $value == 0 || $value != 0* ) ]] || {\n    print -u2 \"vps-deploy: invalid $name; expected a decimal integer\"\n    return 2\n  }\n  (( ${#value} <= 10 )) || {\n    print -u2 \"vps-deploy: invalid $name; value is out of range\"\n    return 2\n  }\n  (( value >= minimum && value <= maximum )) || {\n    print -u2 \"vps-deploy: invalid $name; allowed range is $minimum..$maximum\"\n    return 2\n  }\n}\n\nvalidate_deploy_setting VPS_DEPLOY_LOCK_TTL \"${VPS_DEPLOY_LOCK_TTL:-7200}\" 1 604800 || exit 2\nvalidate_deploy_setting VPS_DEPLOY_RESTORE_WAIT \"${VPS_DEPLOY_RESTORE_WAIT:-240}\" 1 3600 || exit 2\nvalidate_deploy_setting VPS_DEPLOY_POLL \"${VPS_DEPLOY_POLL:-3}\" 1 60 || exit 2\nvalidate_deploy_setting VPS_DEPLOY_SETTLE_WAIT \"${VPS_DEPLOY_SETTLE_WAIT:-240}\" 1 3600 || exit 2\nvalidate_deploy_setting VPS_DEPLOY_MASTER_MIN_AVAIL \"${VPS_DEPLOY_MASTER_MIN_AVAIL:-10}\" 0 100 || exit 2\nvalidate_deploy_setting VPS_DEPLOY_MIN_FREE_KB \"${VPS_DEPLOY_MIN_FREE_KB:-1048576}\" 0 1000000000 || exit 2\n: \"${PUB5_ADMIN:=/usr/local/sbin/pub5-admin}\"\n[[ -f \"$PUB5_ADMIN\" && -x \"$PUB5_ADMIN\" && ! -L \"$PUB5_ADMIN\" ]] || {\n  print -u2 \"vps-deploy: pub5-admin missing; run reviewed OPENBSD/OPERATOR.sh first\"\n  exit 1\n}\n\n# One host, one deploy. The parent all/rails process keeps this lock while its\n# re-entrant child processes deploy each app in order; a second top-level deploy\n# is refused rather than running two pulls, precompiles and restarts together.\nlock_dir=${VPS_DEPLOY_LOCK_DIR:-$DEPLOY_HOME/.pub5-vps-deploy.lock}\nlock_ttl=${VPS_DEPLOY_LOCK_TTL:-7200}\nlock_owner=0\n\n# The lock is a directory, because mkdir is the one atomic test-and-set the\n# base system gives a shell, and the directory holds an `owner` file: pid, host\n# and epoch seconds. A bare directory could not tell a running deploy from the\n# remains of one killed by a dropped session or a reboot, and an operator had to\n# guess before removing it. Now a lock is reaped here, with the reason printed,\n# when its same-host owner PID is gone. Age alone never steals a lock from a\n# live local deployment: a slow or stalled deploy requires operator inspection,\n# not a second deploy running beside it. A remote-host or unreadable lock may\n# be reaped only after the recorded TTL, because this checkout cannot inspect\n# the other host's process.\n# BSD date formats an epoch with -r; GNU date reads -r as a file and needs\n# -d @epoch. The reaper prints the owner's start time on both dev hosts and vm23.\nepoch_date() {\n  date -r \"$1\" '+%Y-%m-%d %H:%M:%S' 2>/dev/null || date -d \"@$1\" '+%Y-%m-%d %H:%M:%S' 2>/dev/null\n}\n\n# BSD stat formats with -f; GNU stat reads -f as filesystem status and needs\n# -c. The reaper pins the observed directory by device and inode on both hosts.\nfile_identity() {\n  local out\n  out=$(stat -f '%d:%i' \"$1\" 2>/dev/null)\n  if [[ $out == <->:* ]]; then\n    print -r -- $out\n    return 0\n  fi\n  stat -c '%d:%i' \"$1\" 2>/dev/null\n}\n\n# A PID can be reused after its original deploy exits. Pair it with the\n# process start time and a checksum of its command line, so an unrelated process\n# cannot keep an abandoned lock alive forever. OpenBSD ps(1) exposes lstart and\n# command; if either cannot be read, callers fail safe and keep the lock.\nlock_process_identity() {\n  local pid=$1 started command digest\n  started=$(LC_ALL=C ps -p \"$pid\" -o lstart= 2>/dev/null) || return 1\n  command=$(ps -ww -p \"$pid\" -o command= 2>/dev/null) || return 1\n  [[ -n $started && -n $command ]] || return 1\n  started=${started//[^A-Za-z0-9]/_}\n  digest=$(print -rn -- \"$command\" | cksum) || return 1\n  digest=${digest// /_}\n  print -r -- \"${started}_${digest}\"\n}\n\nlock_read_owner() {\n  lock_pid= lock_host= lock_epoch= lock_identity=\n  [[ -r $lock_dir/owner ]] || return 1\n  read -r lock_pid lock_host lock_epoch lock_identity < \"$lock_dir/owner\" || true\n  [[ $lock_pid == <-> && $lock_epoch == <-> && $lock_identity != *[^A-Za-z0-9_]* ]]\n}\n\nlock_stale_reason() {\n  local now=$(date +%s) current_identity\n  if lock_read_owner; then\n    if [[ $lock_host == $(hostname -s) ]]; then\n      if ! ps -p \"$lock_pid\" >/dev/null 2>&1; then\n        print -r -- \"owner pid $lock_pid is not running on $lock_host\"\n        return 0\n      fi\n      if [[ -n $lock_identity ]]; then\n        current_identity=$(lock_process_identity \"$lock_pid\") || {\n          # A visible PID whose full identity cannot be measured is ambiguous,\n          # not stale. Never steal its lock based on a partial observation.\n          return 1\n        }\n        if [[ $current_identity != $lock_identity ]]; then\n          print -r -- \"owner pid $lock_pid process identity changed on $lock_host (PID reused)\"\n          return 0\n        fi\n      fi\n      # Legacy three-field owners have no fingerprint: keep them protected while\n      # their PID exists. New owners are distinguished even after PID reuse.\n      # A live same-host owner is authoritative even after the TTL.\n      return 1\n    fi\n    if (( now - lock_epoch > lock_ttl )); then\n      print -r -- \"held $(( (now - lock_epoch) / 60 )) minutes by pid $lock_pid on $lock_host, past the ${lock_ttl}s TTL\"\n      return 0\n    fi\n    return 1\n  fi\n  # mkdir has run but the owner file is not written yet, or never was. Judge\n  # such a lock by the directory's own age so a deploy mid-start is left alone.\n  local -a aged\n  aged=( ${lock_dir}(N/mm+$(( lock_ttl / 60 ))) )\n  (( $#aged )) || return 1\n  print -r -- \"no readable owner and older than the ${lock_ttl}s TTL\"\n}\n\nrelease_deploy_lock() {\n  (( lock_owner )) || return 0\n  rm -f -- \"$lock_dir/owner\"\n  rmdir \"$lock_dir\" 2>/dev/null || true\n}\n\nreap_stale_lock() {\n  local reason before_identity after_identity retired_lock\n\n  # Do not enter recovery for a live lock. The second check below closes the\n  # race between this observation and claiming the recovery marker.\n  reason=$(lock_stale_reason) || return 1\n  before_identity=$(file_identity \"$lock_dir\") || return 1\n\n  # The marker is inside the exact directory observed above, so mkdir is an\n  # atomic claim. Inode identity catches a stale directory being replaced by a\n  # fresh lock between the first observation and this claim.\n  if ! mkdir \"$lock_dir/reaping\" 2>/dev/null; then\n    return 1\n  fi\n  after_identity=$(file_identity \"$lock_dir\") || {\n    rmdir \"$lock_dir/reaping\" 2>/dev/null || true\n    return 1\n  }\n  if [[ $after_identity != $before_identity ]]; then\n    rmdir \"$lock_dir/reaping\" 2>/dev/null || true\n    return 1\n  fi\n\n  # Creating the marker updates the parent directory's mtime. Recheck owners,\n  # but retain the pre-marker age decision for a directory that was already\n  # ownerless; no other process can write its owner file while it exists.\n  if [[ -r \"$lock_dir/owner\" ]]; then\n    reason=$(lock_stale_reason) || {\n      rmdir \"$lock_dir/reaping\" 2>/dev/null || true\n      return 1\n    }\n  fi\n\n  retired_lock=\"${lock_dir}.stale.$$\"\n  if [[ -e \"$retired_lock\" ]]; then\n    print -u2 \"vps-deploy: stale-lock quarantine exists at $retired_lock\"\n    rmdir \"$lock_dir/reaping\" 2>/dev/null || true\n    return 1\n  fi\n\n  print -u2 \"vps-deploy: reaping stale lock $lock_dir - $reason\"\n  # Rename the whole claimed directory, marker included. The old lock path\n  # becomes free atomically; cleanup touches only this unique retired path.\n  if ! mv \"$lock_dir\" \"$retired_lock\" 2>/dev/null; then\n    rmdir \"$lock_dir/reaping\" 2>/dev/null || true\n    return 1\n  fi\n  rm -f -- \"$retired_lock/owner\"\n  # A process killed before the atomic owner rename can leave only this temp.\n  # Remove it with the quarantined lock so crash recovery does not leak dirs.\n  rm -f -- \"$retired_lock\"/owner.tmp.*(N)\n  rmdir \"$retired_lock/reaping\" 2>/dev/null || true\n  rmdir \"$retired_lock\" 2>/dev/null || true\n  return 0\n}\n\nacquire_deploy_lock() {\n  local -i attempt\n  for attempt in 1 2; do\n    if mkdir \"$lock_dir\" 2>/dev/null; then\n      local identity owner_tmp\n      identity=$(lock_process_identity \"${$}\") || {\n        print -u2 \"vps-deploy: cannot identify the lock owner process; refusing an unverifiable deploy\"\n        rmdir \"$lock_dir\" 2>/dev/null || true\n        exit 1\n      }\n      owner_tmp=\"$lock_dir/owner.tmp.${$}\"\n      if ! print -r -- \"${$} $(hostname -s) $(date +%s) $identity\" > \"$owner_tmp\" ||\n         ! mv \"$owner_tmp\" \"$lock_dir/owner\"; then\n        print -u2 \"vps-deploy: cannot write the deploy lock owner record\"\n        rm -f -- \"$owner_tmp\"\n        rmdir \"$lock_dir\" 2>/dev/null || true\n        exit 1\n      fi\n      export VPS_DEPLOY_LOCK_HELD=1\n      export VPS_DEPLOY_LOCK_PID=${$}\n      export VPS_DEPLOY_LOCK_IDENTITY=$identity\n      lock_owner=1\n      return 0\n    fi\n    if reap_stale_lock; then\n      continue\n    fi\n    break\n  done\n  if lock_read_owner; then\n    print -u2 \"vps-deploy: refused - another deploy owns $lock_dir (pid $lock_pid on $lock_host since $(epoch_date $lock_epoch))\"\n  else\n    print -u2 \"vps-deploy: refused - another deploy owns $lock_dir\"\n  fi\n  exit 75\n}\n\n# Re-entrant children must prove the inherited lock belongs to their live\n# parent. An environment flag alone must never bypass top-level serialization.\nif [[ -n ${VPS_DEPLOY_LOCK_HELD:-} ]]; then\n  lock_read_owner || {\n    print -u2 \"vps-deploy: refused - child invocation has no readable parent lock owner\"\n    exit 75\n  }\n  [[ $lock_host == \"$(hostname -s)\" &&\n     $lock_pid == ${VPS_DEPLOY_LOCK_PID:-} &&\n     $lock_identity == ${VPS_DEPLOY_LOCK_IDENTITY:-} ]] || {\n    print -u2 \"vps-deploy: refused - child invocation does not match the parent lock record\"\n    exit 75\n  }\n  ps -p \"$lock_pid\" >/dev/null 2>&1 || {\n    print -u2 \"vps-deploy: refused - parent deploy process is no longer running\"\n    exit 75\n  }\n  current_identity=$(lock_process_identity \"$lock_pid\") || {\n    print -u2 \"vps-deploy: refused - cannot verify the parent deploy process\"\n    exit 75\n  }\n  [[ $current_identity == \"$lock_identity\" ]] || {\n    print -u2 \"vps-deploy: refused - parent deploy PID identity changed\"\n    exit 75\n  }\nelse\n  acquire_deploy_lock\nfi\n\n# Durable deploy logging.\n#\n# A full deploy outlives the ssh session that starts it. When that session is\n# cut the only surviving record is the log file. We keep:\n#   - a rolling aggregate log (VPS_DEPLOY_LOG / /var/log/pub5/vps-deploy.log)\n#   - a per-run log under /var/log/pub5/runs/ with the pid and app in the name\n#   - a stable symlink at $DEPLOY_HOME/vps-deploy.log for operators who still\n#     tail the home path\n# Once per run, not once per app: each group re-invokes one child per app.\n# VPS_DEPLOY_LOGGING is inherited so child processes do not double-tee.\ndeploy_log_dir=${VPS_DEPLOY_LOG_DIR:-/var/log/pub5}\ndeploy_log=${VPS_DEPLOY_LOG:-$deploy_log_dir/vps-deploy.log}\ndeploy_run_log=${VPS_DEPLOY_RUN_LOG:-}\n\nlog_ts() { date -u '+%Y-%m-%dT%H:%M:%SZ'; }\n\n# Structured line: timestamp host pid app stage message. Always goes to stdout\n# (which is already teed into the log files).\nlog() {\n  print -r -- \"$(log_ts) host=$(hostname -s) pid=$$ app=${app:-?} stage=${deploy_stage:-?} $*\"\n}\n\n# The fleet receipt is written only through the fixed root broker. Ruby's JSON\n# encoder keeps service names, paths and metadata out of shell-generated JSON.\nwrite_deploy_run_receipt() {\n  [[ -n ${VPS_DEPLOY_RUN_ID:-} ]] || return 0\n  local payload\n  payload=$(\"$RUBY\" -rjson -e '\n    run_id, target, host, pid, owner_identity, status, recovery_state, stage,\n      started_at, pinned_sha, previous_sha, services, ports_csv, order_csv,\n      run_log, aggregate_log, snapshot_recorded = ARGV\n    ports = {}\n    ports_csv.split(\",\").reject(&:empty?).each do |pair|\n      name, value = pair.split(\"=\", 2)\n      abort \"invalid port pair\" unless name && value && value.match?(/\\A[0-9]+\\z/)\n      ports[name] = Integer(value, 10)\n    end\n    record = {\n      schema: 1, run_id: run_id, target: target, host: host, pid: Integer(pid, 10),\n      owner_identity: owner_identity, status: status, recovery_state: recovery_state,\n      stage: stage, started_at: started_at,\n      pinned_sha: pinned_sha.empty? ? nil : pinned_sha,\n      previous_sha: previous_sha.empty? ? nil : previous_sha,\n      started_services: services.empty? ? [] : services.split(\" \"),\n      ports: ports,\n      restore_order: order_csv.empty? ? [] : order_csv.split(\",\"),\n      run_log: run_log, aggregate_log: aggregate_log,\n      snapshot_recorded: snapshot_recorded == \"1\"\n    }\n    puts JSON.generate(record)\n  ' \"${VPS_DEPLOY_RUN_ID}\" \"${VPS_DEPLOY_RUN_TARGET}\" \"$(hostname -s)\" \\\n    \"${VPS_DEPLOY_LOCK_PID}\" \"${VPS_DEPLOY_LOCK_IDENTITY}\" \\\n    \"${VPS_DEPLOY_RECEIPT_STATUS:-running}\" \"${VPS_DEPLOY_RECEIPT_RECOVERY:-none}\" \\\n    \"${deploy_stage:-start}\" \"${VPS_DEPLOY_RUN_STARTED_AT}\" \\\n    \"${VPS_DEPLOY_RECEIPT_PINNED_SHA:-}\" \"${VPS_DEPLOY_RECEIPT_PREVIOUS_SHA:-}\" \\\n    \"${VPS_DEPLOY_RECEIPT_SERVICES:-}\" \"${VPS_DEPLOY_RECEIPT_PORTS:-}\" \\\n    \"${VPS_DEPLOY_RECEIPT_ORDER:-}\" \\\n    \"${VPS_DEPLOY_RUN_LOG}\" \"${VPS_DEPLOY_LOG}\" \\\n    \"${VPS_DEPLOY_RECEIPT_SNAPSHOT_RECORDED:-0}\") || return 1\n  \"$PUB5_ADMIN\" deploy-run-write \"$VPS_DEPLOY_RUN_ID\" \"$payload\" >/dev/null\n}\n\n# Stage transitions persist the latest recovery breadcrumb before the next\n# stage may stop, replace, migrate or restart a production service.\nset_stage() {\n  deploy_stage=$1\n  log \"enter\"\n  write_deploy_run_receipt || {\n    print -u2 \"vps-deploy: failed to persist recovery receipt at stage $deploy_stage\"\n    return 1\n  }\n}\n# Retry a command with exponential backoff. Use only for transient failures\n# (network, service boot races). Do not wrap git merge conflicts or policy\n# refusals. Logs each attempt. Returns the last exit status.\n#\n#   retry 3 2 curl -fsS http://127.0.0.1:port/up\n#   attempts=3 initial_delay=2s, doubles each time, capped at 30s\nretry() {\n  local max=${1:?} delay=${2:?}\n  shift 2\n  local attempt=1 rc=0\n  while (( attempt <= max )); do\n    if \"$@\"; then\n      (( attempt > 1 )) && log \"retry ok attempt=$attempt cmd=$*\"\n      return 0\n    fi\n    rc=$?\n    if (( attempt == max )); then\n      log \"retry exhausted attempts=$max rc=$rc cmd=$*\"\n      return $rc\n    fi\n    log \"retry fail attempt=$attempt/$max rc=$rc sleep=${delay}s cmd=$*\"\n    sleep $delay\n    delay=$(( delay * 2 ))\n    (( delay > 30 )) && delay=30\n    attempt=$(( attempt + 1 ))\n  done\n  return $rc\n}\n\n# admin_service with retry for start/restart only (not check/list).\nadmin_service_retry() {\n  local action=$1\n  shift || true\n  case $action in\n    start|restart)\n      retry \"${VPS_DEPLOY_SERVICE_RETRIES:-3}\" \"${VPS_DEPLOY_SERVICE_RETRY_DELAY:-2}\" \\\n        admin_service \"$action\" \"$@\"\n      ;;\n    *)\n      admin_service \"$action\" \"$@\"\n      ;;\n  esac\n}\n\nrotate_deploy_log() {\n  # Safety net only. Canonical rotation is newsyslog (OPENBSD/mirror/etc/newsyslog.conf).\n  # This fires if the log grew past the size limit between newsyslog runs.\n  local max_bytes=${VPS_DEPLOY_LOG_MAX_BYTES:-10485760}\n  [[ -f $deploy_log ]] || return 0\n  local size\n  size=$(stat -f %z \"$deploy_log\" 2>/dev/null) || return 0\n  (( size >= max_bytes )) || return 0\n  log \"rotate aggregate log size=$size max=$max_bytes\"\n  mv -f -- \"$deploy_log\" \"$deploy_log.1\" 2>/dev/null || true\n}\n\nif [[ -z ${VPS_DEPLOY_LOGGING:-} ]]; then\n  export VPS_DEPLOY_LOGGING=1\n  mkdir -p -- \"$deploy_log_dir/runs\" 2>/dev/null || {\n    # Fall back to the deploy user's home when /var/log/pub5 is not writable\n    # yet (broker/OPERATOR not installed). Still durable across SSH drops.\n    deploy_log_dir=$DEPLOY_HOME\n    deploy_log=${VPS_DEPLOY_LOG:-$DEPLOY_HOME/vps-deploy.log}\n    mkdir -p -- \"$deploy_log_dir/runs\" 2>/dev/null || true\n  }\n  rotate_deploy_log\n  if [[ -z $deploy_run_log ]]; then\n    deploy_run_log=\"$deploy_log_dir/runs/deploy-$(date -u '+%Y%m%d-%H%M%S')-${app}-$$.log\"\n  fi\n  export VPS_DEPLOY_LOG=$deploy_log\n  export VPS_DEPLOY_RUN_LOG=$deploy_run_log\n  export VPS_DEPLOY_LOG_DIR=$deploy_log_dir\n  # Stable path operators already know about.\n  if [[ -d $DEPLOY_HOME ]]; then\n    ln -sfn \"$deploy_log\" \"$DEPLOY_HOME/vps-deploy.log\" 2>/dev/null || true\n    ln -sfn \"$deploy_run_log\" \"$DEPLOY_HOME/vps-deploy-latest.log\" 2>/dev/null || true\n  fi\n  # Recovery depends on the per-run log retaining the pinned SHA, initial\n  # service snapshot and last entered stage. Do not begin a deploy when that\n  # durable record cannot be opened as a regular file.\n  if ! : >> \"$deploy_run_log\" || [[ ! -f \"$deploy_run_log\" || -L \"$deploy_run_log\" ]]; then\n    print -u2 \"vps-deploy: cannot create a regular per-run recovery log at $deploy_run_log; refusing before any service is touched\"\n    exit 1\n  fi\n  if ! : >> \"$deploy_log\" || [[ ! -f \"$deploy_log\" || -L \"$deploy_log\" ]]; then\n    print -u2 \"vps-deploy: cannot create a regular aggregate log at $deploy_log; refusing before any service is touched\"\n    exit 1\n  fi\n  # Tee stdout/stderr into both the aggregate and the per-run log. process\n  # substitution keeps a single stream so line order stays honest.\n  exec > >(tee -a \"$deploy_log\" \"$deploy_run_log\") 2>&1\n  set_stage start\n  log \"start target=$app uid=$(id -u) user=$(id -un) log=$deploy_log run_log=$deploy_run_log\"\n  log \"argv: $*\"\nfi\n\n# Transitional adapter: use the fixed root-owned broker once installed. The\n# fallback remains until the operator sync has installed it on vm23.\nadmin_service() {\n  local action=${1:-}\n  shift || true\n  if [[ $action == list-started ]]; then\n    \"$PUB5_ADMIN\" service list-started\n  else\n    \"$PUB5_ADMIN\" service \"$action\" \"$@\"\n  fi\n}\n\nadmin_request() {\n  [[ -x \"$PUB5_ADMIN\" ]] || {\n    print -u2 \"vps-deploy: pub5-admin is not installed; run the reviewed OPERATOR sync first\"\n    return 127\n  }\n  \"$PUB5_ADMIN\" \"$@\"\n}\n\n# The guard's deploy flag, held for the whole deploy and not only the restart.\n#\n# rc.d/<app> touches .deploying-<app> from rc_pre to the end of its /up wait and\n# removes it there. CI, migrate, precompile and the checks all run outside that\n# window at the box's highest load, and resource_guard counts every one of those\n# ticks as a strike. So the counter already stands past GUARD_SHED_STRIKES when\n# the restart's flag comes off, and the next tick sheds the first OPTIONAL\n# service. bsdports leads that list, which is why its own deploy exits 0 and\n# vps-state then reads bsdports(failed) with :47312 closed.\n#\n# Taken after the preflight guard run, which may still shed to free memory for\n# CI. A tick that sees the flag resets the counter, so the guard needs two fresh\n# breaching ticks after this deploy ends before it sheds anything. The name is\n# not .deploying-<app>, because rc.d removes that one mid-deploy; it is touched\n# again at each long stage because the guard ignores a flag older than 30 minutes.\ndeploy_flag=\"$repo/.deploying-vps-deploy-${app}\"\nhold_deploy_flag() { touch \"$deploy_flag\" 2>/dev/null || true }\n\n# The stage touches alone did not hold it: brgen's CI ran 34 minutes between two of\n# them on 2026-10-27, the flag aged out at 30, and the guard's emergency stop fired\n# mid-deploy. A keeper touches it every 4 minutes for as long as this process lives;\n# finish() stops it before removing the flag, and it exits on its own if we die.\nflag_keeper=\nkeep_deploy_flag() {\n  hold_deploy_flag\n  ( while kill -0 $$ 2>/dev/null; do touch \"$deploy_flag\" 2>/dev/null; sleep 240; done ) &!\n  flag_keeper=$!\n}\n\n# The terminal result, recorded however the script ends.\n#\n# A failure past `rcctl stop` used to leave the app stopped under a stamp that\n# still said ok, and an ssh disconnect cut the tee pipe before the last line\n# reached the log. Every exit now lands here: it stamps failed (unless a stage\n# already did), names the stage and the services left down, prints the safe next\n# command, and appends the verdict straight to the log file rather than through\n# tee. Each step is `|| true`: a trap that fails would hide the failure it reports.\ndeploy_stage=start\napp_stopped=0\nstamp_written=0\nhalted_app=\npreflight_active=0\npreflight_back_to=\ntypeset -a preflight_back_flag own_stops\n\n# Services the deploy needs to put back, and how it waits for them.\n#\n# On 2026-10-12 a halted `all` pass left brgen, amber and bsdports down for minutes\n# until the operator typed rcctl start by hand: the restore sat at the end of a pass\n# that never got there. The snapshot of started services is taken before anything is\n# touched (VPS_DEPLOY_STARTED, exported so the children see it), and the exit trap\n# of the top-level process puts back whatever is missing, one service at a time.\n# Falcon needs 30-40 s to boot on this box, and two booting at once is what shed\n# the apps in the first place, so each service answers /up before the next starts.\ntypeset -A deploy_ports\nload_deploy_ports() {\n  (( $#deploy_ports )) && return 0\n  local line\n  local -a rows\n  rows=(${(f)\"$(\"$RUBY\" -r\"$repo/OPENBSD/lib/deploy_inventory\" \\\n    -e 'Deploy::Inventory.new(root: ARGV[0]).apps.each { |a| puts \"\#{a.name} \#{a.port}\" }' \"$repo\" 2>/dev/null)\"})\n  for line in $rows; do\n    deploy_ports[${line%% *}]=${line##* }\n  done\n  return 0\n}\n\n# Bounded wait for http://127.0.0.1:PORT/up. Returns 1 when the limit passes.\nwait_up() {\n  local port=$1 limit=${2:-${VPS_DEPLOY_RESTORE_WAIT:-240}} step=${VPS_DEPLOY_POLL:-3} waited=0\n  while (( waited < limit )); do\n    if curl -fsS --max-time 5 \"http://127.0.0.1:${port}/up\" >/dev/null 2>&1; then\n      (( waited > 0 )) && log \"wait_up ok port=$port after=${waited}s\"\n      return 0\n    fi\n    sleep $step\n    waited=$(( waited + step ))\n  done\n  log \"wait_up timeout port=$port limit=${limit}s\"\n  return 1\n}\n\n# Every service that was started and is running must answer /up before the next\n# memory-heavy step (CI, bundle, precompile). A service that is down is not booting\n# and is not waited for; one that never answers is reported, not waited on forever.\nsettle_boots() {\n  local svc port\n  load_deploy_ports\n  for svc in ${=VPS_DEPLOY_STARTED:-}; do\n    port=${deploy_ports[$svc]:-}\n    [[ -n $port ]] || continue\n    admin_service check \"$svc\" >/dev/null 2>&1 || continue\n    wait_up \"$port\" \"${VPS_DEPLOY_SETTLE_WAIT:-240}\" \\\n      || print -u2 \"vps-deploy: WARNING - $svc is running but :${port}/up did not answer within ${VPS_DEPLOY_SETTLE_WAIT:-240}s\"\n  done\n  return 0\n}\n\n# Who stopped it: this deploy, resource_guard (its shed ledger), or neither.\nstop_origin() {\n  local svc=$1 shed_file=${VPS_DEPLOY_SHED_FILE:-/var/db/resource_guard_shed}\n  if [[ $svc == $halted_app ]] || (( ${own_stops[(Ie)$svc]} )); then\n    print \"stopped by this deploy\"\n  elif [[ -r $shed_file ]] && (( ${${(f)\"$(<$shed_file)\"}[(Ie)$svc]} )); then\n    print \"shed by resource_guard\"\n  else\n    print \"down for another reason (crashed or stopped outside the deploy)\"\n  fi\n}\n\n# Starts, one at a time and in serving order, every service that was started when\n# the deploy began and is not running now. Returns 1 if any would not come back.\nrestore_started() {\n  [[ -n ${VPS_DEPLOY_STARTED:-} ]] || return 0\n  if [[ ${VPS_DEPLOY_RESTORE_PORTS_FROM_RECEIPT:-0} != 1 ]]; then\n    load_deploy_ports\n  fi\n  local -a want order failed restored\n  local svc port origin\n  want=( ${=VPS_DEPLOY_STARTED} )\n  for svc in $DEPLOY_ALL_DEGRADED; do (( ${want[(Ie)$svc]} )) && order+=($svc); done\n  for svc in $want; do (( ${order[(Ie)$svc]} )) || order+=($svc); done\n  for svc in $order; do\n    admin_service check \"$svc\" >/dev/null 2>&1 && continue\n    origin=$(stop_origin \"$svc\")\n    print \"vps-deploy: restore $svc ($origin)\"\n    port=${deploy_ports[$svc]:-}\n    # restart, not start: rc.d/master runs its relayd restart, TTS warmup and flag\n    # removal only on restart, and restart on a stopped service is a start.\n    admin_service_retry restart \"$svc\" >/dev/null 2>&1 || true\n    if [[ -n $port ]]; then\n      wait_up \"$port\" || { admin_service_retry restart \"$svc\" >/dev/null 2>&1 || true; wait_up \"$port\"; } || {\n        print -u2 \"vps-deploy: restore $svc FAILED - :${port}/up not ready, read /var/log/daemon\"\n        failed+=($svc); continue\n      }\n    else\n      sleep 3\n      admin_service check \"$svc\" >/dev/null 2>&1 || { print -u2 \"vps-deploy: restore $svc FAILED - rcctl check\"; failed+=($svc); continue }\n    fi\n    print \"vps-deploy: restore $svc ok\"\n    restored+=($svc)\n  done\n  (( $#restored )) && print \"vps-deploy: restored ${restored}\"\n  (( $#failed )) && print -u2 \"vps-deploy: STILL DOWN after restore: ${failed}  (/usr/local/sbin/pub5-admin service restart <app>, one at a time)\"\n  (( $#failed == 0 ))\n}\n\n# A signal or shell exit can arrive while the box is detached for preflight.\n# Keep the original ref as explicit recovery state until it is restored.\nrestore_preflight_checkout() {\n  (( preflight_active )) || return 0\n  [[ -n $preflight_back_to ]] || {\n    print -u2 \"vps-deploy: preflight checkout recovery has no saved ref\"\n    return 1\n  }\n  git_as_deploy_user switch -q \"${preflight_back_flag[@]}\" \"$preflight_back_to\" || {\n    print -u2 \"vps-deploy: FAILED to restore checkout ref $preflight_back_to after preflight\"\n    return 1\n  }\n  preflight_active=0\n  log \"preflight checkout restored ref=$preflight_back_to\"\n}\n\n# Vendorables are not tracked in git (TREE_CONTRACTS.md, 8 November 2026):\n# master's smart-turn runtime bytes are fetched when absent, with pinned\n# versions and sha256-verified digests. Failure leaves ?smart_turn=1\n# unavailable — the feature is default-off — and the deploy continues.\nensure_smart_turn_assets() {\n  local vendor=\"$repo/MASTER/web/public/vendor/onnxruntime\"\n  local model=\"$repo/MASTER/web/public/models/smart-turn-v3.2-cpu.onnx\"\n  [[ -f $vendor/ort-wasm-simd-threaded.wasm && -f $vendor/ort.wasm.min.mjs && -f $model ]] && return 0\n  print \"vps-deploy: smart_turn assets absent; fetching pinned, checksummed copies\"\n  if ! run_as_deploy_user \"cd $(printf '%q' \"$repo/MASTER/web\") && sh script/build_smart_turn.sh\"; then\n    print -u2 \"vps-deploy: smart_turn fetch failed; ?smart_turn=1 stays unavailable (default-off) until MASTER/web/script/build_smart_turn.sh succeeds\"\n  fi\n}\n\nfinish() {\n  local rc=$?\n  trap - EXIT\n  if (( preflight_active )); then\n    restore_preflight_checkout || {\n      print -u2 \"vps-deploy: checkout remains in preflight recovery state; restore ref $preflight_back_to before another deploy\"\n      (( rc == 0 )) && rc=1\n    }\n  fi\n  if (( rc != 0 )) && [[ $deploy_target_kind == all || $deploy_target_kind == group ]]; then\n    print -u2 \"vps-deploy: $app FAILED at stage \\\"$deploy_stage\\\" (exit $rc)\"\n  fi\n  if (( rc != 0 )) && [[ $deploy_target_kind != all && $deploy_target_kind != group && $deploy_target_kind != unknown ]]; then\n    if (( ! stamp_written )) && (( $+functions[write_stamp] )); then\n      write_stamp \"$app\" failed || true\n    fi\n    print -u2 \"vps-deploy: $app FAILED at stage \\\"$deploy_stage\\\" (exit $rc)\"\n    if (( app_stopped )); then\n      print -u2 \"vps-deploy: $app was stopped for this deploy and is DOWN - read /var/log/daemon, then\"\n      print -u2 \"            /usr/local/sbin/pub5-admin service start $app   (or rerun: doas zsh OPENBSD/bin/vps-deploy $app)\"\n    else\n      print -u2 \"vps-deploy: $app was not stopped; the previous release is still serving. Fix the cause and rerun:\"\n      print -u2 \"            doas zsh OPENBSD/bin/vps-deploy $app\"\n    fi\n    local other\n    for other in $DEPLOY_RAILS; do\n      [[ $other == \"$app\" ]] && continue\n      admin_service get \"$other\" >/dev/null 2>&1 || continue\n      admin_service check \"$other\" >/dev/null 2>&1 \\\n        || print -u2 \"vps-deploy: $other is DOWN (shed or stopped) - /usr/local/sbin/pub5-admin service restart $other\"\n    done\n  fi\n  # Only the process that owns the lock restores: the children of `all` fail into\n  # their parent, which knows the whole pass. Each restore result is printed.\n  local restore_ok=1\n  if (( rc != 0 )) && (( lock_owner )) && [[ ${VPS_DEPLOY_RESTORE_ATTEMPTED:-0} != 1 ]]; then\n    restore_started || {\n      restore_ok=0\n      print -u2 \"vps-deploy: production is NOT fully restored - see the lines above\"\n    }\n  fi\n  # A normal EXIT/TERM/HUP reaches this terminal update; SIGKILL/power loss\n  # leaves the last durable receipt pending for an explicit operator recovery.\n  if (( lock_owner )) && [[ -n ${VPS_DEPLOY_RUN_ID:-} ]]; then\n    VPS_DEPLOY_RECEIPT_STATUS=failed\n    (( rc == 0 )) && VPS_DEPLOY_RECEIPT_STATUS=ok\n    if [[ -n ${VPS_DEPLOY_STARTED:-} ]]; then\n      if (( rc == 0 || restore_ok )); then\n        VPS_DEPLOY_RECEIPT_RECOVERY=complete\n      else\n        VPS_DEPLOY_RECEIPT_RECOVERY=pending\n      fi\n    else\n      VPS_DEPLOY_RECEIPT_RECOVERY=none\n    fi\n    export VPS_DEPLOY_RECEIPT_STATUS VPS_DEPLOY_RECEIPT_RECOVERY\n    write_deploy_run_receipt || {\n      print -u2 \"vps-deploy: FAILED to persist terminal recovery receipt\"\n      (( rc == 0 )) && rc=1\n    }\n  fi\n  # Keep resource_guard excluded until every service restored by this failure\n  # has answered, including slow boots; only then stop the heartbeat and remove\n  # this process's deploy flag.\n  [[ -n $flag_keeper ]] && kill $flag_keeper 2>/dev/null\n  rm -f -- \"$deploy_flag\"\n  log \"finished exit=$rc\"\n  # Write the verdict straight to the files as well: if the tee pipe died with\n  # the ssh session the last structured line can still land on disk.\n  {\n    print -r -- \"$(log_ts) host=$(hostname -s) pid=$$ app=$app stage=$deploy_stage finished exit=$rc\"\n  } >> \"${deploy_log:-/var/log/pub5/vps-deploy.log}\" 2>/dev/null || true\n  if [[ -n ${deploy_run_log:-${VPS_DEPLOY_RUN_LOG:-}} ]]; then\n    print -r -- \"$(log_ts) host=$(hostname -s) pid=$$ app=$app stage=$deploy_stage finished exit=$rc\" \\\n      >> \"${deploy_run_log:-$VPS_DEPLOY_RUN_LOG}\" 2>/dev/null || true\n  fi\n  release_deploy_lock\n  exit $rc\n}\n# zsh runs the EXIT trap for HUP but not for TERM or INT, so name all three: a\n# dropped ssh session or a kill is a failure with a recorded stage, not a silent end.\ntrap finish EXIT\ntrap 'exit 129' HUP\ntrap 'exit 130' INT\ntrap 'exit 143' TERM\n\n# One durable receipt belongs to the lock-owning parent. Child app deploys\n# inherit the ID, snapshot and owner identity and update only the current stage.\nif (( lock_owner )) && [[ $app != recover ]]; then\n  unset VPS_DEPLOY_RUN_ID VPS_DEPLOY_RUN_TARGET VPS_DEPLOY_RUN_STARTED_AT\n  unset VPS_DEPLOY_STARTED\n  unset VPS_DEPLOY_RECEIPT_STATUS VPS_DEPLOY_RECEIPT_RECOVERY\n  unset VPS_DEPLOY_RECEIPT_PINNED_SHA VPS_DEPLOY_RECEIPT_PREVIOUS_SHA\n  unset VPS_DEPLOY_RECEIPT_SERVICES VPS_DEPLOY_RECEIPT_PORTS VPS_DEPLOY_RECEIPT_ORDER\n  unset VPS_DEPLOY_RECEIPT_SNAPSHOT_RECORDED\n  VPS_DEPLOY_RUN_ID=\"$(date -u '+%Y%m%dT%H%M%SZ')-$app-${$}\"\n  VPS_DEPLOY_RUN_TARGET=$app\n  VPS_DEPLOY_RUN_STARTED_AT=\"$(date -u '+%Y-%m-%dT%H:%M:%SZ')\"\n  VPS_DEPLOY_RECEIPT_STATUS=running\n  VPS_DEPLOY_RECEIPT_RECOVERY=none\n  VPS_DEPLOY_RECEIPT_PINNED_SHA=\n  VPS_DEPLOY_RECEIPT_PREVIOUS_SHA=\n  VPS_DEPLOY_RECEIPT_SERVICES=\n  VPS_DEPLOY_RECEIPT_PORTS=\n  VPS_DEPLOY_RECEIPT_ORDER=\n  VPS_DEPLOY_RECEIPT_SNAPSHOT_RECORDED=0\n  export VPS_DEPLOY_RECEIPT_SNAPSHOT_RECORDED\n  export VPS_DEPLOY_RUN_ID VPS_DEPLOY_RUN_TARGET VPS_DEPLOY_RUN_STARTED_AT\n  export VPS_DEPLOY_RECEIPT_STATUS VPS_DEPLOY_RECEIPT_RECOVERY\n  export VPS_DEPLOY_RECEIPT_PINNED_SHA VPS_DEPLOY_RECEIPT_PREVIOUS_SHA\n  export VPS_DEPLOY_RECEIPT_SERVICES VPS_DEPLOY_RECEIPT_PORTS VPS_DEPLOY_RECEIPT_ORDER\n  write_deploy_run_receipt || {\n    print -u2 \"vps-deploy: cannot create the durable recovery receipt; refusing before any service is touched\"\n    exit 1\n  }\nfi\n\n# A parent can be killed while a child app deploy is still unwinding. The\n# receipt identifies the parent, so also refuse while another deployer process\n# is visible. Exclude this recovery command by its own PID.\nother_deploy_processes() {\n  local pid exclude_pid=${1:-} matches rc\n  local -a candidates\n  if matches=$(pgrep -f 'OPENBSD/bin/vps-deploy' 2>/dev/null); then\n    :\n  else\n    rc=$?\n    (( rc == 1 )) || return 1\n  fi\n  candidates=(${(f)matches})\n  for pid in $candidates; do\n    [[ $pid == \"${$}\" || ( -n $exclude_pid && $pid == \"$exclude_pid\" ) ]] || print -r -- \"$pid\"\n  done\n}\n\n# Operator-driven recovery is intentionally separate from deploy. It never pulls,\n# checks out code, runs migrations or replays a deploy; it restores only the\n# service set recorded as running before the interrupted run.\nif [[ $app == recover ]]; then\n  runs_json=$(admin_request deploy-run-list) || {\n    print -u2 \"vps-deploy: cannot list recovery receipts; install and verify the pub5-admin mirror first\"\n    exit 1\n  }\n  if [[ $recover_mode == plan ]]; then\n    report=$(\"$RUBY\" -rjson -e '\n      rows = JSON.parse(STDIN.read)\n      rows.select { |r| r[\"recovery_state\"] == \"pending\" && %w[running failed].include?(r[\"status\"]) }.each do |r|\n        puts [r[\"run_id\"], r[\"target\"], r[\"pinned_sha\"] || \"not-pinned\",\n              r[\"stage\"], r[\"status\"], r[\"pid\"], r[\"owner_identity\"],\n              r[\"host\"], r[\"snapshot_recorded\"], r[\"run_log\"]].join(\"|\")\n      end\n    ' <<< \"$runs_json\") || exit 1\n    if [[ -z $report ]]; then\n      print \"vps-deploy: no pending interrupted-deploy recovery receipts\"\n      exit 0\n    fi\n    while IFS='|' read -r run_id target sha stage status owner_pid owner_identity owner_host snapshot_recorded run_log; do\n      [[ -n $run_id ]] || continue\n      if [[ $owner_host != \"$(hostname -s)\" ]]; then\n        print \"vps-deploy: FOREIGN $run_id host=$owner_host target=$target - recover on the recorded host\"\n        continue\n      fi\n      if [[ $snapshot_recorded != true ]]; then\n        print \"vps-deploy: INCOMPLETE $run_id target=$target stage=$stage - service snapshot is not complete; do not apply\"\n        continue\n      fi\n      other_pids=$(other_deploy_processes \"$owner_pid\") || {\n        print \"vps-deploy: AMBIGUOUS $run_id - cannot verify the process table; do not recover\"\n        continue\n      }\n      if [[ -n $other_pids ]]; then\n        print \"vps-deploy: AMBIGUOUS $run_id - another deployer process is still present; do not recover\"\n        print -r -- \"  other deployer PID(s): $other_pids\"\n        continue\n      fi\n      if ps -p \"$owner_pid\" >/dev/null 2>&1; then\n        current_identity=$(lock_process_identity \"$owner_pid\") || {\n          print \"vps-deploy: AMBIGUOUS $run_id - owner PID exists but its identity cannot be checked\"\n          continue\n        }\n        if [[ $current_identity == \"$owner_identity\" ]]; then\n          print \"vps-deploy: LIVE $run_id target=$target sha=$sha stage=$stage pid=$owner_pid - do not recover\"\n          continue\n        fi\n      fi\n      print \"vps-deploy: RECOVERY AVAILABLE $run_id target=$target sha=$sha stage=$stage\"\n      print \"  log=$run_log\"\n    done <<< \"$report\"\n    print \"vps-deploy: inspect the receipt and its log before: zsh OPENBSD/bin/vps-deploy recover --apply <run-id>\"\n    exit 0\n  fi\n\n  receipt_json=$(admin_request deploy-run-read \"$recover_id\") || exit 1\n  [[ $receipt_json != null ]] || {\n    print -u2 \"vps-deploy: no recovery receipt found for $recover_id\"\n    exit 1\n  }\n  details=$(\"$RUBY\" -rjson -e '\n    r = JSON.parse(STDIN.read)\n    abort \"receipt is not pending recovery\" unless\n      r[\"recovery_state\"] == \"pending\" && %w[running failed].include?(r[\"status\"])\n    abort \"receipt snapshot is incomplete\" unless r[\"snapshot_recorded\"] == true\n    puts [\"META\", r.fetch(\"pid\"), r.fetch(\"owner_identity\"), r.fetch(\"stage\"),\n          r.fetch(\"target\"), r[\"pinned_sha\"] || \"not-pinned\", r.fetch(\"host\")].join(\"|\")\n    puts [\"SERVICES\", r.fetch(\"started_services\").join(\" \")].join(\"|\")\n    puts [\"ORDER\", r.fetch(\"restore_order\").join(\" \")].join(\"|\")\n    r.fetch(\"ports\").sort.each { |service, port| puts [\"PORT\", service, port].join(\"|\") }\n  ' <<< \"$receipt_json\") || {\n    print -u2 \"vps-deploy: receipt is corrupt, incomplete or not eligible for recovery\"\n    exit 1\n  }\n\n  typeset -a recovery_lines\n  recovery_lines=(${(f)details})\n  IFS='|' read -r kind owner_pid owner_identity original_stage original_target original_sha owner_host <<< \"$recovery_lines[1]\"\n  [[ $owner_host == \"$(hostname -s)\" ]] || {\n    print -u2 \"vps-deploy: receipt belongs to $owner_host; recovery must run on that host\"\n    exit 1\n  }\n  other_pids=$(other_deploy_processes \"$owner_pid\") || {\n    print -u2 \"vps-deploy: cannot verify the process table; refusing recovery\"\n    exit 75\n  }\n  [[ -z $other_pids ]] || {\n    print -u2 \"vps-deploy: another deployer process is still present; refusing recovery\"\n    print -u2 \"  other deployer PID(s): $other_pids\"\n    exit 75\n  }\n  if ps -p \"$owner_pid\" >/dev/null 2>&1; then\n    current_identity=$(lock_process_identity \"$owner_pid\") || {\n      print -u2 \"vps-deploy: owner PID exists but its identity cannot be checked; refusing recovery\"\n      exit 75\n    }\n    if [[ $current_identity == \"$owner_identity\" ]]; then\n      print -u2 \"vps-deploy: original deploy $recover_id is still running; refusing recovery\"\n      exit 75\n    fi\n  fi\n\n  VPS_DEPLOY_STARTED=\n  DEPLOY_ALL_DEGRADED=()\n  deploy_ports=()\n  for ((i=2; i<=${#recovery_lines}; i++)); do\n    IFS='|' read -r kind one two <<< \"$recovery_lines[$i]\"\n    case $kind in\n      SERVICES) VPS_DEPLOY_STARTED=$one ;;\n      ORDER) DEPLOY_ALL_DEGRADED=(${=one}) ;;\n      PORT) deploy_ports[$one]=$two ;;\n    esac\n  done\n  [[ -n $VPS_DEPLOY_STARTED ]] || {\n    print -u2 \"vps-deploy: receipt has no captured running services; refusing an inferred restore\"\n    exit 1\n  }\n  export VPS_DEPLOY_STARTED\n  export VPS_DEPLOY_RESTORE_PORTS_FROM_RECEIPT=1\n  VPS_DEPLOY_RESTORE_ATTEMPTED=1\n\n  print \"vps-deploy: recovery target=$original_target pinned=$original_sha failed_stage=$original_stage\"\n  print \"vps-deploy: only the recorded service snapshot will be restored\"\n  keep_deploy_flag\n  recovery_payload=$(\"$RUBY\" -rjson -e 'puts JSON.generate(status: \"failed\", recovery_state: \"pending\", stage: \"recovery-restore\")')\n  admin_request deploy-run-write \"$recover_id\" \"$recovery_payload\" || {\n    print -u2 \"vps-deploy: cannot persist recovery-attempt stage; no services were changed\"\n    exit 1\n  }\n\n  recovery_failed=0\n  restore_started || recovery_failed=1\n  if (( ! recovery_failed )); then\n    restore_wait=${VPS_DEPLOY_RESTORE_WAIT:-240}\n    for svc in ${=VPS_DEPLOY_STARTED}; do\n      admin_service check \"$svc\" >/dev/null 2>&1 || {\n        print -u2 \"vps-deploy: recovery verification failed for $svc (rcctl check)\"\n        recovery_failed=1\n        break\n      }\n      port=${deploy_ports[$svc]:-}\n      if [[ -n $port ]] && ! wait_up \"$port\" \"$restore_wait\"; then\n        print -u2 \"vps-deploy: recovery verification failed for $svc /up on port $port\"\n        recovery_failed=1\n        break\n      fi\n    done\n  fi\n\n  if (( recovery_failed )); then\n    recovery_payload=$(\"$RUBY\" -rjson -e 'puts JSON.generate(status: \"failed\", recovery_state: \"pending\", stage: \"recovery-failed\")')\n    admin_request deploy-run-write \"$recover_id\" \"$recovery_payload\" || \\\n      print -u2 \"vps-deploy: recovery failed and the pending receipt could not be updated\"\n    print -u2 \"vps-deploy: recovery remains pending; inspect /var/log/daemon before another attempt\"\n    exit 1\n  fi\n\n  recovery_payload=$(\"$RUBY\" -rjson -e 'puts JSON.generate(status: \"recovered\", recovery_state: \"complete\", stage: \"recovered\", recovered_at: Time.now.utc.iso8601)')\n  admin_request deploy-run-write \"$recover_id\" \"$recovery_payload\" || {\n    print -u2 \"vps-deploy: services answered, but the recovery receipt could not be finalized\"\n    exit 1\n  }\n  print \"vps-deploy: recovered $recover_id target=$original_target pinned=$original_sha (previous stage=$original_stage)\"\n  exit 0\nfi\n\n# Pin the commit, prove it boots, record what is running.\n#\n# Done once, by the process that owns the lock, before any service is touched.\n#\n# The deploy used to pull origin/main at whatever instant each step reached its\n# pull, so a commit pushed mid-pass was deployed half-way through it. On 2026-10-12\n# the pulled tip carried a SyntaxError and later a law that raised at boot; master's\n# container died on the box and the pass halted with production already shed. Now\n# the target is one SHA, resolved here (VPS_DEPLOY_SHA, or origin/main fetched\n# once), and every later step checks HEAD against it instead of pulling.\n#\n# The checks (MASTER/lib/operator/canary.rb) ran on the operator's\n# machine when `bin/operator vps deploy --remote` started this run and said so with\n# VPS_DEPLOY_PREFLIGHT_SHA; a direct run on the box has no such word and runs the\n# MASTER-only subset itself on the pinned commit, detached, so a refusal leaves the\n# checkout where it was. VPS_DEPLOY_SKIP_PREFLIGHT=1 is break-glass and, like the\n# runtime-check skip, needs the human-only I_UNDERSTAND_FAST_DEPLOY=1.\ndeploy_refuse() { log \"refused: $1\"; print -u2 \"vps-deploy: refused - $1\"; print -u2 \"            Nothing was stopped.\"; exit 1 }\nif (( lock_owner )); then\n  set_stage pin\n    retry \"${VPS_DEPLOY_GIT_RETRIES:-3}\" 2 git_as_deploy_user fetch -q origin main \\\n    || deploy_refuse \"git fetch origin main failed (as $DEPLOY_USER) after retries\"\n  pin=$(git_as_deploy_user rev-parse --verify -q \"${VPS_DEPLOY_SHA:-FETCH_HEAD}^{commit}\") \\\n    || deploy_refuse \"${VPS_DEPLOY_SHA:-origin/main} is not a commit this checkout can see\"\n  head_before=$(git_as_deploy_user rev-parse HEAD)\n  git_as_deploy_user merge-base --is-ancestor \"$head_before\" \"$pin\" \\\n    || deploy_refuse \"$pin is not a fast-forward of the box's HEAD $head_before\"\n  export VPS_DEPLOY_SHA=$pin\n  print \"vps-deploy: pinned $pin (box HEAD was $head_before; git as $DEPLOY_USER)\"\n  # A CI attestation names one commit; tests that passed on another prove nothing here.\n  if [[ -n ${VPS_DEPLOY_CI_SHA:-} ]]; then\n    [[ $VPS_DEPLOY_CI_SHA == $pin ]] \\\n      || deploy_refuse \"bin/ci ran on $VPS_DEPLOY_CI_SHA but this run would deploy $pin\"\n    print \"vps-deploy: bin/ci passed on the operator's machine for $pin; the box skips its suite\"\n  fi\n  if [[ -n ${VPS_DEPLOY_PREFLIGHT_SHA:-} ]]; then\n    [[ $VPS_DEPLOY_PREFLIGHT_SHA == $pin ]] \\\n      || deploy_refuse \"preflight ran on $VPS_DEPLOY_PREFLIGHT_SHA but this run would deploy $pin\"\n    print \"vps-deploy: preflight passed on the operator's machine for $pin\"\n  elif [[ ${VPS_DEPLOY_SKIP_PREFLIGHT:-} == 1 ]]; then\n    [[ ${I_UNDERSTAND_FAST_DEPLOY:-} == 1 ]] \\\n      || deploy_refuse \"VPS_DEPLOY_SKIP_PREFLIGHT=1 needs the human-only I_UNDERSTAND_FAST_DEPLOY=1\"\n    print \"vps-deploy: WARNING - preflight skipped (break-glass); $pin was not proven to boot\"\n  else\n    set_stage preflight-boot\n    print \"vps-deploy: preflight on $pin (syntax, law files, boot smoke)\"\n    preflight_back_to=$(git_as_deploy_user symbolic-ref -q --short HEAD || print -r -- \"$head_before\")\n    # A box left detached comes back to its commit, not to a guessed branch.\n    preflight_back_flag=()\n    git_as_deploy_user symbolic-ref -q HEAD >/dev/null || preflight_back_flag=(--detach)\n    # Set recovery state before switching: the EXIT/HUP/INT/TERM path now owns\n    # restoration even if the signal lands immediately after the detach.\n    preflight_active=1\n    git_as_deploy_user switch -q --detach \"$pin\" || deploy_refuse \"cannot check out $pin in $repo (local changes?)\"\n    if ! \"$RUBY\" \"$repo/MASTER/lib/operator/canary.rb\" --no-rehearse; then\n      restore_preflight_checkout || deploy_refuse \"preflight failed on $pin and checkout restoration failed; restore $preflight_back_to before retrying\"\n      deploy_refuse \"preflight failed on $pin; the checkout is back on $preflight_back_to and the previous release is still serving\"\n    fi\n    restore_preflight_checkout || deploy_refuse \"cannot return to $preflight_back_to after preflight\"\n  fi\n  git_as_deploy_user merge -q --ff-only \"$pin\" || deploy_refuse \"cannot fast-forward $repo to $pin\"\n  VPS_DEPLOY_STARTED=\"$(admin_service list-started 2>/dev/null)\"\n  VPS_DEPLOY_STARTED=${VPS_DEPLOY_STARTED//$'\\n'/ }\n  export VPS_DEPLOY_STARTED\n  VPS_DEPLOY_RECEIPT_PREVIOUS_SHA=$head_before\n  VPS_DEPLOY_RECEIPT_PINNED_SHA=$pin\n  VPS_DEPLOY_RECEIPT_SERVICES=$VPS_DEPLOY_STARTED\n  VPS_DEPLOY_RECEIPT_RECOVERY=none\n  [[ -z $VPS_DEPLOY_STARTED ]] || VPS_DEPLOY_RECEIPT_RECOVERY=pending\n  export VPS_DEPLOY_RECEIPT_PREVIOUS_SHA VPS_DEPLOY_RECEIPT_PINNED_SHA\n  export VPS_DEPLOY_RECEIPT_SERVICES VPS_DEPLOY_RECEIPT_RECOVERY\n  print \"vps-deploy: started before this deploy: $VPS_DEPLOY_STARTED\"\n  set_stage snapshot\nfi\n\n  # Resolve against the inventory in the checkout after it has been fast-forwarded\n  # to VPS_DEPLOY_SHA. A stale box checkout must not define target membership.\n  deploy_target_kind=$(inventory_target_kind \"$app\") || deploy_refuse \"could not read the pinned deployment inventory for $app\"\n  [[ $deploy_target_kind != unknown ]] || deploy_refuse \"unknown app or group $app in the pinned RAILS/apps.yml\"\n  DEPLOY_ALL=(${(@f)$(inventory_names all)})\n  DEPLOY_ALL_DEGRADED=(${(@f)$(inventory_names all 1)})\n  DEPLOY_RAILS=(${(@f)$(inventory_names rails)})\n  load_deploy_ports\n  typeset -a receipt_ports\n  for svc in ${(k)deploy_ports}; do\n    receipt_ports+=(\"$svc=${deploy_ports[$svc]}\")\n  done\n  VPS_DEPLOY_RECEIPT_PORTS=\"${(j:,:)receipt_ports}\"\n  VPS_DEPLOY_RECEIPT_ORDER=\"${(j:,:)DEPLOY_ALL_DEGRADED}\"\n  VPS_DEPLOY_RECEIPT_SNAPSHOT_RECORDED=1\n  export VPS_DEPLOY_RECEIPT_PORTS VPS_DEPLOY_RECEIPT_ORDER\n  export VPS_DEPLOY_RECEIPT_SNAPSHOT_RECORDED\n  set_stage inventory-ready\n  (( ${#DEPLOY_ALL} )) || deploy_refuse \"the pinned deployment inventory has no active apps\"\n  deploy_app_group=$(inventory_app_group \"$app\")\n  if [[ $deploy_target_kind == all || $deploy_app_group == master ]]; then\n    ensure_smart_turn_assets\n  fi\n\n# The deployment sequence is data in RAILS/apps.yml. Degraded mode\n# moves the MASTER namespace last; all other relative order stays unchanged.\nchoose_deploy_order() {\n  local svc port avail why=\n  load_deploy_ports\n  avail=$(\"$RUBY\" -e 'print File.readlines(ARGV[0]).last.to_s[/mem_avail=(\\d+)/, 1].to_s' \\\n    \"${VPS_DEPLOY_GUARD_LOG:-/var/log/resource_guard_history.log}\" 2>/dev/null)\n  for svc in $DEPLOY_RAILS; do\n    (( ${${=VPS_DEPLOY_STARTED:-}[(Ie)$svc]} )) || continue\n    port=${deploy_ports[$svc]:-}\n    [[ -n $port ]] && wait_up \"$port\" 5 || why=\"$svc is not answering /up\"\n  done\n  if [[ -z $why ]]; then\n    [[ $avail == <-> ]] || why=\"memory availability unreadable\"\n  fi\n  if [[ -z $why ]] && (( avail < ${VPS_DEPLOY_MASTER_MIN_AVAIL:-10} )); then\n    why=\"memory availability ${avail}% is under ${VPS_DEPLOY_MASTER_MIN_AVAIL:-10}%\"\n  fi\n  if [[ -z $why ]]; then\n    deploy_order=($DEPLOY_ALL)\n  else\n    print \"vps-deploy: all - MASTER moved to the end: $why\"\n    deploy_order=($DEPLOY_ALL_DEGRADED)\n  fi\n}\n\nif [[ $deploy_target_kind == all || $deploy_target_kind == group ]]; then\n  # The parent spans child deploys and cooldowns. Its own flag keeps the\n  # resource guard out until the whole batch (or its failure recovery) ends.\n  keep_deploy_flag\n  typeset -i first=1\n  typeset -a deploy_order\n  if [[ $deploy_target_kind == all ]]; then\n    choose_deploy_order\n  else\n    deploy_order=(${(@f)$(inventory_names \"$app\")})\n  fi\n  (( ${#deploy_order} )) || { print -u2 \"vps-deploy: deployment group $app is empty\"; exit 2 }\n  print \"vps-deploy: $app order: ${deploy_order}\"\n  if [[ $deploy_target_kind == all ]]; then\n    set_stage openbsd-mirror\n    print \"vps-deploy: applying tracked OpenBSD filesystem mirror\"\n    zsh \"$repo/OPENBSD/OPERATOR.sh\" --apply || deploy_refuse \"OpenBSD filesystem mirror apply failed; no application deployment was started\"\n  fi\n  for one in $deploy_order; do\n    # Let the previous app's deploy drain before starting the next one's CI.\n    # Serial builds protect this one-core host from overlapping load peaks.\n    if (( ! first )); then\n      print \"vps-deploy: $app - settling for ${PUB5_DEPLOY_COOLDOWN:-60}s before $one\"\n      sleep ${PUB5_DEPLOY_COOLDOWN:-60}\n    fi\n    first=0\n    print \"vps-deploy: $app -> $one\"\n    # Each child executes the complete per-app CI, sync, migration, restart,\n    # health-check and deploy-stamp path. A failure halts this group.\n    set_stage \"$app:$one\"\n    zsh \"$0\" \"$one\" || { halted_app=$one; print -u2 \"vps-deploy: $app halted at $one\"; exit 1 }\n  done\n  print \"vps-deploy: $app ok (${deploy_order})\"\n  exit 0\nfi\n\n# SHA and time are captured at write_stamp, not here. vps_deploy_master.sh\n# pulls origin/main; reading HEAD before that call stamped the pre-pull SHA\n# (76f1a65fa on 2026-08-12 while /health correctly said 5eff4d87a).\n\nwrite_stamp() {\n  local name=$1 stamp_status=$2\n  local sha\n  stamp_written=1\n  sha=$(git_as_deploy_user rev-parse --short HEAD)\n  admin_request stamp \"$name\" \"$sha\" \"$stamp_status\"\n}\n\nset_stage \"$deploy_app_group\"\nif [[ $deploy_app_group == master ]]; then\n  print \"vps-deploy: master web\"\n  # master's bundle, native gem builds and precompile run as long as an app's CI,\n  # and this branch never reached the keeper below: on 2026-10-27 the guard read\n  # deploy=0 through a 30-minute master deploy and shed amber and bsdports twice.\n  keep_deploy_flag\n  settle_boots\n  if zsh \"$repo/OPENBSD/bin/vps_deploy_master.sh\"; then\n    # The loopback journeys that need no browser, for this app only.\n    set_stage master-flows\n    hold_deploy_flag\n    if ! FLOW_APP=\"$app\" MASTER_STRICT=deploy CHECK_AUTOFIX=0 CHECK_REQUIRE_LIVE=1 \"$RUBY\" \"$repo/MASTER/checks/runner.rb\" flow_journey; then\n      write_stamp \"$app\" failed\n      print -u2 \"vps-deploy: master flow journeys failed - see the per-flow lines above\"\n      exit 1\n    fi\n    set_stage master-canary\n    if ! \"$RUBY\" \"$repo/OPENBSD/bin/canary\" --local --app \"$app\" --no-flows; then\n      write_stamp \"$app\" failed\n      print -u2 \"vps-deploy: master local canary failed - see the failed checks above\"\n      exit 1\n    fi\n    set_stage restore-shed\n    if restore_started; then\n      restore_ok=1\n    else\n      restore_ok=0\n    fi\n    if (( ! restore_ok )); then\n      write_stamp \"$app\" failed\n      print -u2 \"vps-deploy: master is not complete; a service that was running before is still down\"\n      exit 1\n    fi\n    write_stamp \"$app\" ok\n    exit 0\n  fi\n  # A failed run stamps failed, or last_deploy_master.json keeps saying ok\n  # from the last good run and /health reports a live SHA while :53187 is\n  # closed (measured 2026-08-13).\n  write_stamp \"$app\" failed\n  exit 1\nfi\n\n[[ -d /home/${app}/app ]] || { print -u2 \"missing /home/${app}/app\"; exit 1 }\n\n# Rails apps do not pull inside vps_ci.sh, so without this step a deploy CIs and\n# syncs whatever the VPS checkout already holds, which can lag origin/main by\n# days. master pulls inside vps_deploy_master.sh; this is the matching step for\n# the three copy-tree apps.\nset_stage pull\n# No pull here: the top-level process pinned one SHA and moved the checkout to it.\n# A checkout that has moved off it since means another hand touched the box.\nprint \"vps-deploy: $app (checkout must be ${VPS_DEPLOY_SHA:?no pinned SHA, run through the top-level vps-deploy})\"\n[[ $(git_as_deploy_user rev-parse HEAD) == \"$VPS_DEPLOY_SHA\" ]] \\\n  || { print -u2 \"vps-deploy: $app refused - HEAD is $(git_as_deploy_user rev-parse HEAD), not the pinned $VPS_DEPLOY_SHA\"; exit 1 }\n\n# Free space, before anything is stopped. A bundle install, an asset precompile or\n# a CI mirror that runs out of disk dies halfway, after the app was stopped for the\n# migrate; refusing here costs nothing and leaves the previous release serving.\n# /home holds the checkout, the app trees, bundles and storage (3.9 GB free of\n# 17 GB on 2026-10-12), so it is the filesystem that fills. The floor is a\n# variable because it is a measurement of this box, not a law.\nset_stage preflight-disk\nmin_free_kb=${VPS_DEPLOY_MIN_FREE_KB:-1048576}\nfor mount_point in /home /var; do\n  free_kb=${${(z)${${(f)\"$(${VPS_DEPLOY_DF:-df} -Pk $mount_point 2>/dev/null)\"}[2]}}[4]:-}\n  if [[ $free_kb != <-> ]]; then\n    print -u2 \"vps-deploy: $app refused - free-space measurement for $mount_point is unavailable.\"\n    print -u2 \"            Nothing was stopped. Check the filesystem and measurement command, then rerun:\"\n    print -u2 \"            doas zsh OPENBSD/bin/vps-deploy $app\"\n    exit 1\n  fi\n  if (( free_kb < min_free_kb )); then\n    print -u2 \"vps-deploy: $app refused - $mount_point has $(( free_kb / 1024 )) MB free, the floor is $(( min_free_kb / 1024 )) MB.\"\n    print -u2 \"            Nothing was stopped. Inspect usage with: doas /usr/local/sbin/pub5-admin disk-usage $mount_point ; clear old\"\n    print -u2 \"            logs or bundles, then rerun: doas zsh OPENBSD/bin/vps-deploy $app\"\n    exit 1\n  fi\ndone\n\nset_stage preflight\nprint \"vps-deploy: $app (preflight load)\"\nadmin_request resource-guard || true\n\nkeep_deploy_flag\n\n# What is about to be deployed, against what is actually running.\n#\n# The check compares OPENBSD/mirror/etc byte-for-byte with live /etc and root's crontab\n# with data/root.crontab. A deploy is exactly when that matters — it is the moment\n# the repo becomes the box.\n#\n# Advisory, not blocking, and deliberately so. Measured 2026-09-10 with\n# `SSH_HOST=dev@brgen.no ruby OPENBSD/checks/config_drift_check.rb --remote`: eight files\n# differ and two more are absent from the box entirely — emergency_cpu.sh, which\n# is resource_guard's crisis tier, and vps_weekly_integrity.sh, which the crontab\n# has scheduled for weeks. A hard check here would refuse every deploy until an\n# OPERATOR.sh run nobody has scheduled, and one of the eight is rc.d/master, which\n# starts a production service. Make it blocking the first time a deploy runs with\n# this clean; a warning that names a number is still the difference between\n# knowing and not.\nif [[ -x /usr/local/bin/config_drift_check.rb ]]; then\n  if ! \"$RUBY\" /usr/local/bin/config_drift_check.rb > /tmp/vps-deploy-drift.out 2>&1; then\n    # ruby, not grep: coreutils are banned in committed scripts here, and the\n    # check's own summary line already carries the count in the shape we want.\n    drift_count=$(\"$RUBY\" -e \\\n      'print File.read(\"/tmp/vps-deploy-drift.out\").scan(/: DRIFT -/).size' 2>/dev/null)\n    print \"vps-deploy: $app WARNING - ${drift_count:-?} config(s) on the box differ from the repo being deployed\"\n    print \"vps-deploy: $app see /tmp/vps-deploy-drift.out\"\n    # Blocking is opt-in until the box has run clean once; the default stays the\n    # warning above so an unscheduled OPERATOR.sh run cannot freeze every deploy.\n    if [[ ${VPS_DEPLOY_DRIFT:-warn} == fail ]]; then\n      print -u2 \"vps-deploy: $app refused - VPS_DEPLOY_DRIFT=fail and the box has drifted\"\n      exit 1\n    fi\n  fi\nfi\n\nif [[ ${SKIP_CI:-} == 1 ]]; then\n  print \"vps-deploy: $app (sync via ${app}.sh, SKIP_CI=1)\"\n  # The app entrypoints refuse root (line 5 of every one of them: checkout\n  # code is operator-owned, privileged actions go through dev's doas and the\n  # pub5_rcctl broker), so the sync runs as the deploy user, not as this\n  # script's root. First exercised on vm23 2026-11-08: as root, brgen.sh\n  # refused and the app stage died before stop, leaving the release serving.\n  run_as_deploy_user \"cd $(printf '%q' \"$repo\") && zsh $(printf '%q' \"$repo/RAILS/${app}/${app}.sh\")\"\nelse\n  # The app's resident Solid Queue worker stands down for the CI run.\n  #\n  # vm23 is 1 GB. brgen_jobs holds ~380 MB resident and brgen's suite is 896\n  # tests: on 2026-08-23 the runner was OOM-killed mid-suite with swap at 898 MB\n  # of 1264, and the failure is silent in the worst way — 155 dots, no summary,\n  # no failure block, and vps_ci reporting \"Tests: Rails failed\". Dots then\n  # nothing in a CI test section means the runner was killed, not that a test\n  # failed. Stopped and restarted here so the knowledge lives in the deploy\n  # rather than in whoever remembers to type it.\n  # No other app may still be booting when CI starts: a Falcon cold boot and a test\n  # suite on one core is how the apps were shed.\n  settle_boots\n  jobs_service=\"${app}_jobs\"\n  jobs_was_up=0\n  if admin_service get \"$jobs_service\" >/dev/null 2>&1 && admin_service check \"$jobs_service\" >/dev/null 2>&1; then\n    print \"vps-deploy: $app (stand down $jobs_service for the CI run)\"\n    admin_service stop \"$jobs_service\" >/dev/null 2>&1 && { jobs_was_up=1; own_stops+=(\"$jobs_service\") }\n  fi\n\n  # Swap floor, measured after the stand-down above has freed what it frees.\n  # CI, bundle and asset work on a 1 GB box with swap nearly gone is how every\n  # app was shed: on 2026-11-03 a gem fetch at ~200 MB free took all sites down\n  # for 17 minutes, and on 2026-11-04 a MASTER run at 217 MB free timed brgen out.\n  set_stage preflight-swap\n  min_swap_mb=${VPS_DEPLOY_MIN_FREE_SWAP_MB:-400}\n  swap_blocks=${${(M)${(z)\"$(${VPS_DEPLOY_SWAPCTL:-swapctl} -s 2>/dev/null)\"}:#<->}[3]:-}\n  if [[ $swap_blocks == <-> ]] && (( swap_blocks * 512 / 1048576 < min_swap_mb )); then\n    (( jobs_was_up )) && admin_service start \"$jobs_service\" >/dev/null 2>&1\n    free_swap_mb=$(( swap_blocks * 512 / 1048576 ))\n    print -u2 \"vps-deploy: $app refused - $free_swap_mb MB of swap free, the floor is $min_swap_mb MB.\"\n    print -u2 \"            The app's jobs service is back up; nothing else was stopped. Free memory (stop another\"\n    print -u2 \"            app's *_jobs service), then rerun: doas zsh OPENBSD/bin/vps-deploy $app\"\n    exit 1\n  fi\n\n  set_stage ci+sync\n  print \"vps-deploy: $app (ci + sync)\"\n  # `|| ci_status=$?` rather than a bare call and `ci_status=$?` after it. This\n  # script runs under `set -e`, so a bare failing CI run exits here and the\n  # restart below — the whole reason the stand-down records jobs_was_up — never\n  # runs, leaving the worker stopped with nothing 5xx'ing and the queue simply not\n  # draining. A command followed by `||` is exempt from errexit, which is what\n  # makes the status capture reachable at all.\n  ci_status=0\n  zsh \"$repo/OPENBSD/bin/vps_ci.sh\" \"$app\" || ci_status=$?\n\n  if (( jobs_was_up )); then\n    admin_service start \"$jobs_service\" >/dev/null 2>&1 \\\n      || print -u2 \"vps-deploy: $jobs_service did not come back - start it\"\n  fi\n  (( ci_status == 0 )) || exit $ci_status\nfi\n\n# Primary database, then the cache/queue/cable schemas.\n#\n# db_create_migrate_as_app runs db:prepare, which is idempotent — it creates the\n# database when absent, loads the schema when empty, and otherwise runs only\n# pending migrations — and then does the secondary schemas itself. Loading the\n# secondaries alone leaves the primary unmigrated, so a merged migration never\n# reaches production: on 2026-07-29 brgen shipped a counter-cache column its\n# views already read, and every feed render 500'd until an operator ran the\n# three pending migrations by hand.\nset_stage migrate\nprint \"vps-deploy: $app (primary db migrate + secondary schemas)\"\n. \"$repo/RAILS/_core.sh\"\n. \"$repo/RAILS/_bundle.sh\"\n. \"$repo/RAILS/_runtime_check.sh\"\n. \"$repo/RAILS/_database.sh\"\n. \"$repo/RAILS/_assets.sh\"\n# Same reason RAILS/_deploy.sh stops first: db:prepare hangs on the write\n# lock a live Falcon worker already holds on production.sqlite3.\nprint \"vps-deploy: $app (stop for sqlite migrate)\"\nadmin_service stop \"$app\" 2>/dev/null || true\napp_stopped=1\nown_stops+=(\"$app\")\ndb_create_migrate_as_app \"$app\" \"/home/${app}/app\"\n\n# The SKIP_CI=1 path (${app}.sh -> deploy_tracked_app -> rails_runtime_check)\n# already precompiles as part of its check. The default ci+sync path\n# (vps_ci.sh) only runs tests against a throwaway test DB and never\n# touches production assets -- without this, public/assets/ is simply\n# never created for a plain `vps-deploy $app` run, so Propshaft 404s\n# every asset_path() and the page renders with no CSS at all. Safe to\n# call unconditionally: Propshaft precompile is idempotent.\nset_stage precompile\nprint \"vps-deploy: $app (assets precompile)\"\nsettle_boots\nrails_assets_precompile_as_app \"$app\" \"/home/${app}/app\" || exit 1\n\nset_stage restart\nprint \"vps-deploy: $app (restart)\"\nhold_deploy_flag\nif ! admin_service_retry restart \"$app\"; then\n  log \"restart failed after retries\"\n  write_stamp \"$app\" failed\n  exit 1\nfi\nif ! retry 5 2 admin_service check \"$app\"; then\n  log \"service check failed after restart\"\n  write_stamp \"$app\" failed\n  exit 1\nfi\napp_stopped=0\nset_stage health\nport=$(\"$RUBY\" -ryaml -e 'puts YAML.safe_load(File.read(\"'\"$repo\"'/RAILS/apps.yml\")).dig(\"apps\",\"'\"$app\"'\",\"port\")')\nup_ok=0\nfor i in {1..150}; do\n  curl -fsS \"http://127.0.0.1:${port}/up\" >/dev/null 2>&1 && { up_ok=1; break; }\n  sleep 2\ndone\n[[ $up_ok -eq 1 ]] || { write_stamp \"$app\" failed; print -u2 \"vps-deploy: $app /up not ready on :${port}\"; exit 1; }\n\n# /up answers before Propshaft is involved, so it says nothing about whether the\n# page a visitor gets is styled. On 2026-08-14 brgen served every page with a\n# <link> to a stylesheet that 404'd — public/assets held zero files — and /up,\n# rcctl check, the port probe and the public TLS check all passed. The site was\n# plainly broken and the only thing that noticed was a person looking at it.\n#\n# So ask for the stylesheet the page actually links, through the Host that owns\n# it: these apps are domain-constrained and a bare-IP request is a 403, which\n# would make this check pass by never finding a link to test.\ndomain=$(\"$RUBY\" -ryaml -e 'puts YAML.safe_load(File.read(\"'\"$repo\"'/RAILS/apps.yml\")).dig(\"apps\",\"'\"$app\"'\",\"domain\")')\n# The page and the link are two reads, because under set -e and pipefail one\n# pipeline that finds no link fails the assignment and ends the deploy with no\n# stamp and no message. A home page that does not answer is a failed deploy; a\n# home page that links no stylesheet is the else branch below.\nhome_page=$(curl -fsS --max-time 30 -H \"Host: ${domain}\" \"http://127.0.0.1:${port}/\" 2>/dev/null) || {\n  write_stamp \"$app\" failed\n  print -u2 \"vps-deploy: $app / did not answer on :${port} with Host ${domain}\"\n  exit 1\n}\ncss_href=$(print -r -- \"$home_page\" | \"$RUBY\" -e 'print $stdin.binmode.read[%r{/assets/[^\"]+\\.css}n].to_s')\nif [[ -n $css_href ]]; then\n  if curl -fsS -o /dev/null --max-time 30 -H \"Host: ${domain}\" \"http://127.0.0.1:${port}${css_href}\" 2>/dev/null; then\n    print \"vps-deploy: $app stylesheet ok (${css_href})\"\n  else\n    write_stamp \"$app\" failed\n    print -u2 \"vps-deploy: $app serves ${css_href} as 404 - precompiled assets missing, the site is unstyled\"\n    exit 1\n  fi\nelse\n  print \"vps-deploy: $app no /assets stylesheet linked on / - nothing to verify\"\nfi\n\nset_stage solid_queue\nqueue_schema=\"$repo/RAILS/${app}/db/queue_schema.rb\"\n# An app whose production config runs jobs in-process has no worker to prove;\n# demanding one, or an hourly drain, would be meaningless.\n# UTF-8 explicitly: the box runs this with no LANG, and production.rb has em dashes.\nqueue_async=0\n\"$RUBY\" -e 'exit File.read(ARGV[0], encoding: \"UTF-8\").match?(/queue_adapter\\s*=\\s*:async\\b/)' \\\n  \"$repo/RAILS/${app}/config/environments/production.rb\" 2>/dev/null && queue_async=1\nqueue_empty=0\n[[ -f $queue_schema ]] && \"$RUBY\" -e 'exit File.read(ARGV[0]).include?(\"define(version: 0)\")' \\\n  \"$queue_schema\" && queue_empty=1\nif (( ! queue_async && ! queue_empty )) && [[ -f $queue_schema ]]; then\n  print \"vps-deploy: $app (solid_queue proof)\"\n  admin_request solid-queue-proof \"$app\" \\\n    || { write_stamp \"$app\" failed; print -u2 \"vps-deploy: solid_queue proof failed for $app\"; exit 1; }\nfi\n\n# Restore whatever the guard shed while we were deploying.\n#\n# resource_guard sheds its OPTIONAL list — bsdports, then amber — under\n# sustained pressure, and a deploy is exactly that. relayd keeps answering TLS\n# with their ports closed, so the outage reads as a hang rather than a 5xx and\n# nothing reports it: on 2026-08-23 amber and bsdports were restarted by hand\n# three times in one session, each time discovered by a person running curl.\n#\n# `vps-deploy all` folds the restore into its ordering; this pass gives a\n# single-app deploy, the common case, the same restore. Reported, not silent — a\n# service that needed restoring is a fact about the box worth seeing.\nset_stage restore-shed\nrestore_failed=0\nrestore_started || restore_failed=1\n\n# The checks, on the deployed thing.\n#\n# Outside this pass the checks run only when a person types a command, so the\n# half that can only be measured against a running app would be measured on a\n# laptop where it means less. A user journey is real only here: production\n# answers 403 to a request without the right Host header, which development\n# never does.\n#\n# This set is the cheap half: source checks plus the live probes against the\n# loopback. It runs in about forty seconds on this one core with the fleet up,\n# which is what makes it affordable on every deploy.\n#\n# The browser half — rendered_suite, layout_suite, visual_contract — is NOT here\n# and is opt-in through PUB5_DEPLOY_BROWSER_CHECKS=1. Chrome is installed, but\n# this box is 1GB with four apps resident and resource_guard sheds amber and\n# bsdports under exactly that pressure. A deploy that takes the site down to\n# check the site has failed at the only thing it was for.\n#\n# CHECK_AUTOFIX=0 because a check must not rewrite the tree it is judging, least\n# of all the one that is deployed.\n#\n# CHECK_REQUIRE_LIVE=1 because this is the one host where a live check has no\n# excuse. It turns \"port closed, skipping\" into a failure, and half this set —\n# flow_journey, page_simulation, port_inventory — measures nothing without a\n# listening app. The fleet was just restarted and the shed apps restored a few\n# lines above, so a closed port here is a deploy that did not come up, not a\n# laptop. bin/check-rails takes the same position on the box through\n# Environment.on_vps?, and a second caller on the same host must not quietly\n# accept a skip.\ncheck_names=(production port_inventory generated_asset schema_migration\n            phantom_foreign_keys frontend_production stimulus_components\n            flow_journey page_simulation)\nif [[ -n ${PUB5_DEPLOY_BROWSER_CHECKS:-} ]]; then\n  check_names+=(rendered_suite layout_suite)\nfi\nset_stage checks\nprint \"vps-deploy: $app (checks: ${check_names})\"\nhold_deploy_flag\nif ! FLOW_APP=\"$app\" MASTER_STRICT=deploy CHECK_AUTOFIX=0 \"$RUBY\" \"$repo/MASTER/checks/runner.rb\" \"${check_names[@]}\"; then\n  write_stamp \"$app\" failed\n  print -u2 \"vps-deploy: $app checks failed - the app is deployed and serving, and something\"\n  print -u2 \"            it is now serving does not hold. Fix or revert; do not deploy over it.\"\n  exit 1\nfi\n\nset_stage final-check\n# The app deployed last has to be the app still running when the deploy says ok.\n# Every check above ran minutes ago, and a deploy whose own app is down on exit\n# is the one outcome the stamp must never call ok.\nif ! admin_service check \"$app\" >/dev/null 2>&1; then\n  write_stamp \"$app\" failed\n  print -u2 \"vps-deploy: $app is down at the end of its own deploy - /usr/local/sbin/pub5-admin service restart $app, then read /var/log/daemon\"\n  exit 1\nfi\n\n# The local canary is a release gate, not an advisory log line.\nif ! \"$RUBY\" \"$repo/OPENBSD/bin/canary\" --local --app \"$app\"; then\n  write_stamp \"$app\" failed\n  print -u2 \"vps-deploy: $app local canary failed - see the failed checks above\"\n  exit 1\nfi\n\nif (( restore_failed )); then\n  write_stamp \"$app\" failed\n  print -u2 \"vps-deploy: $app is not complete; a service that was running before is still down\"\n  exit 1\nfi\n\nprint \"vps-deploy: $app ok\"\nwrite_stamp \"$app\" ok\nexit 0\n" to include "def other_deploy_processes".
       
         2) Failure:
       VpsDeployContractTest#test_operator_installs_pub5_broker_and_its_solid_queue_helper [test/test_vps_deploy_contract.rb:214]:
       Expected "#!/usr/bin/env zsh\n# Copy-ready OpenBSD filesystem payload. No Ruby, rendering, templating or deletion.\n# OPENBSD/mirror/<path> installs to /<path>. Operator tools live in OPENBSD/bin and lib.\nset -euo pipefail\nsetopt extended_glob null_glob\n\nSCRIPT_DIR=${0:A:h}\nREPO_ROOT=${SCRIPT_DIR:h}\nMIRROR=${SCRIPT_DIR}/mirror\nDATA=${SCRIPT_DIR}/data\nBACKUP_ROOT=${OPENBSD_BACKUP_ROOT:-/var/backups/openbsd_setup}\nACTION=plan\nINSTALL_DOAS=0\nDOAS_NEEDED=0\nRUN_ID=\nLOCK_DIR=${OPENBSD_APPLY_LOCK_DIR:-/var/run/pub5-openbsd-apply.lock}\nLOCK_HELD=0\ntypeset -a FILES CHANGED SIGNED_BEFORE\ntypeset -A BACKUPS\ntypeset -i failures=0 CRONTAB_APPLIED=0\n\nusage() {\n  print -r -- \"usage: doas zsh OPENBSD/OPERATOR.sh [--plan|--apply|--verify] [--doas]\"\n  print -r -- \"  --plan    compare tracked mirror files with vm23 (default, read-only)\"\n  print -r -- \"  --apply   back up and install changed files, then validate/reload affected services\"\n  print -r -- \"  --verify  require the installed filesystem and managed root cron to match\"\n  print -r -- \"  --doas    explicitly install etc/doas.conf through validate_doas.ksh with its canary/rollback\"\n  print -r -- \"  sshd_config is always protected from automatic replacement.\"\n}\n\nfor arg in \"$@\"; do\n  case \"$arg\" in\n    --plan) ACTION=plan ;;\n    --apply) ACTION=apply ;;\n    --verify) ACTION=verify ;;\n    --doas) INSTALL_DOAS=1 ;;\n    -h|--help) usage; exit 0 ;;\n    *) print -u2 -- \"OPERATOR: unknown argument: $arg\"; usage >&2; exit 64 ;;\n  esac\ndone\n\n[[ $(uname -s) == OpenBSD ]] || {\n  print -u2 -- \"OPERATOR: BLOCKED - this installer compares and changes the local OpenBSD host; current host is $(uname -s)\"\n  exit 78\n}\nif [[ $ACTION != plan && $(id -u) != 0 ]]; then\n  print -u2 -- \"OPERATOR: BLOCKED - --$ACTION must run as root (use doas)\"\n  exit 78\nfi\nif (( INSTALL_DOAS )) && [[ $ACTION != apply ]]; then\n  print -u2 -- \"OPERATOR: --doas is valid only with --apply\"\n  exit 64\nfi\n[[ -d $MIRROR && -d $DATA ]] || {\n  print -u2 -- \"OPERATOR: required OPENBSD/mirror or OPENBSD/data directory is missing\"\n  exit 1\n}\n\n# Only Git-tracked regular files under the four filesystem roots are eligible.\nwhile IFS= read -r -d '' tracked; do\n  rel=${tracked#OPENBSD/mirror/}\n  case \"$rel\" in\n    etc/*|home/*|usr/*|var/*) ;;\n    *) print -u2 -- \"OPERATOR: invalid mirror path: $rel\"; exit 1 ;;\n  esac\n  source_file=$MIRROR/$rel\n  [[ -f $source_file && ! -L $source_file ]] || {\n    print -u2 -- \"OPERATOR: BLOCKED - mirror source is missing, non-regular or symlinked: $rel\"\n    exit 1\n  }\n  FILES+=(\"$rel\")\ndone < <(git -c safe.directory=\"$REPO_ROOT\" -C \"$REPO_ROOT\" ls-files -z -- OPENBSD/mirror)\n\n(( ${#FILES} > 0 )) || {\n  print -u2 -- \"OPERATOR: BLOCKED - no tracked filesystem payload found under OPENBSD/mirror\"\n  exit 1\n}\n\nrelease_apply_lock() {\n  (( LOCK_HELD )) || return 0\n  rm -f \"$LOCK_DIR/owner\"\n  rmdir \"$LOCK_DIR\" 2>/dev/null || print -u2 -- \"OPERATOR: stale lock directory remains: $LOCK_DIR\"\n  LOCK_HELD=0\n}\n\nacquire_apply_lock() {\n  if ! mkdir -m 700 \"$LOCK_DIR\" 2>/dev/null; then\n    print -u2 -- \"OPERATOR: BLOCKED - another OpenBSD apply owns $LOCK_DIR (inspect it; do not remove a live lock)\"\n    return 1\n  fi\n  LOCK_HELD=1\n  print -r -- \"pid=$$ host=$(hostname) started=$(date -u +%Y-%m-%dT%H:%M:%SZ)\" > \"$LOCK_DIR/owner\" || {\n    release_apply_lock\n    return 1\n  }\n  trap release_apply_lock EXIT\n  trap 'exit 129' HUP\n  trap 'exit 130' INT\n  trap 'exit 143' TERM\n}\n\nfile_mode() {\n  local rel=$1 source_file=$MIRROR/$1\n  case \"$rel\" in\n    etc/rc.d/master|etc/rc.d/brgen|etc/rc.d/amber|etc/rc.d/bsdports|etc/rc.d/mrdamascy|usr/local/sbin/*) print 555 ;;\n    etc/doas.conf|etc/ssh/sshd_config) print 600 ;;\n    etc/rc.d/*) print 755 ;;\n    etc/daily.local|etc/weekly.local|usr/local/bin/*|usr/local/libexec/*) print 755 ;;\n    home/johann/.muttrc|home/johann/.mailcap) print 600 ;;\n    home/johann/bin/*) print 755 ;;\n    var/nsd/zones/master/*) print 640 ;;\n    *) if [[ -x $source_file ]]; then print 755; else print 644; fi ;;\n  esac\n}\n\nfile_owner() {\n  case \"$1\" in\n    home/johann/*) print johann:johann ;;\n    var/nsd/zones/master/*) print _nsd:_nsd ;;\n    *) print root:wheel ;;\n  esac\n}\n\nfile_matches() {\n  local rel=$1 source_file=$MIRROR/$1 target=/$1 attrs owner_group owner group mode\n  [[ -f $target && ! -L $target ]] || return 1\n  cmp -s \"$source_file\" \"$target\" || return 1\n  mode=$(file_mode \"$rel\")\n  owner_group=$(file_owner \"$rel\")\n  owner=${owner_group%%:*}\n  group=${owner_group##*:}\n  attrs=$(stat -f '%Lp:%Su:%Sg' \"$target\") || return 1\n  [[ $attrs == \"$mode:$owner:$group\" ]]\n}\n\nprint_plan() {\n  local rel\n  for rel in $FILES; do\n    if [[ $rel == etc/ssh/sshd_config ]]; then\n      if file_matches \"$rel\"; then\n        print \"SAME      /$rel (protected)\"\n      else\n        print -u2 -- \"PROTECTED /$rel (SSH configuration requires an out-of-band recovery path)\"\n        failures=1\n      fi\n    elif [[ $rel == etc/doas.conf ]]; then\n      if file_matches \"$rel\"; then\n        print \"SAME      /$rel (protected)\"\n      elif (( INSTALL_DOAS )); then\n        print \"GUARDED   /$rel (dedicated doas canary and rollback)\"\n        DOAS_NEEDED=1\n      else\n        print -u2 -- \"PROTECTED /$rel (pass --doas to use validate_doas.ksh)\"\n        failures=1\n      fi\n    elif file_matches \"$rel\"; then\n      print \"SAME      /$rel\"\n    else\n      print \"CHANGE    /$rel\"\n      CHANGED+=(\"$rel\")\n    fi\n  done\n  print \"SPECIAL   root crontab: OPENBSD/data/root.crontab (merged through crontab(1), never copied to /etc)\"\n}\n\nvalidate_payload() {\n  [[ ! -f $MIRROR/etc/pf.conf ]] || /sbin/pfctl -nf \"$MIRROR/etc/pf.conf\"\n  [[ ! -f $MIRROR/etc/relayd.conf ]] || /usr/sbin/relayd -n -f \"$MIRROR/etc/relayd.conf\"\n  [[ ! -f $MIRROR/etc/httpd.conf ]] || /usr/sbin/httpd -n -f \"$MIRROR/etc/httpd.conf\"\n  [[ ! -f $MIRROR/etc/mail/smtpd.conf ]] || /usr/sbin/smtpd -n -f \"$MIRROR/etc/mail/smtpd.conf\"\n  [[ ! -f $MIRROR/etc/ssh/sshd_config ]] || /usr/sbin/sshd -t -f \"$MIRROR/etc/ssh/sshd_config\"\n  if [[ -f $MIRROR/var/nsd/etc/nsd.conf ]]; then\n    /usr/local/sbin/nsd-checkconf \"$MIRROR/var/nsd/etc/nsd.conf\" || return 1\n    local zone\n    for zone in \"$MIRROR\"/var/nsd/zones/master/*.zone(N); do\n      /usr/local/sbin/nsd-checkzone \"${zone:t:r}\" \"$zone\" || {\n        print -u2 -- \"OPERATOR: NSD zone validation failed: $zone\"\n        return 1\n      }\n    done\n  fi\n}\n\nno_symlink_path() {\n  local path=$1 current=/ component\n  local -a pieces\n  pieces=(${(s:/:)path})\n  for component in $pieces; do\n    [[ -n $component ]] || continue\n    current=${current%/}/$component\n    [[ ! -L $current ]] || return 1\n  done\n  return 0\n}\n\nbackup_signed_zones() {\n  local dir=$BACKUP_ROOT/$RUN_ID/nsd-signed zone_file\n  install -d -o root -g wheel -m 700 \"$dir\" || return 1\n  SIGNED_BEFORE=(/var/nsd/zones/master/*.zone.signed(N))\n  for zone_file in $SIGNED_BEFORE; do\n    cp -p \"$zone_file\" \"$dir/${zone_file:t}\" || return 1\n  done\n}\n\nrestore_signed_zones() {\n  local zone_file base dir=$BACKUP_ROOT/$RUN_ID/nsd-signed\n  for zone_file in /var/nsd/zones/master/*.zone.signed(N); do\n    (( ${SIGNED_BEFORE[(Ie)$zone_file]} )) || rm -f \"$zone_file\" || return 1\n  done\n  for zone_file in $SIGNED_BEFORE; do\n    base=${zone_file:t}\n    [[ -f $dir/$base ]] && cp -p \"$dir/$base\" \"$zone_file\" || return 1\n  done\n  /usr/local/sbin/nsd-control reload\n}\n\nbackup_file() {\n  local rel=$1 target=/$1 backup=$BACKUP_ROOT/$RUN_ID/$1\n  install -d -o root -g wheel -m 700 \"$backup:h\"\n  if [[ -e $target ]]; then\n    [[ -f $target && ! -L $target ]] || {\n      print -u2 -- \"OPERATOR: refusing non-regular destination: $target\"\n      return 1\n    }\n    cp -p \"$target\" \"$backup\" || return 1\n    cmp -s \"$target\" \"$backup\" || return 1\n    BACKUPS[$rel]=$backup\n  else\n    BACKUPS[$rel]=MISSING\n  fi\n}\n\nrestore_files() {\n  local rel target backup restore_failed=0 tmp\n  for rel in $CHANGED; do\n    target=/$rel\n    backup=${BACKUPS[$rel]:-}\n    [[ -n $backup ]] || continue\n    if [[ $backup == MISSING ]]; then\n      rm -f \"$target\" || restore_failed=1\n    elif [[ -f $backup ]]; then\n      tmp=$(mktemp \"$target:h/.pub5-restore.XXXXXXXX\") || { restore_failed=1; continue; }\n      cp -p \"$backup\" \"$tmp\" && mv -f \"$tmp\" \"$target\" || {\n        rm -f \"$tmp\"\n        restore_failed=1\n      }\n    else\n      restore_failed=1\n    fi\n  done\n  return $restore_failed\n}\n\ninstall_one() {\n  local rel=$1 source_file=$MIRROR/$1 target=/$1 owner_group owner group mode tmp\n  owner_group=$(file_owner \"$rel\")\n  owner=${owner_group%%:*}\n  group=${owner_group##*:}\n  mode=$(file_mode \"$rel\")\n  no_symlink_path \"$target\" || {\n    print -u2 -- \"OPERATOR: refusing symlinked destination path: $target\"\n    return 1\n  }\n  if [[ $rel == home/johann/* ]] && ! id johann >/dev/null 2>&1; then\n    print -u2 -- \"OPERATOR: account johann is absent; refusing to create a home implicitly\"\n    return 1\n  fi\n  if [[ ! -d $target:h ]]; then\n    install -d -o \"$owner\" -g \"$group\" -m 755 \"$target:h\" || return 1\n  fi\n  tmp=$(mktemp \"$target:h/.pub5-install.XXXXXXXX\") || return 1\n  if ! install -o \"$owner\" -g \"$group\" -m \"$mode\" \"$source_file\" \"$tmp\" ||\n     ! mv -f \"$tmp\" \"$target\"; then\n    rm -f \"$tmp\"\n    return 1\n  fi\n  print \"INSTALLED /$rel ($owner:$group $mode)\"\n}\n\nbackup_root_crontab() {\n  local backup=$BACKUP_ROOT/$RUN_ID/root.crontab error_file=$BACKUP_ROOT/$RUN_ID/root.crontab.err detail\n  if crontab -l -u root > \"$backup\" 2> \"$error_file\"; then\n    rm -f \"$error_file\"\n    return 0\n  fi\n  detail=$(<\"$error_file\")\n  rm -f \"$error_file\"\n  if [[ $detail == *\"no crontab for root\"* ]]; then\n    : > \"$backup\"\n    return 0\n  fi\n  print -u2 -- \"OPERATOR: cannot read root crontab safely: $detail\"\n  rm -f \"$backup\"\n  return 1\n}\n\nrestore_root_crontab() {\n  local backup=$BACKUP_ROOT/$RUN_ID/root.crontab\n  [[ -f $backup ]] || return 1\n  crontab -u root \"$backup\" || return 1\n  CRONTAB_APPLIED=0\n  print \"RESTORED  root crontab\"\n}\n\ninstall_root_crontab() {\n  local tracked=$DATA/root.crontab current=$BACKUP_ROOT/$RUN_ID/root.crontab merged path_line line cmdpath index work\n  [[ -f $tracked && -f $current ]] || {\n    print -u2 -- \"OPERATOR: root crontab source or verified backup is missing\"\n    return 1\n  }\n  work=$(mktemp -d /var/db/pub5-crontab.XXXXXXXX) || return 1\n  if ! chown root:wheel \"$work\" || ! chmod 700 \"$work\"; then\n    rm -rf \"$work\"\n    return 1\n  fi\n  merged=$work/merged\n  path_line=\n  while IFS= read -r line; do\n    [[ $line == PATH=* ]] && { path_line=$line; break; }\n  done < \"$tracked\"\n  if [[ -n $path_line ]]; then print -r -- \"$path_line\" > \"$merged\"; else : > \"$merged\"; fi\n  while IFS= read -r line; do\n    [[ $line == PATH=* ]] && continue\n    print -r -- \"$line\" >> \"$merged\"\n  done < \"$current\"\n\n  while IFS= read -r line; do\n    [[ -z $line || $line == \\#* || $line == PATH=* ]] && continue\n    local -a fields\n    fields=(${=line})\n    if [[ ${fields[1]} == @* ]]; then\n      (( ${#fields} >= 2 )) || {\n        print -u2 -- \"OPERATOR: malformed tracked cron entry: $line\"\n        rm -rf \"$work\"\n        return 1\n      }\n      index=2\n    else\n      (( ${#fields} >= 6 )) || {\n        print -u2 -- \"OPERATOR: malformed tracked cron entry: $line\"\n        rm -rf \"$work\"\n        return 1\n      }\n      index=6\n    fi\n    cmdpath=\n    while (( index <= ${#fields} )); do\n      if [[ ${fields[index]} == /* ]]; then cmdpath=${fields[index]}; break; fi\n      (( index++ ))\n    done\n    [[ -n $cmdpath && -x $cmdpath ]] || {\n      print -u2 -- \"OPERATOR: tracked cron command is missing or not executable: $line\"\n      rm -rf \"$work\"\n      return 1\n    }\n    cron_has_command \"$cmdpath\" \"$merged\" || print -r -- \"$line\" >> \"$merged\"\n  done < \"$tracked\"\n\n  if ! crontab -u root \"$merged\"; then\n    rm -rf \"$work\"\n    return 1\n  fi\n  CRONTAB_APPLIED=1\n  rm -rf \"$work\"\n}\n\ncron_has_command() {\n  local command=$1 file=$2\n  awk -v command=\"$command\" '\n    /^([[:space:]]*#|[[:space:]]*$)/ { next }\n    {\n      start = ($1 ~ /^@/) ? 2 : 6\n      for (i = start; i <= NF; i++) if ($i == command) found = 1\n    }\n    END { exit(found ? 0 : 1) }\n  ' \"$file\"\n}\n\nverify_root_crontab() {\n  local tracked=$DATA/root.crontab live line cmdpath index expected_path live_path\n  [[ -f $tracked ]] || return 1\n  live=$(mktemp /tmp/pub5-crontab.XXXXXXXX) || return 1\n  local error_file\n  error_file=$(mktemp /tmp/pub5-crontab-error.XXXXXXXX) || { rm -f \"$live\"; return 1; }\n  if ! crontab -l -u root > \"$live\" 2>\"$error_file\"; then\n    local detail\n    detail=$(<\"$error_file\")\n    if [[ $detail == *\"no crontab for root\"* ]]; then\n      : > \"$live\"\n    else\n      print -u2 -- \"DRIFT     cannot read root crontab: $detail\"\n      rm -f \"$error_file\" \"$live\"\n      return 1\n    fi\n  fi\n  rm -f \"$error_file\"\n  expected_path=\n  while IFS= read -r line; do\n    [[ $line == PATH=* ]] && { expected_path=$line; break; }\n  done < \"$tracked\"\n  live_path=\n  while IFS= read -r line; do\n    [[ $line == PATH=* ]] && { live_path=$line; break; }\n  done < \"$live\"\n  if [[ -n $expected_path && $expected_path != $live_path ]]; then\n    print -u2 -- \"DRIFT     root crontab PATH assignment differs\"\n    rm -f \"$live\"\n    return 1\n  fi\n  while IFS= read -r line; do\n    [[ -z $line || $line == \\#* || $line == PATH=* ]] && continue\n    local -a fields\n    fields=(${=line})\n    index=6\n    [[ ${fields[1]} == @* ]] && index=2\n    cmdpath=\n    while (( index <= ${#fields} )); do\n      if [[ ${fields[index]} == /* ]]; then cmdpath=${fields[index]}; break; fi\n      (( index++ ))\n    done\n    if [[ -z $cmdpath ]] || ! cron_has_command \"$cmdpath\" \"$live\"; then\n      print -u2 -- \"DRIFT     root crontab missing $cmdpath\"\n      rm -f \"$live\"\n      return 1\n    fi\n  done < \"$tracked\"\n  rm -f \"$live\"\n  print \"SAME      managed root-crontab entries\"\n}\n\ncase \"$ACTION\" in\n  plan)\n    [[ ! -d $LOCK_DIR ]] || { print -u2 -- \"OPERATOR: BLOCKED - apply lock is active: $LOCK_DIR\"; exit 75; }\n    print \"OPENBSD mirror plan (read-only; source paths map directly to /)\"\n    print_plan\n    (( failures == 0 )) || exit 78\n    ;;\n  verify)\n    [[ ! -d $LOCK_DIR ]] || { print -u2 -- \"OPERATOR: BLOCKED - apply lock is active: $LOCK_DIR\"; exit 75; }\n    print \"OPENBSD mirror verification\"\n    print_plan\n    verify_root_crontab || failures=1\n    (( failures == 0 )) || exit 1\n    ;;\n  apply)\n    print \"OPENBSD mirror apply\"\n    print_plan\n    if (( DOAS_NEEDED && ${#CHANGED} )); then\n      print -u2 -- \"OPERATOR: BLOCKED - --doas may only change doas.conf by itself; apply other mirror changes separately\"\n      exit 78\n    fi\n    (( failures == 0 )) || {\n      print -u2 -- \"OPERATOR: BLOCKED - protected or invalid mirror differences must be handled separately\"\n      exit 78\n    }\n    validate_payload || {\n      print -u2 -- \"OPERATOR: BLOCKED - native OpenBSD configuration validation failed; nothing was copied\"\n      exit 1\n    }\n    acquire_apply_lock || exit 75\n    RUN_ID=$(date -u +%Y%m%dT%H%M%SZ)-$$\n    install -d -o root -g wheel -m 700 \"$BACKUP_ROOT/$RUN_ID\"\n    # Every recovery input is captured and verified before the first live write.\n    for rel in $CHANGED; do\n      backup_file \"$rel\" || {\n        print -u2 -- \"OPERATOR: backup failed for /$rel; nothing was installed\"\n        exit 1\n      }\n    done\n    backup_root_crontab || {\n      print -u2 -- \"OPERATOR: root crontab backup failed; nothing was installed\"\n      exit 1\n    }\n    for rel in $CHANGED; do\n      install_one \"$rel\" || {\n        print -u2 -- \"OPERATOR: install failed for /$rel\"\n        restore_files || print -u2 -- \"DELIVERY_FAILED: automatic file restoration failed\"\n        exit 1\n      }\n    done\n    for rel in $CHANGED; do\n      file_matches \"$rel\" || {\n        print -u2 -- \"OPERATOR: installed file did not verify: /$rel\"\n        restore_files || print -u2 -- \"DELIVERY_FAILED: automatic file restoration failed\"\n        exit 1\n      }\n    done\n    install_root_crontab || {\n      print -u2 -- \"OPERATOR: root crontab merge failed\"\n      if (( CRONTAB_APPLIED )); then\n        restore_root_crontab || { print -u2 -- \"DELIVERY_FAILED: root crontab restoration failed\"; exit 1; }\n      fi\n      restore_files || print -u2 -- \"DELIVERY_FAILED: automatic file restoration failed\"\n      exit 1\n    }\n    verify_root_crontab || {\n      print -u2 -- \"OPERATOR: root crontab read-back verification failed\"\n      restore_root_crontab || { print -u2 -- \"DELIVERY_FAILED: root crontab restoration failed\"; exit 1; }\n      restore_files || print -u2 -- \"DELIVERY_FAILED: automatic file restoration failed\"\n      exit 1\n    }\n    if (( ${CHANGED[(Ie)etc/pf.conf]} )); then\n      /sbin/pfctl -nf /etc/pf.conf && /sbin/pfctl -f /etc/pf.conf || failures=1\n    fi\n    if (( failures == 0 && ${CHANGED[(Ie)etc/relayd.conf]} )); then\n      /usr/sbin/rcctl restart relayd || failures=1\n    fi\n    if (( failures == 0 && ${CHANGED[(Ie)etc/httpd.conf]} )); then\n      /usr/sbin/rcctl restart httpd || failures=1\n    fi\n    if (( failures == 0 && ${CHANGED[(Ie)etc/mail/smtpd.conf]} )); then\n      /usr/sbin/rcctl restart smtpd || failures=1\n    fi\n    if (( failures == 0 && ${CHANGED[(Ie)var/nsd/etc/nsd.conf]} )); then\n      /usr/sbin/rcctl restart nsd || failures=1\n    fi\n    if (( failures == 0 && ${CHANGED[(I)var/nsd/zones/master/*.zone]} )); then\n      if backup_signed_zones; then\n        /usr/local/bin/nsd-resign || failures=1\n      else\n        failures=1\n      fi\n    fi\n    if (( failures )); then\n      print -u2 -- \"OPERATOR: activation failed; restoring the previous tracked files\"\n      if (( CRONTAB_APPLIED )); then\n        restore_root_crontab || { print -u2 -- \"DELIVERY_FAILED: root crontab restoration failed\"; exit 1; }\n      fi\n      restore_files || { print -u2 -- \"DELIVERY_FAILED: automatic file restoration failed\"; exit 1; }\n      if (( ${#SIGNED_BEFORE} )) || [[ -d $BACKUP_ROOT/$RUN_ID/nsd-signed ]]; then\n        restore_signed_zones || { print -u2 -- \"DELIVERY_FAILED: DNSSEC signature restoration failed\"; exit 1; }\n      fi\n      typeset -i recovery_failed=0\n      if (( ${CHANGED[(Ie)etc/pf.conf]} )); then\n        /sbin/pfctl -nf /etc/pf.conf && /sbin/pfctl -f /etc/pf.conf || recovery_failed=1\n      fi\n      if (( ${CHANGED[(Ie)etc/relayd.conf]} )); then\n        /usr/sbin/rcctl restart relayd && /usr/sbin/rcctl check relayd || recovery_failed=1\n      fi\n      if (( ${CHANGED[(Ie)etc/httpd.conf]} )); then\n        /usr/sbin/rcctl restart httpd && /usr/sbin/rcctl check httpd || recovery_failed=1\n      fi\n      if (( ${CHANGED[(Ie)etc/mail/smtpd.conf]} )); then\n        /usr/sbin/rcctl restart smtpd && /usr/sbin/rcctl check smtpd || recovery_failed=1\n      fi\n      if (( ${CHANGED[(Ie)var/nsd/etc/nsd.conf]} )); then\n        /usr/sbin/rcctl restart nsd && /usr/sbin/rcctl check nsd || recovery_failed=1\n      fi\n      if (( recovery_failed )); then\n        print -u2 -- \"DELIVERY_FAILED: prior files were restored, but one or more services did not recover; inspect the backup and daemon logs\"\n        exit 1\n      fi\n      print \"RECOVERED prior tracked files and affected services\"\n      exit 1\n    fi\n    if (( DOAS_NEEDED )); then\n      if ! \"$SCRIPT_DIR/bin/validate_doas.ksh\" install \"$MIRROR/etc/doas.conf\" \"pub5-openbsd\"; then\n        print -u2 -- \"OPERATOR: guarded doas configuration change failed (helper restored the prior policy)\"\n        restore_root_crontab || { print -u2 -- \"DELIVERY_FAILED: root crontab restoration failed\"; exit 1; }\n        exit 1\n      fi\n    fi\n    print \"PASS      OPENBSD filesystem mirror applied\"\n    print \"BACKUP    $BACKUP_ROOT/$RUN_ID\"\n    ;;\nesac\n" to include "usr/local/sbin/pub5-admin".
       
         3) Failure:
       VpsDeployContractTest#test_a_matching_live_process_fingerprint_keeps_its_lock [test/test_vps_deploy_contract.rb:612]:
       Expected: 75
         Actual: 0
       
         4) Failure:
       VpsDeployContractTest#test_fleet_recovery_receipt_is_created_and_snapshot_precedes_service_stops [test/test_vps_deploy_contract.rb:1066]:
       Expected "#!/usr/bin/env zsh\n# Serial VPS deploy entrypoint — sync, CI, restart, health, deploy stamp.\n# Usage: zsh OPENBSD/bin/vps-deploy all|GROUP|APP\nset -euo pipefail\n\n# SSH executes this as a non-login shell on vm23; pin the OpenBSD package path.\nexport PATH=/usr/local/bin:/usr/local/sbin:/usr/bin:/usr/sbin:/bin:/sbin\n\nusage() {\n  print -r -- \"usage: zsh OPENBSD/bin/vps-deploy all|GROUP|APP   (runs as root; re-execs via doas)\n\n  all          every active registered surface, including MASTER web\n  GROUP        every registered app whose deploy_root belongs to that namespace\n  APP          one exact app ID from RAILS/apps.yml\n  recover --plan              inspect interrupted-deploy recovery receipts\n  recover --apply <run-id>     restore only services recorded as up before that run\n  VPS_DEPLOY_SHA=<sha>   deploy exactly this commit (default: origin/main, fetched once)\n  VPS_DEPLOY_PREFLIGHT_SHA=<sha>   the operator's machine already proved this commit boots\n  VPS_DEPLOY_CI_SHA=<sha>   the operator's machine ran the app's bin/ci green on this commit; the box skips its suite\n  VPS_DEPLOY_SKIP_PREFLIGHT=1      break-glass, needs I_UNDERSTAND_FAST_DEPLOY=1\n  VPS_DEPLOY_SKIP_TESTS=1          break-glass: no test suite anywhere, needs I_UNDERSTAND_FAST_DEPLOY=1\n  SKIP_CI=1    skip vps_ci.sh; \\${app}.sh and its runtime check still run\n  VPS_DEPLOY_DRIFT=fail   refuse to deploy when the box's configs differ from the repo\n  PUB5_DEPLOY_BROWSER_CHECKS=1   add the browser checks after restart\n  VPS_DEPLOY_LOG=/path            aggregate log (default /var/log/pub5/vps-deploy.log)\n  VPS_DEPLOY_LOG_DIR=/path        log directory (default /var/log/pub5)\n  VPS_DEPLOY_RUN_LOG=/path        per-run log (default auto under LOG_DIR/runs/)\n\n  Git fetch/checkout always runs as user dev so the /home/dev/pub5 tree and the\n  GitHub deploy key stay owned and reachable. Service control uses pub5-admin\n  directly when already root.\"\n}\ncase ${1:-} in\n  -h|--help) usage; exit 0 ;;\nesac\n\n# Prefer root. Passwordless doas (dev in wheel / permit nopass) re-execs so the\n# operator can still type the command as dev. Git never runs as root: root has\n# no GitHub host key by design, and a root checkout would leave root-owned files\n# in the dev tree. Service control calls the broker directly when uid 0.\n#\n# doas strips the caller environment (the dev rule has no keepenv — intentional,\n# so RUBYOPT and friends cannot cross the privilege boundary). Re-exec through\n# /usr/bin/env with a prefix allowlist (VPS_DEPLOY_*, PUB5_DEPLOY_*, plus PUB5_ROOT,\n# SKIP_CI, I_UNDERSTAND_FAST_DEPLOY) so pin, preflight, canary and logging\n# knobs survive. Not keepenv: RUBYOPT and friends still cannot cross.\nif [[ $(id -u) -ne 0 ]]; then\n  if command -v doas >/dev/null 2>&1; then\n    self=$0\n    if [[ $self != /* ]]; then\n      self=\"$(cd -- \"$(dirname -- \"$self\")\" && pwd)/$(basename -- \"$self\")\"\n    fi\n    typeset -a deploy_env\n    # Prefix allowlist (not keepenv): every VPS_DEPLOY_* and PUB5_DEPLOY_* that\n    # is set in the caller crosses doas via /usr/bin/env. Plus a few one-offs\n    # the deploy reads. New knobs named under those prefixes need no edit here.\n    for _name in ${(k)parameters}; do\n      case $_name in\n        VPS_DEPLOY_*|PUB5_DEPLOY_*|PUB5_ROOT|SKIP_CI|I_UNDERSTAND_FAST_DEPLOY)\n          deploy_env+=(\"$_name=${(P)_name}\")\n          ;;\n      esac\n    done\n    exec doas -n /usr/bin/env \"${deploy_env[@]}\" zsh \"$self\" \"$@\"\n  fi\n  print -u2 \"vps-deploy: must run as root (or with passwordless doas)\"\n  print -u2 \"            try: doas zsh OPENBSD/bin/vps-deploy ${1:-all}\"\n  exit 2\nfi\n\n# Unprivileged owner of the checkout and GitHub credentials on this box.\nDEPLOY_USER=${VPS_DEPLOY_USER:-dev}\nDEPLOY_HOME=${VPS_DEPLOY_HOME:-/home/$DEPLOY_USER}\nrepo=${PUB5_ROOT:-/home/dev/pub5}\nsource \"$repo/OPENBSD/lib/ruby_select.sh\"\ninventory_target_kind() {\n  \"$RUBY\" -r\"$repo/OPENBSD/lib/deploy_inventory\" -e \\\n    'print Deploy::Inventory.new(root: ARGV[0]).target_kind(ARGV[1])' \"$repo\" \"$1\"\n}\n\ninventory_app_group() {\n  \"$RUBY\" -r\"$repo/OPENBSD/lib/deploy_inventory\" -e '\n    app = Deploy::Inventory.new(root: ARGV[0]).apps.find { |row| row.name == ARGV[1] }\n    print app&.deploy_group\n  ' \"$repo\" \"$1\"\n}\n\ninventory_names() {\n  local target=$1 degraded=${2:-0}\n  \"$RUBY\" -r\"$repo/OPENBSD/lib/deploy_inventory\" -e '\n    inventory = Deploy::Inventory.new(root: ARGV[0])\n    rows = ARGV[1] == \"all\" ? inventory.ordered_apps(degraded: ARGV[2] == \"1\") :\n                              inventory.ordered_apps(group: ARGV[1])\n    puts rows.map(&:name)\n  ' \"$repo\" \"$target\" \"$degraded\"\n}\n\n# Run a shell snippet as DEPLOY_USER without pulling root's environment into\n# the checkout. Always su: root may change to DEPLOY_USER without a doas rule.\nrun_as_deploy_user() {\n  # Root may su to DEPLOY_USER without a doas rule. Avoid doas -u: the box\n  # only permits dev→root and root→root by default.\n  local snippet=$1\n  su -s /bin/sh \"$DEPLOY_USER\" -c \"$snippet\"\n}\n\ngit_as_deploy_user() {\n  # All git mutations against the production checkout run as DEPLOY_USER.\n  local args_q\n  args_q=$(printf '%q ' \"$@\")\n  run_as_deploy_user \"cd $(printf '%q' \"$repo\") && git $args_q\"\n}\n\napp=${1:-}\n[[ -n $app ]] || { usage >&2; exit 2 }\nrecover_mode=\nrecover_id=\nif [[ $app == recover ]]; then\n  case ${2:-} in\n    --plan)\n      [[ $# -eq 2 ]] || { usage >&2; exit 2 }\n      recover_mode=plan\n      ;;\n    --apply)\n      [[ $# -eq 3 ]] && \"$RUBY\" -e 'exit(ARGV.fetch(0).match?(/\\A[0-9]{8}T[0-9]{6}Z-[a-z0-9_-]{1,32}-[0-9]+\\z/) ? 0 : 1)' \"$3\" || {\n        print -u2 \"vps-deploy: recover --apply needs one exact run ID from --plan\"\n        exit 2\n      }\n      recover_mode=apply\n      recover_id=$3\n      ;;\n    *) usage >&2; exit 2 ;;\n  esac\n  unset VPS_DEPLOY_RUN_ID VPS_DEPLOY_RUN_TARGET VPS_DEPLOY_RUN_STARTED_AT\n  unset VPS_DEPLOY_STARTED VPS_DEPLOY_LOCK_HELD\nelse\n  [[ $# -eq 1 ]] || { usage >&2; exit 2 }\n  # Restrict the log/path component to a safe identifier before inventory lookup.\n  [[ $app != *[^A-Za-z0-9_-]* ]] || {\n    print -u2 \"vps-deploy: invalid target identifier\"\n    exit 2\n  }\nfi\n# Until the pinned checkout is ready, the current inventory is only a\n# best-effort restore-order hint for services shed during preflight. These arrays\n# are refreshed from VPS_DEPLOY_SHA before target validation or deployment.\ndeploy_target_kind=unknown\ntypeset -a DEPLOY_ALL DEPLOY_ALL_DEGRADED DEPLOY_RAILS\nDEPLOY_ALL=(${(@f)\"$(inventory_names all 2>/dev/null || true)\"})\nDEPLOY_ALL_DEGRADED=(${(@f)\"$(inventory_names all 1 2>/dev/null || true)\"})\nDEPLOY_RAILS=(${(@f)\"$(inventory_names rails 2>/dev/null || true)\"})\n\n# Environment values below participate in shell arithmetic and bounded waits.\n# Validate their decimal spelling and limits before acquiring a lock or touching\n# a service. In particular, a zero poll interval otherwise creates an endless\n# wait_up loop during a failed-deploy restore.\nvalidate_deploy_setting() {\n  local name=$1 value=$2 minimum=$3 maximum=$4\n  [[ $value == <-> && ( $value == 0 || $value != 0* ) ]] || {\n    print -u2 \"vps-deploy: invalid $name; expected a decimal integer\"\n    return 2\n  }\n  (( ${#value} <= 10 )) || {\n    print -u2 \"vps-deploy: invalid $name; value is out of range\"\n    return 2\n  }\n  (( value >= minimum && value <= maximum )) || {\n    print -u2 \"vps-deploy: invalid $name; allowed range is $minimum..$maximum\"\n    return 2\n  }\n}\n\nvalidate_deploy_setting VPS_DEPLOY_LOCK_TTL \"${VPS_DEPLOY_LOCK_TTL:-7200}\" 1 604800 || exit 2\nvalidate_deploy_setting VPS_DEPLOY_RESTORE_WAIT \"${VPS_DEPLOY_RESTORE_WAIT:-240}\" 1 3600 || exit 2\nvalidate_deploy_setting VPS_DEPLOY_POLL \"${VPS_DEPLOY_POLL:-3}\" 1 60 || exit 2\nvalidate_deploy_setting VPS_DEPLOY_SETTLE_WAIT \"${VPS_DEPLOY_SETTLE_WAIT:-240}\" 1 3600 || exit 2\nvalidate_deploy_setting VPS_DEPLOY_MASTER_MIN_AVAIL \"${VPS_DEPLOY_MASTER_MIN_AVAIL:-10}\" 0 100 || exit 2\nvalidate_deploy_setting VPS_DEPLOY_MIN_FREE_KB \"${VPS_DEPLOY_MIN_FREE_KB:-1048576}\" 0 1000000000 || exit 2\n: \"${PUB5_ADMIN:=/usr/local/sbin/pub5-admin}\"\n[[ -f \"$PUB5_ADMIN\" && -x \"$PUB5_ADMIN\" && ! -L \"$PUB5_ADMIN\" ]] || {\n  print -u2 \"vps-deploy: pub5-admin missing; run reviewed OPENBSD/OPERATOR.sh first\"\n  exit 1\n}\n\n# One host, one deploy. The parent all/rails process keeps this lock while its\n# re-entrant child processes deploy each app in order; a second top-level deploy\n# is refused rather than running two pulls, precompiles and restarts together.\nlock_dir=${VPS_DEPLOY_LOCK_DIR:-$DEPLOY_HOME/.pub5-vps-deploy.lock}\nlock_ttl=${VPS_DEPLOY_LOCK_TTL:-7200}\nlock_owner=0\n\n# The lock is a directory, because mkdir is the one atomic test-and-set the\n# base system gives a shell, and the directory holds an `owner` file: pid, host\n# and epoch seconds. A bare directory could not tell a running deploy from the\n# remains of one killed by a dropped session or a reboot, and an operator had to\n# guess before removing it. Now a lock is reaped here, with the reason printed,\n# when its same-host owner PID is gone. Age alone never steals a lock from a\n# live local deployment: a slow or stalled deploy requires operator inspection,\n# not a second deploy running beside it. A remote-host or unreadable lock may\n# be reaped only after the recorded TTL, because this checkout cannot inspect\n# the other host's process.\n# BSD date formats an epoch with -r; GNU date reads -r as a file and needs\n# -d @epoch. The reaper prints the owner's start time on both dev hosts and vm23.\nepoch_date() {\n  date -r \"$1\" '+%Y-%m-%d %H:%M:%S' 2>/dev/null || date -d \"@$1\" '+%Y-%m-%d %H:%M:%S' 2>/dev/null\n}\n\n# BSD stat formats with -f; GNU stat reads -f as filesystem status and needs\n# -c. The reaper pins the observed directory by device and inode on both hosts.\nfile_identity() {\n  local out\n  out=$(stat -f '%d:%i' \"$1\" 2>/dev/null)\n  if [[ $out == <->:* ]]; then\n    print -r -- $out\n    return 0\n  fi\n  stat -c '%d:%i' \"$1\" 2>/dev/null\n}\n\n# A PID can be reused after its original deploy exits. Pair it with the\n# process start time and a checksum of its command line, so an unrelated process\n# cannot keep an abandoned lock alive forever. OpenBSD ps(1) exposes lstart and\n# command; if either cannot be read, callers fail safe and keep the lock.\nlock_process_identity() {\n  local pid=$1 started command digest\n  started=$(LC_ALL=C ps -p \"$pid\" -o lstart= 2>/dev/null) || return 1\n  command=$(ps -ww -p \"$pid\" -o command= 2>/dev/null) || return 1\n  [[ -n $started && -n $command ]] || return 1\n  started=${started//[^A-Za-z0-9]/_}\n  digest=$(print -rn -- \"$command\" | cksum) || return 1\n  digest=${digest// /_}\n  print -r -- \"${started}_${digest}\"\n}\n\nlock_read_owner() {\n  lock_pid= lock_host= lock_epoch= lock_identity=\n  [[ -r $lock_dir/owner ]] || return 1\n  read -r lock_pid lock_host lock_epoch lock_identity < \"$lock_dir/owner\" || true\n  [[ $lock_pid == <-> && $lock_epoch == <-> && $lock_identity != *[^A-Za-z0-9_]* ]]\n}\n\nlock_stale_reason() {\n  local now=$(date +%s) current_identity\n  if lock_read_owner; then\n    if [[ $lock_host == $(hostname -s) ]]; then\n      if ! ps -p \"$lock_pid\" >/dev/null 2>&1; then\n        print -r -- \"owner pid $lock_pid is not running on $lock_host\"\n        return 0\n      fi\n      if [[ -n $lock_identity ]]; then\n        current_identity=$(lock_process_identity \"$lock_pid\") || {\n          # A visible PID whose full identity cannot be measured is ambiguous,\n          # not stale. Never steal its lock based on a partial observation.\n          return 1\n        }\n        if [[ $current_identity != $lock_identity ]]; then\n          print -r -- \"owner pid $lock_pid process identity changed on $lock_host (PID reused)\"\n          return 0\n        fi\n      fi\n      # Legacy three-field owners have no fingerprint: keep them protected while\n      # their PID exists. New owners are distinguished even after PID reuse.\n      # A live same-host owner is authoritative even after the TTL.\n      return 1\n    fi\n    if (( now - lock_epoch > lock_ttl )); then\n      print -r -- \"held $(( (now - lock_epoch) / 60 )) minutes by pid $lock_pid on $lock_host, past the ${lock_ttl}s TTL\"\n      return 0\n    fi\n    return 1\n  fi\n  # mkdir has run but the owner file is not written yet, or never was. Judge\n  # such a lock by the directory's own age so a deploy mid-start is left alone.\n  local -a aged\n  aged=( ${lock_dir}(N/mm+$(( lock_ttl / 60 ))) )\n  (( $#aged )) || return 1\n  print -r -- \"no readable owner and older than the ${lock_ttl}s TTL\"\n}\n\nrelease_deploy_lock() {\n  (( lock_owner )) || return 0\n  rm -f -- \"$lock_dir/owner\"\n  rmdir \"$lock_dir\" 2>/dev/null || true\n}\n\nreap_stale_lock() {\n  local reason before_identity after_identity retired_lock\n\n  # Do not enter recovery for a live lock. The second check below closes the\n  # race between this observation and claiming the recovery marker.\n  reason=$(lock_stale_reason) || return 1\n  before_identity=$(file_identity \"$lock_dir\") || return 1\n\n  # The marker is inside the exact directory observed above, so mkdir is an\n  # atomic claim. Inode identity catches a stale directory being replaced by a\n  # fresh lock between the first observation and this claim.\n  if ! mkdir \"$lock_dir/reaping\" 2>/dev/null; then\n    return 1\n  fi\n  after_identity=$(file_identity \"$lock_dir\") || {\n    rmdir \"$lock_dir/reaping\" 2>/dev/null || true\n    return 1\n  }\n  if [[ $after_identity != $before_identity ]]; then\n    rmdir \"$lock_dir/reaping\" 2>/dev/null || true\n    return 1\n  fi\n\n  # Creating the marker updates the parent directory's mtime. Recheck owners,\n  # but retain the pre-marker age decision for a directory that was already\n  # ownerless; no other process can write its owner file while it exists.\n  if [[ -r \"$lock_dir/owner\" ]]; then\n    reason=$(lock_stale_reason) || {\n      rmdir \"$lock_dir/reaping\" 2>/dev/null || true\n      return 1\n    }\n  fi\n\n  retired_lock=\"${lock_dir}.stale.$$\"\n  if [[ -e \"$retired_lock\" ]]; then\n    print -u2 \"vps-deploy: stale-lock quarantine exists at $retired_lock\"\n    rmdir \"$lock_dir/reaping\" 2>/dev/null || true\n    return 1\n  fi\n\n  print -u2 \"vps-deploy: reaping stale lock $lock_dir - $reason\"\n  # Rename the whole claimed directory, marker included. The old lock path\n  # becomes free atomically; cleanup touches only this unique retired path.\n  if ! mv \"$lock_dir\" \"$retired_lock\" 2>/dev/null; then\n    rmdir \"$lock_dir/reaping\" 2>/dev/null || true\n    return 1\n  fi\n  rm -f -- \"$retired_lock/owner\"\n  # A process killed before the atomic owner rename can leave only this temp.\n  # Remove it with the quarantined lock so crash recovery does not leak dirs.\n  rm -f -- \"$retired_lock\"/owner.tmp.*(N)\n  rmdir \"$retired_lock/reaping\" 2>/dev/null || true\n  rmdir \"$retired_lock\" 2>/dev/null || true\n  return 0\n}\n\nacquire_deploy_lock() {\n  local -i attempt\n  for attempt in 1 2; do\n    if mkdir \"$lock_dir\" 2>/dev/null; then\n      local identity owner_tmp\n      identity=$(lock_process_identity \"${$}\") || {\n        print -u2 \"vps-deploy: cannot identify the lock owner process; refusing an unverifiable deploy\"\n        rmdir \"$lock_dir\" 2>/dev/null || true\n        exit 1\n      }\n      owner_tmp=\"$lock_dir/owner.tmp.${$}\"\n      if ! print -r -- \"${$} $(hostname -s) $(date +%s) $identity\" > \"$owner_tmp\" ||\n         ! mv \"$owner_tmp\" \"$lock_dir/owner\"; then\n        print -u2 \"vps-deploy: cannot write the deploy lock owner record\"\n        rm -f -- \"$owner_tmp\"\n        rmdir \"$lock_dir\" 2>/dev/null || true\n        exit 1\n      fi\n      export VPS_DEPLOY_LOCK_HELD=1\n      export VPS_DEPLOY_LOCK_PID=${$}\n      export VPS_DEPLOY_LOCK_IDENTITY=$identity\n      lock_owner=1\n      return 0\n    fi\n    if reap_stale_lock; then\n      continue\n    fi\n    break\n  done\n  if lock_read_owner; then\n    print -u2 \"vps-deploy: refused - another deploy owns $lock_dir (pid $lock_pid on $lock_host since $(epoch_date $lock_epoch))\"\n  else\n    print -u2 \"vps-deploy: refused - another deploy owns $lock_dir\"\n  fi\n  exit 75\n}\n\n# Re-entrant children must prove the inherited lock belongs to their live\n# parent. An environment flag alone must never bypass top-level serialization.\nif [[ -n ${VPS_DEPLOY_LOCK_HELD:-} ]]; then\n  lock_read_owner || {\n    print -u2 \"vps-deploy: refused - child invocation has no readable parent lock owner\"\n    exit 75\n  }\n  [[ $lock_host == \"$(hostname -s)\" &&\n     $lock_pid == ${VPS_DEPLOY_LOCK_PID:-} &&\n     $lock_identity == ${VPS_DEPLOY_LOCK_IDENTITY:-} ]] || {\n    print -u2 \"vps-deploy: refused - child invocation does not match the parent lock record\"\n    exit 75\n  }\n  ps -p \"$lock_pid\" >/dev/null 2>&1 || {\n    print -u2 \"vps-deploy: refused - parent deploy process is no longer running\"\n    exit 75\n  }\n  current_identity=$(lock_process_identity \"$lock_pid\") || {\n    print -u2 \"vps-deploy: refused - cannot verify the parent deploy process\"\n    exit 75\n  }\n  [[ $current_identity == \"$lock_identity\" ]] || {\n    print -u2 \"vps-deploy: refused - parent deploy PID identity changed\"\n    exit 75\n  }\nelse\n  acquire_deploy_lock\nfi\n\n# Durable deploy logging.\n#\n# A full deploy outlives the ssh session that starts it. When that session is\n# cut the only surviving record is the log file. We keep:\n#   - a rolling aggregate log (VPS_DEPLOY_LOG / /var/log/pub5/vps-deploy.log)\n#   - a per-run log under /var/log/pub5/runs/ with the pid and app in the name\n#   - a stable symlink at $DEPLOY_HOME/vps-deploy.log for operators who still\n#     tail the home path\n# Once per run, not once per app: each group re-invokes one child per app.\n# VPS_DEPLOY_LOGGING is inherited so child processes do not double-tee.\ndeploy_log_dir=${VPS_DEPLOY_LOG_DIR:-/var/log/pub5}\ndeploy_log=${VPS_DEPLOY_LOG:-$deploy_log_dir/vps-deploy.log}\ndeploy_run_log=${VPS_DEPLOY_RUN_LOG:-}\n\nlog_ts() { date -u '+%Y-%m-%dT%H:%M:%SZ'; }\n\n# Structured line: timestamp host pid app stage message. Always goes to stdout\n# (which is already teed into the log files).\nlog() {\n  print -r -- \"$(log_ts) host=$(hostname -s) pid=$$ app=${app:-?} stage=${deploy_stage:-?} $*\"\n}\n\n# The fleet receipt is written only through the fixed root broker. Ruby's JSON\n# encoder keeps service names, paths and metadata out of shell-generated JSON.\nwrite_deploy_run_receipt() {\n  [[ -n ${VPS_DEPLOY_RUN_ID:-} ]] || return 0\n  local payload\n  payload=$(\"$RUBY\" -rjson -e '\n    run_id, target, host, pid, owner_identity, status, recovery_state, stage,\n      started_at, pinned_sha, previous_sha, services, ports_csv, order_csv,\n      run_log, aggregate_log, snapshot_recorded = ARGV\n    ports = {}\n    ports_csv.split(\",\").reject(&:empty?).each do |pair|\n      name, value = pair.split(\"=\", 2)\n      abort \"invalid port pair\" unless name && value && value.match?(/\\A[0-9]+\\z/)\n      ports[name] = Integer(value, 10)\n    end\n    record = {\n      schema: 1, run_id: run_id, target: target, host: host, pid: Integer(pid, 10),\n      owner_identity: owner_identity, status: status, recovery_state: recovery_state,\n      stage: stage, started_at: started_at,\n      pinned_sha: pinned_sha.empty? ? nil : pinned_sha,\n      previous_sha: previous_sha.empty? ? nil : previous_sha,\n      started_services: services.empty? ? [] : services.split(\" \"),\n      ports: ports,\n      restore_order: order_csv.empty? ? [] : order_csv.split(\",\"),\n      run_log: run_log, aggregate_log: aggregate_log,\n      snapshot_recorded: snapshot_recorded == \"1\"\n    }\n    puts JSON.generate(record)\n  ' \"${VPS_DEPLOY_RUN_ID}\" \"${VPS_DEPLOY_RUN_TARGET}\" \"$(hostname -s)\" \\\n    \"${VPS_DEPLOY_LOCK_PID}\" \"${VPS_DEPLOY_LOCK_IDENTITY}\" \\\n    \"${VPS_DEPLOY_RECEIPT_STATUS:-running}\" \"${VPS_DEPLOY_RECEIPT_RECOVERY:-none}\" \\\n    \"${deploy_stage:-start}\" \"${VPS_DEPLOY_RUN_STARTED_AT}\" \\\n    \"${VPS_DEPLOY_RECEIPT_PINNED_SHA:-}\" \"${VPS_DEPLOY_RECEIPT_PREVIOUS_SHA:-}\" \\\n    \"${VPS_DEPLOY_RECEIPT_SERVICES:-}\" \"${VPS_DEPLOY_RECEIPT_PORTS:-}\" \\\n    \"${VPS_DEPLOY_RECEIPT_ORDER:-}\" \\\n    \"${VPS_DEPLOY_RUN_LOG}\" \"${VPS_DEPLOY_LOG}\" \\\n    \"${VPS_DEPLOY_RECEIPT_SNAPSHOT_RECORDED:-0}\") || return 1\n  \"$PUB5_ADMIN\" deploy-run-write \"$VPS_DEPLOY_RUN_ID\" \"$payload\" >/dev/null\n}\n\n# Stage transitions persist the latest recovery breadcrumb before the next\n# stage may stop, replace, migrate or restart a production service.\nset_stage() {\n  deploy_stage=$1\n  log \"enter\"\n  write_deploy_run_receipt || {\n    print -u2 \"vps-deploy: failed to persist recovery receipt at stage $deploy_stage\"\n    return 1\n  }\n}\n# Retry a command with exponential backoff. Use only for transient failures\n# (network, service boot races). Do not wrap git merge conflicts or policy\n# refusals. Logs each attempt. Returns the last exit status.\n#\n#   retry 3 2 curl -fsS http://127.0.0.1:port/up\n#   attempts=3 initial_delay=2s, doubles each time, capped at 30s\nretry() {\n  local max=${1:?} delay=${2:?}\n  shift 2\n  local attempt=1 rc=0\n  while (( attempt <= max )); do\n    if \"$@\"; then\n      (( attempt > 1 )) && log \"retry ok attempt=$attempt cmd=$*\"\n      return 0\n    fi\n    rc=$?\n    if (( attempt == max )); then\n      log \"retry exhausted attempts=$max rc=$rc cmd=$*\"\n      return $rc\n    fi\n    log \"retry fail attempt=$attempt/$max rc=$rc sleep=${delay}s cmd=$*\"\n    sleep $delay\n    delay=$(( delay * 2 ))\n    (( delay > 30 )) && delay=30\n    attempt=$(( attempt + 1 ))\n  done\n  return $rc\n}\n\n# admin_service with retry for start/restart only (not check/list).\nadmin_service_retry() {\n  local action=$1\n  shift || true\n  case $action in\n    start|restart)\n      retry \"${VPS_DEPLOY_SERVICE_RETRIES:-3}\" \"${VPS_DEPLOY_SERVICE_RETRY_DELAY:-2}\" \\\n        admin_service \"$action\" \"$@\"\n      ;;\n    *)\n      admin_service \"$action\" \"$@\"\n      ;;\n  esac\n}\n\nrotate_deploy_log() {\n  # Safety net only. Canonical rotation is newsyslog (OPENBSD/mirror/etc/newsyslog.conf).\n  # This fires if the log grew past the size limit between newsyslog runs.\n  local max_bytes=${VPS_DEPLOY_LOG_MAX_BYTES:-10485760}\n  [[ -f $deploy_log ]] || return 0\n  local size\n  size=$(stat -f %z \"$deploy_log\" 2>/dev/null) || return 0\n  (( size >= max_bytes )) || return 0\n  log \"rotate aggregate log size=$size max=$max_bytes\"\n  mv -f -- \"$deploy_log\" \"$deploy_log.1\" 2>/dev/null || true\n}\n\nif [[ -z ${VPS_DEPLOY_LOGGING:-} ]]; then\n  export VPS_DEPLOY_LOGGING=1\n  mkdir -p -- \"$deploy_log_dir/runs\" 2>/dev/null || {\n    # Fall back to the deploy user's home when /var/log/pub5 is not writable\n    # yet (broker/OPERATOR not installed). Still durable across SSH drops.\n    deploy_log_dir=$DEPLOY_HOME\n    deploy_log=${VPS_DEPLOY_LOG:-$DEPLOY_HOME/vps-deploy.log}\n    mkdir -p -- \"$deploy_log_dir/runs\" 2>/dev/null || true\n  }\n  rotate_deploy_log\n  if [[ -z $deploy_run_log ]]; then\n    deploy_run_log=\"$deploy_log_dir/runs/deploy-$(date -u '+%Y%m%d-%H%M%S')-${app}-$$.log\"\n  fi\n  export VPS_DEPLOY_LOG=$deploy_log\n  export VPS_DEPLOY_RUN_LOG=$deploy_run_log\n  export VPS_DEPLOY_LOG_DIR=$deploy_log_dir\n  # Stable path operators already know about.\n  if [[ -d $DEPLOY_HOME ]]; then\n    ln -sfn \"$deploy_log\" \"$DEPLOY_HOME/vps-deploy.log\" 2>/dev/null || true\n    ln -sfn \"$deploy_run_log\" \"$DEPLOY_HOME/vps-deploy-latest.log\" 2>/dev/null || true\n  fi\n  # Recovery depends on the per-run log retaining the pinned SHA, initial\n  # service snapshot and last entered stage. Do not begin a deploy when that\n  # durable record cannot be opened as a regular file.\n  if ! : >> \"$deploy_run_log\" || [[ ! -f \"$deploy_run_log\" || -L \"$deploy_run_log\" ]]; then\n    print -u2 \"vps-deploy: cannot create a regular per-run recovery log at $deploy_run_log; refusing before any service is touched\"\n    exit 1\n  fi\n  if ! : >> \"$deploy_log\" || [[ ! -f \"$deploy_log\" || -L \"$deploy_log\" ]]; then\n    print -u2 \"vps-deploy: cannot create a regular aggregate log at $deploy_log; refusing before any service is touched\"\n    exit 1\n  fi\n  # Tee stdout/stderr into both the aggregate and the per-run log. process\n  # substitution keeps a single stream so line order stays honest.\n  exec > >(tee -a \"$deploy_log\" \"$deploy_run_log\") 2>&1\n  set_stage start\n  log \"start target=$app uid=$(id -u) user=$(id -un) log=$deploy_log run_log=$deploy_run_log\"\n  log \"argv: $*\"\nfi\n\n# Transitional adapter: use the fixed root-owned broker once installed. The\n# fallback remains until the operator sync has installed it on vm23.\nadmin_service() {\n  local action=${1:-}\n  shift || true\n  if [[ $action == list-started ]]; then\n    \"$PUB5_ADMIN\" service list-started\n  else\n    \"$PUB5_ADMIN\" service \"$action\" \"$@\"\n  fi\n}\n\nadmin_request() {\n  [[ -x \"$PUB5_ADMIN\" ]] || {\n    print -u2 \"vps-deploy: pub5-admin is not installed; run the reviewed OPERATOR sync first\"\n    return 127\n  }\n  \"$PUB5_ADMIN\" \"$@\"\n}\n\n# The guard's deploy flag, held for the whole deploy and not only the restart.\n#\n# rc.d/<app> touches .deploying-<app> from rc_pre to the end of its /up wait and\n# removes it there. CI, migrate, precompile and the checks all run outside that\n# window at the box's highest load, and resource_guard counts every one of those\n# ticks as a strike. So the counter already stands past GUARD_SHED_STRIKES when\n# the restart's flag comes off, and the next tick sheds the first OPTIONAL\n# service. bsdports leads that list, which is why its own deploy exits 0 and\n# vps-state then reads bsdports(failed) with :47312 closed.\n#\n# Taken after the preflight guard run, which may still shed to free memory for\n# CI. A tick that sees the flag resets the counter, so the guard needs two fresh\n# breaching ticks after this deploy ends before it sheds anything. The name is\n# not .deploying-<app>, because rc.d removes that one mid-deploy; it is touched\n# again at each long stage because the guard ignores a flag older than 30 minutes.\ndeploy_flag=\"$repo/.deploying-vps-deploy-${app}\"\nhold_deploy_flag() { touch \"$deploy_flag\" 2>/dev/null || true }\n\n# The stage touches alone did not hold it: brgen's CI ran 34 minutes between two of\n# them on 2026-10-27, the flag aged out at 30, and the guard's emergency stop fired\n# mid-deploy. A keeper touches it every 4 minutes for as long as this process lives;\n# finish() stops it before removing the flag, and it exits on its own if we die.\nflag_keeper=\nkeep_deploy_flag() {\n  hold_deploy_flag\n  ( while kill -0 $$ 2>/dev/null; do touch \"$deploy_flag\" 2>/dev/null; sleep 240; done ) &!\n  flag_keeper=$!\n}\n\n# The terminal result, recorded however the script ends.\n#\n# A failure past `rcctl stop` used to leave the app stopped under a stamp that\n# still said ok, and an ssh disconnect cut the tee pipe before the last line\n# reached the log. Every exit now lands here: it stamps failed (unless a stage\n# already did), names the stage and the services left down, prints the safe next\n# command, and appends the verdict straight to the log file rather than through\n# tee. Each step is `|| true`: a trap that fails would hide the failure it reports.\ndeploy_stage=start\napp_stopped=0\nstamp_written=0\nhalted_app=\npreflight_active=0\npreflight_back_to=\ntypeset -a preflight_back_flag own_stops\n\n# Services the deploy needs to put back, and how it waits for them.\n#\n# On 2026-10-12 a halted `all` pass left brgen, amber and bsdports down for minutes\n# until the operator typed rcctl start by hand: the restore sat at the end of a pass\n# that never got there. The snapshot of started services is taken before anything is\n# touched (VPS_DEPLOY_STARTED, exported so the children see it), and the exit trap\n# of the top-level process puts back whatever is missing, one service at a time.\n# Falcon needs 30-40 s to boot on this box, and two booting at once is what shed\n# the apps in the first place, so each service answers /up before the next starts.\ntypeset -A deploy_ports\nload_deploy_ports() {\n  (( $#deploy_ports )) && return 0\n  local line\n  local -a rows\n  rows=(${(f)\"$(\"$RUBY\" -r\"$repo/OPENBSD/lib/deploy_inventory\" \\\n    -e 'Deploy::Inventory.new(root: ARGV[0]).apps.each { |a| puts \"\#{a.name} \#{a.port}\" }' \"$repo\" 2>/dev/null)\"})\n  for line in $rows; do\n    deploy_ports[${line%% *}]=${line##* }\n  done\n  return 0\n}\n\n# Bounded wait for http://127.0.0.1:PORT/up. Returns 1 when the limit passes.\nwait_up() {\n  local port=$1 limit=${2:-${VPS_DEPLOY_RESTORE_WAIT:-240}} step=${VPS_DEPLOY_POLL:-3} waited=0\n  while (( waited < limit )); do\n    if curl -fsS --max-time 5 \"http://127.0.0.1:${port}/up\" >/dev/null 2>&1; then\n      (( waited > 0 )) && log \"wait_up ok port=$port after=${waited}s\"\n      return 0\n    fi\n    sleep $step\n    waited=$(( waited + step ))\n  done\n  log \"wait_up timeout port=$port limit=${limit}s\"\n  return 1\n}\n\n# Every service that was started and is running must answer /up before the next\n# memory-heavy step (CI, bundle, precompile). A service that is down is not booting\n# and is not waited for; one that never answers is reported, not waited on forever.\nsettle_boots() {\n  local svc port\n  load_deploy_ports\n  for svc in ${=VPS_DEPLOY_STARTED:-}; do\n    port=${deploy_ports[$svc]:-}\n    [[ -n $port ]] || continue\n    admin_service check \"$svc\" >/dev/null 2>&1 || continue\n    wait_up \"$port\" \"${VPS_DEPLOY_SETTLE_WAIT:-240}\" \\\n      || print -u2 \"vps-deploy: WARNING - $svc is running but :${port}/up did not answer within ${VPS_DEPLOY_SETTLE_WAIT:-240}s\"\n  done\n  return 0\n}\n\n# Who stopped it: this deploy, resource_guard (its shed ledger), or neither.\nstop_origin() {\n  local svc=$1 shed_file=${VPS_DEPLOY_SHED_FILE:-/var/db/resource_guard_shed}\n  if [[ $svc == $halted_app ]] || (( ${own_stops[(Ie)$svc]} )); then\n    print \"stopped by this deploy\"\n  elif [[ -r $shed_file ]] && (( ${${(f)\"$(<$shed_file)\"}[(Ie)$svc]} )); then\n    print \"shed by resource_guard\"\n  else\n    print \"down for another reason (crashed or stopped outside the deploy)\"\n  fi\n}\n\n# Starts, one at a time and in serving order, every service that was started when\n# the deploy began and is not running now. Returns 1 if any would not come back.\nrestore_started() {\n  [[ -n ${VPS_DEPLOY_STARTED:-} ]] || return 0\n  if [[ ${VPS_DEPLOY_RESTORE_PORTS_FROM_RECEIPT:-0} != 1 ]]; then\n    load_deploy_ports\n  fi\n  local -a want order failed restored\n  local svc port origin\n  want=( ${=VPS_DEPLOY_STARTED} )\n  for svc in $DEPLOY_ALL_DEGRADED; do (( ${want[(Ie)$svc]} )) && order+=($svc); done\n  for svc in $want; do (( ${order[(Ie)$svc]} )) || order+=($svc); done\n  for svc in $order; do\n    admin_service check \"$svc\" >/dev/null 2>&1 && continue\n    origin=$(stop_origin \"$svc\")\n    print \"vps-deploy: restore $svc ($origin)\"\n    port=${deploy_ports[$svc]:-}\n    # restart, not start: rc.d/master runs its relayd restart, TTS warmup and flag\n    # removal only on restart, and restart on a stopped service is a start.\n    admin_service_retry restart \"$svc\" >/dev/null 2>&1 || true\n    if [[ -n $port ]]; then\n      wait_up \"$port\" || { admin_service_retry restart \"$svc\" >/dev/null 2>&1 || true; wait_up \"$port\"; } || {\n        print -u2 \"vps-deploy: restore $svc FAILED - :${port}/up not ready, read /var/log/daemon\"\n        failed+=($svc); continue\n      }\n    else\n      sleep 3\n      admin_service check \"$svc\" >/dev/null 2>&1 || { print -u2 \"vps-deploy: restore $svc FAILED - rcctl check\"; failed+=($svc); continue }\n    fi\n    print \"vps-deploy: restore $svc ok\"\n    restored+=($svc)\n  done\n  (( $#restored )) && print \"vps-deploy: restored ${restored}\"\n  (( $#failed )) && print -u2 \"vps-deploy: STILL DOWN after restore: ${failed}  (/usr/local/sbin/pub5-admin service restart <app>, one at a time)\"\n  (( $#failed == 0 ))\n}\n\n# A signal or shell exit can arrive while the box is detached for preflight.\n# Keep the original ref as explicit recovery state until it is restored.\nrestore_preflight_checkout() {\n  (( preflight_active )) || return 0\n  [[ -n $preflight_back_to ]] || {\n    print -u2 \"vps-deploy: preflight checkout recovery has no saved ref\"\n    return 1\n  }\n  git_as_deploy_user switch -q \"${preflight_back_flag[@]}\" \"$preflight_back_to\" || {\n    print -u2 \"vps-deploy: FAILED to restore checkout ref $preflight_back_to after preflight\"\n    return 1\n  }\n  preflight_active=0\n  log \"preflight checkout restored ref=$preflight_back_to\"\n}\n\n# Vendorables are not tracked in git (TREE_CONTRACTS.md, 8 November 2026):\n# master's smart-turn runtime bytes are fetched when absent, with pinned\n# versions and sha256-verified digests. Failure leaves ?smart_turn=1\n# unavailable — the feature is default-off — and the deploy continues.\nensure_smart_turn_assets() {\n  local vendor=\"$repo/MASTER/web/public/vendor/onnxruntime\"\n  local model=\"$repo/MASTER/web/public/models/smart-turn-v3.2-cpu.onnx\"\n  [[ -f $vendor/ort-wasm-simd-threaded.wasm && -f $vendor/ort.wasm.min.mjs && -f $model ]] && return 0\n  print \"vps-deploy: smart_turn assets absent; fetching pinned, checksummed copies\"\n  if ! run_as_deploy_user \"cd $(printf '%q' \"$repo/MASTER/web\") && sh script/build_smart_turn.sh\"; then\n    print -u2 \"vps-deploy: smart_turn fetch failed; ?smart_turn=1 stays unavailable (default-off) until MASTER/web/script/build_smart_turn.sh succeeds\"\n  fi\n}\n\nfinish() {\n  local rc=$?\n  trap - EXIT\n  if (( preflight_active )); then\n    restore_preflight_checkout || {\n      print -u2 \"vps-deploy: checkout remains in preflight recovery state; restore ref $preflight_back_to before another deploy\"\n      (( rc == 0 )) && rc=1\n    }\n  fi\n  if (( rc != 0 )) && [[ $deploy_target_kind == all || $deploy_target_kind == group ]]; then\n    print -u2 \"vps-deploy: $app FAILED at stage \\\"$deploy_stage\\\" (exit $rc)\"\n  fi\n  if (( rc != 0 )) && [[ $deploy_target_kind != all && $deploy_target_kind != group && $deploy_target_kind != unknown ]]; then\n    if (( ! stamp_written )) && (( $+functions[write_stamp] )); then\n      write_stamp \"$app\" failed || true\n    fi\n    print -u2 \"vps-deploy: $app FAILED at stage \\\"$deploy_stage\\\" (exit $rc)\"\n    if (( app_stopped )); then\n      print -u2 \"vps-deploy: $app was stopped for this deploy and is DOWN - read /var/log/daemon, then\"\n      print -u2 \"            /usr/local/sbin/pub5-admin service start $app   (or rerun: doas zsh OPENBSD/bin/vps-deploy $app)\"\n    else\n      print -u2 \"vps-deploy: $app was not stopped; the previous release is still serving. Fix the cause and rerun:\"\n      print -u2 \"            doas zsh OPENBSD/bin/vps-deploy $app\"\n    fi\n    local other\n    for other in $DEPLOY_RAILS; do\n      [[ $other == \"$app\" ]] && continue\n      admin_service get \"$other\" >/dev/null 2>&1 || continue\n      admin_service check \"$other\" >/dev/null 2>&1 \\\n        || print -u2 \"vps-deploy: $other is DOWN (shed or stopped) - /usr/local/sbin/pub5-admin service restart $other\"\n    done\n  fi\n  # Only the process that owns the lock restores: the children of `all` fail into\n  # their parent, which knows the whole pass. Each restore result is printed.\n  local restore_ok=1\n  if (( rc != 0 )) && (( lock_owner )) && [[ ${VPS_DEPLOY_RESTORE_ATTEMPTED:-0} != 1 ]]; then\n    restore_started || {\n      restore_ok=0\n      print -u2 \"vps-deploy: production is NOT fully restored - see the lines above\"\n    }\n  fi\n  # A normal EXIT/TERM/HUP reaches this terminal update; SIGKILL/power loss\n  # leaves the last durable receipt pending for an explicit operator recovery.\n  if (( lock_owner )) && [[ -n ${VPS_DEPLOY_RUN_ID:-} ]]; then\n    VPS_DEPLOY_RECEIPT_STATUS=failed\n    (( rc == 0 )) && VPS_DEPLOY_RECEIPT_STATUS=ok\n    if [[ -n ${VPS_DEPLOY_STARTED:-} ]]; then\n      if (( rc == 0 || restore_ok )); then\n        VPS_DEPLOY_RECEIPT_RECOVERY=complete\n      else\n        VPS_DEPLOY_RECEIPT_RECOVERY=pending\n      fi\n    else\n      VPS_DEPLOY_RECEIPT_RECOVERY=none\n    fi\n    export VPS_DEPLOY_RECEIPT_STATUS VPS_DEPLOY_RECEIPT_RECOVERY\n    write_deploy_run_receipt || {\n      print -u2 \"vps-deploy: FAILED to persist terminal recovery receipt\"\n      (( rc == 0 )) && rc=1\n    }\n  fi\n  # Keep resource_guard excluded until every service restored by this failure\n  # has answered, including slow boots; only then stop the heartbeat and remove\n  # this process's deploy flag.\n  [[ -n $flag_keeper ]] && kill $flag_keeper 2>/dev/null\n  rm -f -- \"$deploy_flag\"\n  log \"finished exit=$rc\"\n  # Write the verdict straight to the files as well: if the tee pipe died with\n  # the ssh session the last structured line can still land on disk.\n  {\n    print -r -- \"$(log_ts) host=$(hostname -s) pid=$$ app=$app stage=$deploy_stage finished exit=$rc\"\n  } >> \"${deploy_log:-/var/log/pub5/vps-deploy.log}\" 2>/dev/null || true\n  if [[ -n ${deploy_run_log:-${VPS_DEPLOY_RUN_LOG:-}} ]]; then\n    print -r -- \"$(log_ts) host=$(hostname -s) pid=$$ app=$app stage=$deploy_stage finished exit=$rc\" \\\n      >> \"${deploy_run_log:-$VPS_DEPLOY_RUN_LOG}\" 2>/dev/null || true\n  fi\n  release_deploy_lock\n  exit $rc\n}\n# zsh runs the EXIT trap for HUP but not for TERM or INT, so name all three: a\n# dropped ssh session or a kill is a failure with a recorded stage, not a silent end.\ntrap finish EXIT\ntrap 'exit 129' HUP\ntrap 'exit 130' INT\ntrap 'exit 143' TERM\n\n# One durable receipt belongs to the lock-owning parent. Child app deploys\n# inherit the ID, snapshot and owner identity and update only the current stage.\nif (( lock_owner )) && [[ $app != recover ]]; then\n  unset VPS_DEPLOY_RUN_ID VPS_DEPLOY_RUN_TARGET VPS_DEPLOY_RUN_STARTED_AT\n  unset VPS_DEPLOY_STARTED\n  unset VPS_DEPLOY_RECEIPT_STATUS VPS_DEPLOY_RECEIPT_RECOVERY\n  unset VPS_DEPLOY_RECEIPT_PINNED_SHA VPS_DEPLOY_RECEIPT_PREVIOUS_SHA\n  unset VPS_DEPLOY_RECEIPT_SERVICES VPS_DEPLOY_RECEIPT_PORTS VPS_DEPLOY_RECEIPT_ORDER\n  unset VPS_DEPLOY_RECEIPT_SNAPSHOT_RECORDED\n  VPS_DEPLOY_RUN_ID=\"$(date -u '+%Y%m%dT%H%M%SZ')-$app-${$}\"\n  VPS_DEPLOY_RUN_TARGET=$app\n  VPS_DEPLOY_RUN_STARTED_AT=\"$(date -u '+%Y-%m-%dT%H:%M:%SZ')\"\n  VPS_DEPLOY_RECEIPT_STATUS=running\n  VPS_DEPLOY_RECEIPT_RECOVERY=none\n  VPS_DEPLOY_RECEIPT_PINNED_SHA=\n  VPS_DEPLOY_RECEIPT_PREVIOUS_SHA=\n  VPS_DEPLOY_RECEIPT_SERVICES=\n  VPS_DEPLOY_RECEIPT_PORTS=\n  VPS_DEPLOY_RECEIPT_ORDER=\n  VPS_DEPLOY_RECEIPT_SNAPSHOT_RECORDED=0\n  export VPS_DEPLOY_RECEIPT_SNAPSHOT_RECORDED\n  export VPS_DEPLOY_RUN_ID VPS_DEPLOY_RUN_TARGET VPS_DEPLOY_RUN_STARTED_AT\n  export VPS_DEPLOY_RECEIPT_STATUS VPS_DEPLOY_RECEIPT_RECOVERY\n  export VPS_DEPLOY_RECEIPT_PINNED_SHA VPS_DEPLOY_RECEIPT_PREVIOUS_SHA\n  export VPS_DEPLOY_RECEIPT_SERVICES VPS_DEPLOY_RECEIPT_PORTS VPS_DEPLOY_RECEIPT_ORDER\n  write_deploy_run_receipt || {\n    print -u2 \"vps-deploy: cannot create the durable recovery receipt; refusing before any service is touched\"\n    exit 1\n  }\nfi\n\n# A parent can be killed while a child app deploy is still unwinding. The\n# receipt identifies the parent, so also refuse while another deployer process\n# is visible. Exclude this recovery command by its own PID.\nother_deploy_processes() {\n  local pid exclude_pid=${1:-} matches rc\n  local -a candidates\n  if matches=$(pgrep -f 'OPENBSD/bin/vps-deploy' 2>/dev/null); then\n    :\n  else\n    rc=$?\n    (( rc == 1 )) || return 1\n  fi\n  candidates=(${(f)matches})\n  for pid in $candidates; do\n    [[ $pid == \"${$}\" || ( -n $exclude_pid && $pid == \"$exclude_pid\" ) ]] || print -r -- \"$pid\"\n  done\n}\n\n# Operator-driven recovery is intentionally separate from deploy. It never pulls,\n# checks out code, runs migrations or replays a deploy; it restores only the\n# service set recorded as running before the interrupted run.\nif [[ $app == recover ]]; then\n  runs_json=$(admin_request deploy-run-list) || {\n    print -u2 \"vps-deploy: cannot list recovery receipts; install and verify the pub5-admin mirror first\"\n    exit 1\n  }\n  if [[ $recover_mode == plan ]]; then\n    report=$(\"$RUBY\" -rjson -e '\n      rows = JSON.parse(STDIN.read)\n      rows.select { |r| r[\"recovery_state\"] == \"pending\" && %w[running failed].include?(r[\"status\"]) }.each do |r|\n        puts [r[\"run_id\"], r[\"target\"], r[\"pinned_sha\"] || \"not-pinned\",\n              r[\"stage\"], r[\"status\"], r[\"pid\"], r[\"owner_identity\"],\n              r[\"host\"], r[\"snapshot_recorded\"], r[\"run_log\"]].join(\"|\")\n      end\n    ' <<< \"$runs_json\") || exit 1\n    if [[ -z $report ]]; then\n      print \"vps-deploy: no pending interrupted-deploy recovery receipts\"\n      exit 0\n    fi\n    while IFS='|' read -r run_id target sha stage status owner_pid owner_identity owner_host snapshot_recorded run_log; do\n      [[ -n $run_id ]] || continue\n      if [[ $owner_host != \"$(hostname -s)\" ]]; then\n        print \"vps-deploy: FOREIGN $run_id host=$owner_host target=$target - recover on the recorded host\"\n        continue\n      fi\n      if [[ $snapshot_recorded != true ]]; then\n        print \"vps-deploy: INCOMPLETE $run_id target=$target stage=$stage - service snapshot is not complete; do not apply\"\n        continue\n      fi\n      other_pids=$(other_deploy_processes \"$owner_pid\") || {\n        print \"vps-deploy: AMBIGUOUS $run_id - cannot verify the process table; do not recover\"\n        continue\n      }\n      if [[ -n $other_pids ]]; then\n        print \"vps-deploy: AMBIGUOUS $run_id - another deployer process is still present; do not recover\"\n        print -r -- \"  other deployer PID(s): $other_pids\"\n        continue\n      fi\n      if ps -p \"$owner_pid\" >/dev/null 2>&1; then\n        current_identity=$(lock_process_identity \"$owner_pid\") || {\n          print \"vps-deploy: AMBIGUOUS $run_id - owner PID exists but its identity cannot be checked\"\n          continue\n        }\n        if [[ $current_identity == \"$owner_identity\" ]]; then\n          print \"vps-deploy: LIVE $run_id target=$target sha=$sha stage=$stage pid=$owner_pid - do not recover\"\n          continue\n        fi\n      fi\n      print \"vps-deploy: RECOVERY AVAILABLE $run_id target=$target sha=$sha stage=$stage\"\n      print \"  log=$run_log\"\n    done <<< \"$report\"\n    print \"vps-deploy: inspect the receipt and its log before: zsh OPENBSD/bin/vps-deploy recover --apply <run-id>\"\n    exit 0\n  fi\n\n  receipt_json=$(admin_request deploy-run-read \"$recover_id\") || exit 1\n  [[ $receipt_json != null ]] || {\n    print -u2 \"vps-deploy: no recovery receipt found for $recover_id\"\n    exit 1\n  }\n  details=$(\"$RUBY\" -rjson -e '\n    r = JSON.parse(STDIN.read)\n    abort \"receipt is not pending recovery\" unless\n      r[\"recovery_state\"] == \"pending\" && %w[running failed].include?(r[\"status\"])\n    abort \"receipt snapshot is incomplete\" unless r[\"snapshot_recorded\"] == true\n    puts [\"META\", r.fetch(\"pid\"), r.fetch(\"owner_identity\"), r.fetch(\"stage\"),\n          r.fetch(\"target\"), r[\"pinned_sha\"] || \"not-pinned\", r.fetch(\"host\")].join(\"|\")\n    puts [\"SERVICES\", r.fetch(\"started_services\").join(\" \")].join(\"|\")\n    puts [\"ORDER\", r.fetch(\"restore_order\").join(\" \")].join(\"|\")\n    r.fetch(\"ports\").sort.each { |service, port| puts [\"PORT\", service, port].join(\"|\") }\n  ' <<< \"$receipt_json\") || {\n    print -u2 \"vps-deploy: receipt is corrupt, incomplete or not eligible for recovery\"\n    exit 1\n  }\n\n  typeset -a recovery_lines\n  recovery_lines=(${(f)details})\n  IFS='|' read -r kind owner_pid owner_identity original_stage original_target original_sha owner_host <<< \"$recovery_lines[1]\"\n  [[ $owner_host == \"$(hostname -s)\" ]] || {\n    print -u2 \"vps-deploy: receipt belongs to $owner_host; recovery must run on that host\"\n    exit 1\n  }\n  other_pids=$(other_deploy_processes \"$owner_pid\") || {\n    print -u2 \"vps-deploy: cannot verify the process table; refusing recovery\"\n    exit 75\n  }\n  [[ -z $other_pids ]] || {\n    print -u2 \"vps-deploy: another deployer process is still present; refusing recovery\"\n    print -u2 \"  other deployer PID(s): $other_pids\"\n    exit 75\n  }\n  if ps -p \"$owner_pid\" >/dev/null 2>&1; then\n    current_identity=$(lock_process_identity \"$owner_pid\") || {\n      print -u2 \"vps-deploy: owner PID exists but its identity cannot be checked; refusing recovery\"\n      exit 75\n    }\n    if [[ $current_identity == \"$owner_identity\" ]]; then\n      print -u2 \"vps-deploy: original deploy $recover_id is still running; refusing recovery\"\n      exit 75\n    fi\n  fi\n\n  VPS_DEPLOY_STARTED=\n  DEPLOY_ALL_DEGRADED=()\n  deploy_ports=()\n  for ((i=2; i<=${#recovery_lines}; i++)); do\n    IFS='|' read -r kind one two <<< \"$recovery_lines[$i]\"\n    case $kind in\n      SERVICES) VPS_DEPLOY_STARTED=$one ;;\n      ORDER) DEPLOY_ALL_DEGRADED=(${=one}) ;;\n      PORT) deploy_ports[$one]=$two ;;\n    esac\n  done\n  [[ -n $VPS_DEPLOY_STARTED ]] || {\n    print -u2 \"vps-deploy: receipt has no captured running services; refusing an inferred restore\"\n    exit 1\n  }\n  export VPS_DEPLOY_STARTED\n  export VPS_DEPLOY_RESTORE_PORTS_FROM_RECEIPT=1\n  VPS_DEPLOY_RESTORE_ATTEMPTED=1\n\n  print \"vps-deploy: recovery target=$original_target pinned=$original_sha failed_stage=$original_stage\"\n  print \"vps-deploy: only the recorded service snapshot will be restored\"\n  keep_deploy_flag\n  recovery_payload=$(\"$RUBY\" -rjson -e 'puts JSON.generate(status: \"failed\", recovery_state: \"pending\", stage: \"recovery-restore\")')\n  admin_request deploy-run-write \"$recover_id\" \"$recovery_payload\" || {\n    print -u2 \"vps-deploy: cannot persist recovery-attempt stage; no services were changed\"\n    exit 1\n  }\n\n  recovery_failed=0\n  restore_started || recovery_failed=1\n  if (( ! recovery_failed )); then\n    restore_wait=${VPS_DEPLOY_RESTORE_WAIT:-240}\n    for svc in ${=VPS_DEPLOY_STARTED}; do\n      admin_service check \"$svc\" >/dev/null 2>&1 || {\n        print -u2 \"vps-deploy: recovery verification failed for $svc (rcctl check)\"\n        recovery_failed=1\n        break\n      }\n      port=${deploy_ports[$svc]:-}\n      if [[ -n $port ]] && ! wait_up \"$port\" \"$restore_wait\"; then\n        print -u2 \"vps-deploy: recovery verification failed for $svc /up on port $port\"\n        recovery_failed=1\n        break\n      fi\n    done\n  fi\n\n  if (( recovery_failed )); then\n    recovery_payload=$(\"$RUBY\" -rjson -e 'puts JSON.generate(status: \"failed\", recovery_state: \"pending\", stage: \"recovery-failed\")')\n    admin_request deploy-run-write \"$recover_id\" \"$recovery_payload\" || \\\n      print -u2 \"vps-deploy: recovery failed and the pending receipt could not be updated\"\n    print -u2 \"vps-deploy: recovery remains pending; inspect /var/log/daemon before another attempt\"\n    exit 1\n  fi\n\n  recovery_payload=$(\"$RUBY\" -rjson -e 'puts JSON.generate(status: \"recovered\", recovery_state: \"complete\", stage: \"recovered\", recovered_at: Time.now.utc.iso8601)')\n  admin_request deploy-run-write \"$recover_id\" \"$recovery_payload\" || {\n    print -u2 \"vps-deploy: services answered, but the recovery receipt could not be finalized\"\n    exit 1\n  }\n  print \"vps-deploy: recovered $recover_id target=$original_target pinned=$original_sha (previous stage=$original_stage)\"\n  exit 0\nfi\n\n# Pin the commit, prove it boots, record what is running.\n#\n# Done once, by the process that owns the lock, before any service is touched.\n#\n# The deploy used to pull origin/main at whatever instant each step reached its\n# pull, so a commit pushed mid-pass was deployed half-way through it. On 2026-10-12\n# the pulled tip carried a SyntaxError and later a law that raised at boot; master's\n# container died on the box and the pass halted with production already shed. Now\n# the target is one SHA, resolved here (VPS_DEPLOY_SHA, or origin/main fetched\n# once), and every later step checks HEAD against it instead of pulling.\n#\n# The checks (MASTER/lib/operator/canary.rb) ran on the operator's\n# machine when `bin/operator vps deploy --remote` started this run and said so with\n# VPS_DEPLOY_PREFLIGHT_SHA; a direct run on the box has no such word and runs the\n# MASTER-only subset itself on the pinned commit, detached, so a refusal leaves the\n# checkout where it was. VPS_DEPLOY_SKIP_PREFLIGHT=1 is break-glass and, like the\n# runtime-check skip, needs the human-only I_UNDERSTAND_FAST_DEPLOY=1.\ndeploy_refuse() { log \"refused: $1\"; print -u2 \"vps-deploy: refused - $1\"; print -u2 \"            Nothing was stopped.\"; exit 1 }\nif (( lock_owner )); then\n  set_stage pin\n    retry \"${VPS_DEPLOY_GIT_RETRIES:-3}\" 2 git_as_deploy_user fetch -q origin main \\\n    || deploy_refuse \"git fetch origin main failed (as $DEPLOY_USER) after retries\"\n  pin=$(git_as_deploy_user rev-parse --verify -q \"${VPS_DEPLOY_SHA:-FETCH_HEAD}^{commit}\") \\\n    || deploy_refuse \"${VPS_DEPLOY_SHA:-origin/main} is not a commit this checkout can see\"\n  head_before=$(git_as_deploy_user rev-parse HEAD)\n  git_as_deploy_user merge-base --is-ancestor \"$head_before\" \"$pin\" \\\n    || deploy_refuse \"$pin is not a fast-forward of the box's HEAD $head_before\"\n  export VPS_DEPLOY_SHA=$pin\n  print \"vps-deploy: pinned $pin (box HEAD was $head_before; git as $DEPLOY_USER)\"\n  # A CI attestation names one commit; tests that passed on another prove nothing here.\n  if [[ -n ${VPS_DEPLOY_CI_SHA:-} ]]; then\n    [[ $VPS_DEPLOY_CI_SHA == $pin ]] \\\n      || deploy_refuse \"bin/ci ran on $VPS_DEPLOY_CI_SHA but this run would deploy $pin\"\n    print \"vps-deploy: bin/ci passed on the operator's machine for $pin; the box skips its suite\"\n  fi\n  if [[ -n ${VPS_DEPLOY_PREFLIGHT_SHA:-} ]]; then\n    [[ $VPS_DEPLOY_PREFLIGHT_SHA == $pin ]] \\\n      || deploy_refuse \"preflight ran on $VPS_DEPLOY_PREFLIGHT_SHA but this run would deploy $pin\"\n    print \"vps-deploy: preflight passed on the operator's machine for $pin\"\n  elif [[ ${VPS_DEPLOY_SKIP_PREFLIGHT:-} == 1 ]]; then\n    [[ ${I_UNDERSTAND_FAST_DEPLOY:-} == 1 ]] \\\n      || deploy_refuse \"VPS_DEPLOY_SKIP_PREFLIGHT=1 needs the human-only I_UNDERSTAND_FAST_DEPLOY=1\"\n    print \"vps-deploy: WARNING - preflight skipped (break-glass); $pin was not proven to boot\"\n  else\n    set_stage preflight-boot\n    print \"vps-deploy: preflight on $pin (syntax, law files, boot smoke)\"\n    preflight_back_to=$(git_as_deploy_user symbolic-ref -q --short HEAD || print -r -- \"$head_before\")\n    # A box left detached comes back to its commit, not to a guessed branch.\n    preflight_back_flag=()\n    git_as_deploy_user symbolic-ref -q HEAD >/dev/null || preflight_back_flag=(--detach)\n    # Set recovery state before switching: the EXIT/HUP/INT/TERM path now owns\n    # restoration even if the signal lands immediately after the detach.\n    preflight_active=1\n    git_as_deploy_user switch -q --detach \"$pin\" || deploy_refuse \"cannot check out $pin in $repo (local changes?)\"\n    if ! \"$RUBY\" \"$repo/MASTER/lib/operator/canary.rb\" --no-rehearse; then\n      restore_preflight_checkout || deploy_refuse \"preflight failed on $pin and checkout restoration failed; restore $preflight_back_to before retrying\"\n      deploy_refuse \"preflight failed on $pin; the checkout is back on $preflight_back_to and the previous release is still serving\"\n    fi\n    restore_preflight_checkout || deploy_refuse \"cannot return to $preflight_back_to after preflight\"\n  fi\n  git_as_deploy_user merge -q --ff-only \"$pin\" || deploy_refuse \"cannot fast-forward $repo to $pin\"\n  VPS_DEPLOY_STARTED=\"$(admin_service list-started 2>/dev/null)\"\n  VPS_DEPLOY_STARTED=${VPS_DEPLOY_STARTED//$'\\n'/ }\n  export VPS_DEPLOY_STARTED\n  VPS_DEPLOY_RECEIPT_PREVIOUS_SHA=$head_before\n  VPS_DEPLOY_RECEIPT_PINNED_SHA=$pin\n  VPS_DEPLOY_RECEIPT_SERVICES=$VPS_DEPLOY_STARTED\n  VPS_DEPLOY_RECEIPT_RECOVERY=none\n  [[ -z $VPS_DEPLOY_STARTED ]] || VPS_DEPLOY_RECEIPT_RECOVERY=pending\n  export VPS_DEPLOY_RECEIPT_PREVIOUS_SHA VPS_DEPLOY_RECEIPT_PINNED_SHA\n  export VPS_DEPLOY_RECEIPT_SERVICES VPS_DEPLOY_RECEIPT_RECOVERY\n  print \"vps-deploy: started before this deploy: $VPS_DEPLOY_STARTED\"\n  set_stage snapshot\nfi\n\n  # Resolve against the inventory in the checkout after it has been fast-forwarded\n  # to VPS_DEPLOY_SHA. A stale box checkout must not define target membership.\n  deploy_target_kind=$(inventory_target_kind \"$app\") || deploy_refuse \"could not read the pinned deployment inventory for $app\"\n  [[ $deploy_target_kind != unknown ]] || deploy_refuse \"unknown app or group $app in the pinned RAILS/apps.yml\"\n  DEPLOY_ALL=(${(@f)$(inventory_names all)})\n  DEPLOY_ALL_DEGRADED=(${(@f)$(inventory_names all 1)})\n  DEPLOY_RAILS=(${(@f)$(inventory_names rails)})\n  load_deploy_ports\n  typeset -a receipt_ports\n  for svc in ${(k)deploy_ports}; do\n    receipt_ports+=(\"$svc=${deploy_ports[$svc]}\")\n  done\n  VPS_DEPLOY_RECEIPT_PORTS=\"${(j:,:)receipt_ports}\"\n  VPS_DEPLOY_RECEIPT_ORDER=\"${(j:,:)DEPLOY_ALL_DEGRADED}\"\n  VPS_DEPLOY_RECEIPT_SNAPSHOT_RECORDED=1\n  export VPS_DEPLOY_RECEIPT_PORTS VPS_DEPLOY_RECEIPT_ORDER\n  export VPS_DEPLOY_RECEIPT_SNAPSHOT_RECORDED\n  set_stage inventory-ready\n  (( ${#DEPLOY_ALL} )) || deploy_refuse \"the pinned deployment inventory has no active apps\"\n  deploy_app_group=$(inventory_app_group \"$app\")\n  if [[ $deploy_target_kind == all || $deploy_app_group == master ]]; then\n    ensure_smart_turn_assets\n  fi\n\n# The deployment sequence is data in RAILS/apps.yml. Degraded mode\n# moves the MASTER namespace last; all other relative order stays unchanged.\nchoose_deploy_order() {\n  local svc port avail why=\n  load_deploy_ports\n  avail=$(\"$RUBY\" -e 'print File.readlines(ARGV[0]).last.to_s[/mem_avail=(\\d+)/, 1].to_s' \\\n    \"${VPS_DEPLOY_GUARD_LOG:-/var/log/resource_guard_history.log}\" 2>/dev/null)\n  for svc in $DEPLOY_RAILS; do\n    (( ${${=VPS_DEPLOY_STARTED:-}[(Ie)$svc]} )) || continue\n    port=${deploy_ports[$svc]:-}\n    [[ -n $port ]] && wait_up \"$port\" 5 || why=\"$svc is not answering /up\"\n  done\n  if [[ -z $why ]]; then\n    [[ $avail == <-> ]] || why=\"memory availability unreadable\"\n  fi\n  if [[ -z $why ]] && (( avail < ${VPS_DEPLOY_MASTER_MIN_AVAIL:-10} )); then\n    why=\"memory availability ${avail}% is under ${VPS_DEPLOY_MASTER_MIN_AVAIL:-10}%\"\n  fi\n  if [[ -z $why ]]; then\n    deploy_order=($DEPLOY_ALL)\n  else\n    print \"vps-deploy: all - MASTER moved to the end: $why\"\n    deploy_order=($DEPLOY_ALL_DEGRADED)\n  fi\n}\n\nif [[ $deploy_target_kind == all || $deploy_target_kind == group ]]; then\n  # The parent spans child deploys and cooldowns. Its own flag keeps the\n  # resource guard out until the whole batch (or its failure recovery) ends.\n  keep_deploy_flag\n  typeset -i first=1\n  typeset -a deploy_order\n  if [[ $deploy_target_kind == all ]]; then\n    choose_deploy_order\n  else\n    deploy_order=(${(@f)$(inventory_names \"$app\")})\n  fi\n  (( ${#deploy_order} )) || { print -u2 \"vps-deploy: deployment group $app is empty\"; exit 2 }\n  print \"vps-deploy: $app order: ${deploy_order}\"\n  if [[ $deploy_target_kind == all ]]; then\n    set_stage openbsd-mirror\n    print \"vps-deploy: applying tracked OpenBSD filesystem mirror\"\n    zsh \"$repo/OPENBSD/OPERATOR.sh\" --apply || deploy_refuse \"OpenBSD filesystem mirror apply failed; no application deployment was started\"\n  fi\n  for one in $deploy_order; do\n    # Let the previous app's deploy drain before starting the next one's CI.\n    # Serial builds protect this one-core host from overlapping load peaks.\n    if (( ! first )); then\n      print \"vps-deploy: $app - settling for ${PUB5_DEPLOY_COOLDOWN:-60}s before $one\"\n      sleep ${PUB5_DEPLOY_COOLDOWN:-60}\n    fi\n    first=0\n    print \"vps-deploy: $app -> $one\"\n    # Each child executes the complete per-app CI, sync, migration, restart,\n    # health-check and deploy-stamp path. A failure halts this group.\n    set_stage \"$app:$one\"\n    zsh \"$0\" \"$one\" || { halted_app=$one; print -u2 \"vps-deploy: $app halted at $one\"; exit 1 }\n  done\n  print \"vps-deploy: $app ok (${deploy_order})\"\n  exit 0\nfi\n\n# SHA and time are captured at write_stamp, not here. vps_deploy_master.sh\n# pulls origin/main; reading HEAD before that call stamped the pre-pull SHA\n# (76f1a65fa on 2026-08-12 while /health correctly said 5eff4d87a).\n\nwrite_stamp() {\n  local name=$1 stamp_status=$2\n  local sha\n  stamp_written=1\n  sha=$(git_as_deploy_user rev-parse --short HEAD)\n  admin_request stamp \"$name\" \"$sha\" \"$stamp_status\"\n}\n\nset_stage \"$deploy_app_group\"\nif [[ $deploy_app_group == master ]]; then\n  print \"vps-deploy: master web\"\n  # master's bundle, native gem builds and precompile run as long as an app's CI,\n  # and this branch never reached the keeper below: on 2026-10-27 the guard read\n  # deploy=0 through a 30-minute master deploy and shed amber and bsdports twice.\n  keep_deploy_flag\n  settle_boots\n  if zsh \"$repo/OPENBSD/bin/vps_deploy_master.sh\"; then\n    # The loopback journeys that need no browser, for this app only.\n    set_stage master-flows\n    hold_deploy_flag\n    if ! FLOW_APP=\"$app\" MASTER_STRICT=deploy CHECK_AUTOFIX=0 CHECK_REQUIRE_LIVE=1 \"$RUBY\" \"$repo/MASTER/checks/runner.rb\" flow_journey; then\n      write_stamp \"$app\" failed\n      print -u2 \"vps-deploy: master flow journeys failed - see the per-flow lines above\"\n      exit 1\n    fi\n    set_stage master-canary\n    if ! \"$RUBY\" \"$repo/OPENBSD/bin/canary\" --local --app \"$app\" --no-flows; then\n      write_stamp \"$app\" failed\n      print -u2 \"vps-deploy: master local canary failed - see the failed checks above\"\n      exit 1\n    fi\n    set_stage restore-shed\n    if restore_started; then\n      restore_ok=1\n    else\n      restore_ok=0\n    fi\n    if (( ! restore_ok )); then\n      write_stamp \"$app\" failed\n      print -u2 \"vps-deploy: master is not complete; a service that was running before is still down\"\n      exit 1\n    fi\n    write_stamp \"$app\" ok\n    exit 0\n  fi\n  # A failed run stamps failed, or last_deploy_master.json keeps saying ok\n  # from the last good run and /health reports a live SHA while :53187 is\n  # closed (measured 2026-08-13).\n  write_stamp \"$app\" failed\n  exit 1\nfi\n\n[[ -d /home/${app}/app ]] || { print -u2 \"missing /home/${app}/app\"; exit 1 }\n\n# Rails apps do not pull inside vps_ci.sh, so without this step a deploy CIs and\n# syncs whatever the VPS checkout already holds, which can lag origin/main by\n# days. master pulls inside vps_deploy_master.sh; this is the matching step for\n# the three copy-tree apps.\nset_stage pull\n# No pull here: the top-level process pinned one SHA and moved the checkout to it.\n# A checkout that has moved off it since means another hand touched the box.\nprint \"vps-deploy: $app (checkout must be ${VPS_DEPLOY_SHA:?no pinned SHA, run through the top-level vps-deploy})\"\n[[ $(git_as_deploy_user rev-parse HEAD) == \"$VPS_DEPLOY_SHA\" ]] \\\n  || { print -u2 \"vps-deploy: $app refused - HEAD is $(git_as_deploy_user rev-parse HEAD), not the pinned $VPS_DEPLOY_SHA\"; exit 1 }\n\n# Free space, before anything is stopped. A bundle install, an asset precompile or\n# a CI mirror that runs out of disk dies halfway, after the app was stopped for the\n# migrate; refusing here costs nothing and leaves the previous release serving.\n# /home holds the checkout, the app trees, bundles and storage (3.9 GB free of\n# 17 GB on 2026-10-12), so it is the filesystem that fills. The floor is a\n# variable because it is a measurement of this box, not a law.\nset_stage preflight-disk\nmin_free_kb=${VPS_DEPLOY_MIN_FREE_KB:-1048576}\nfor mount_point in /home /var; do\n  free_kb=${${(z)${${(f)\"$(${VPS_DEPLOY_DF:-df} -Pk $mount_point 2>/dev/null)\"}[2]}}[4]:-}\n  if [[ $free_kb != <-> ]]; then\n    print -u2 \"vps-deploy: $app refused - free-space measurement for $mount_point is unavailable.\"\n    print -u2 \"            Nothing was stopped. Check the filesystem and measurement command, then rerun:\"\n    print -u2 \"            doas zsh OPENBSD/bin/vps-deploy $app\"\n    exit 1\n  fi\n  if (( free_kb < min_free_kb )); then\n    print -u2 \"vps-deploy: $app refused - $mount_point has $(( free_kb / 1024 )) MB free, the floor is $(( min_free_kb / 1024 )) MB.\"\n    print -u2 \"            Nothing was stopped. Inspect usage with: doas /usr/local/sbin/pub5-admin disk-usage $mount_point ; clear old\"\n    print -u2 \"            logs or bundles, then rerun: doas zsh OPENBSD/bin/vps-deploy $app\"\n    exit 1\n  fi\ndone\n\nset_stage preflight\nprint \"vps-deploy: $app (preflight load)\"\nadmin_request resource-guard || true\n\nkeep_deploy_flag\n\n# What is about to be deployed, against what is actually running.\n#\n# The check compares OPENBSD/mirror/etc byte-for-byte with live /etc and root's crontab\n# with data/root.crontab. A deploy is exactly when that matters — it is the moment\n# the repo becomes the box.\n#\n# Advisory, not blocking, and deliberately so. Measured 2026-09-10 with\n# `SSH_HOST=dev@brgen.no ruby OPENBSD/checks/config_drift_check.rb --remote`: eight files\n# differ and two more are absent from the box entirely — emergency_cpu.sh, which\n# is resource_guard's crisis tier, and vps_weekly_integrity.sh, which the crontab\n# has scheduled for weeks. A hard check here would refuse every deploy until an\n# OPERATOR.sh run nobody has scheduled, and one of the eight is rc.d/master, which\n# starts a production service. Make it blocking the first time a deploy runs with\n# this clean; a warning that names a number is still the difference between\n# knowing and not.\nif [[ -x /usr/local/bin/config_drift_check.rb ]]; then\n  if ! \"$RUBY\" /usr/local/bin/config_drift_check.rb > /tmp/vps-deploy-drift.out 2>&1; then\n    # ruby, not grep: coreutils are banned in committed scripts here, and the\n    # check's own summary line already carries the count in the shape we want.\n    drift_count=$(\"$RUBY\" -e \\\n      'print File.read(\"/tmp/vps-deploy-drift.out\").scan(/: DRIFT -/).size' 2>/dev/null)\n    print \"vps-deploy: $app WARNING - ${drift_count:-?} config(s) on the box differ from the repo being deployed\"\n    print \"vps-deploy: $app see /tmp/vps-deploy-drift.out\"\n    # Blocking is opt-in until the box has run clean once; the default stays the\n    # warning above so an unscheduled OPERATOR.sh run cannot freeze every deploy.\n    if [[ ${VPS_DEPLOY_DRIFT:-warn} == fail ]]; then\n      print -u2 \"vps-deploy: $app refused - VPS_DEPLOY_DRIFT=fail and the box has drifted\"\n      exit 1\n    fi\n  fi\nfi\n\nif [[ ${SKIP_CI:-} == 1 ]]; then\n  print \"vps-deploy: $app (sync via ${app}.sh, SKIP_CI=1)\"\n  # The app entrypoints refuse root (line 5 of every one of them: checkout\n  # code is operator-owned, privileged actions go through dev's doas and the\n  # pub5_rcctl broker), so the sync runs as the deploy user, not as this\n  # script's root. First exercised on vm23 2026-11-08: as root, brgen.sh\n  # refused and the app stage died before stop, leaving the release serving.\n  run_as_deploy_user \"cd $(printf '%q' \"$repo\") && zsh $(printf '%q' \"$repo/RAILS/${app}/${app}.sh\")\"\nelse\n  # The app's resident Solid Queue worker stands down for the CI run.\n  #\n  # vm23 is 1 GB. brgen_jobs holds ~380 MB resident and brgen's suite is 896\n  # tests: on 2026-08-23 the runner was OOM-killed mid-suite with swap at 898 MB\n  # of 1264, and the failure is silent in the worst way — 155 dots, no summary,\n  # no failure block, and vps_ci reporting \"Tests: Rails failed\". Dots then\n  # nothing in a CI test section means the runner was killed, not that a test\n  # failed. Stopped and restarted here so the knowledge lives in the deploy\n  # rather than in whoever remembers to type it.\n  # No other app may still be booting when CI starts: a Falcon cold boot and a test\n  # suite on one core is how the apps were shed.\n  settle_boots\n  jobs_service=\"${app}_jobs\"\n  jobs_was_up=0\n  if admin_service get \"$jobs_service\" >/dev/null 2>&1 && admin_service check \"$jobs_service\" >/dev/null 2>&1; then\n    print \"vps-deploy: $app (stand down $jobs_service for the CI run)\"\n    admin_service stop \"$jobs_service\" >/dev/null 2>&1 && { jobs_was_up=1; own_stops+=(\"$jobs_service\") }\n  fi\n\n  # Swap floor, measured after the stand-down above has freed what it frees.\n  # CI, bundle and asset work on a 1 GB box with swap nearly gone is how every\n  # app was shed: on 2026-11-03 a gem fetch at ~200 MB free took all sites down\n  # for 17 minutes, and on 2026-11-04 a MASTER run at 217 MB free timed brgen out.\n  set_stage preflight-swap\n  min_swap_mb=${VPS_DEPLOY_MIN_FREE_SWAP_MB:-400}\n  swap_blocks=${${(M)${(z)\"$(${VPS_DEPLOY_SWAPCTL:-swapctl} -s 2>/dev/null)\"}:#<->}[3]:-}\n  if [[ $swap_blocks == <-> ]] && (( swap_blocks * 512 / 1048576 < min_swap_mb )); then\n    (( jobs_was_up )) && admin_service start \"$jobs_service\" >/dev/null 2>&1\n    free_swap_mb=$(( swap_blocks * 512 / 1048576 ))\n    print -u2 \"vps-deploy: $app refused - $free_swap_mb MB of swap free, the floor is $min_swap_mb MB.\"\n    print -u2 \"            The app's jobs service is back up; nothing else was stopped. Free memory (stop another\"\n    print -u2 \"            app's *_jobs service), then rerun: doas zsh OPENBSD/bin/vps-deploy $app\"\n    exit 1\n  fi\n\n  set_stage ci+sync\n  print \"vps-deploy: $app (ci + sync)\"\n  # `|| ci_status=$?` rather than a bare call and `ci_status=$?` after it. This\n  # script runs under `set -e`, so a bare failing CI run exits here and the\n  # restart below — the whole reason the stand-down records jobs_was_up — never\n  # runs, leaving the worker stopped with nothing 5xx'ing and the queue simply not\n  # draining. A command followed by `||` is exempt from errexit, which is what\n  # makes the status capture reachable at all.\n  ci_status=0\n  zsh \"$repo/OPENBSD/bin/vps_ci.sh\" \"$app\" || ci_status=$?\n\n  if (( jobs_was_up )); then\n    admin_service start \"$jobs_service\" >/dev/null 2>&1 \\\n      || print -u2 \"vps-deploy: $jobs_service did not come back - start it\"\n  fi\n  (( ci_status == 0 )) || exit $ci_status\nfi\n\n# Primary database, then the cache/queue/cable schemas.\n#\n# db_create_migrate_as_app runs db:prepare, which is idempotent — it creates the\n# database when absent, loads the schema when empty, and otherwise runs only\n# pending migrations — and then does the secondary schemas itself. Loading the\n# secondaries alone leaves the primary unmigrated, so a merged migration never\n# reaches production: on 2026-07-29 brgen shipped a counter-cache column its\n# views already read, and every feed render 500'd until an operator ran the\n# three pending migrations by hand.\nset_stage migrate\nprint \"vps-deploy: $app (primary db migrate + secondary schemas)\"\n. \"$repo/RAILS/_core.sh\"\n. \"$repo/RAILS/_bundle.sh\"\n. \"$repo/RAILS/_runtime_check.sh\"\n. \"$repo/RAILS/_database.sh\"\n. \"$repo/RAILS/_assets.sh\"\n# Same reason RAILS/_deploy.sh stops first: db:prepare hangs on the write\n# lock a live Falcon worker already holds on production.sqlite3.\nprint \"vps-deploy: $app (stop for sqlite migrate)\"\nadmin_service stop \"$app\" 2>/dev/null || true\napp_stopped=1\nown_stops+=(\"$app\")\ndb_create_migrate_as_app \"$app\" \"/home/${app}/app\"\n\n# The SKIP_CI=1 path (${app}.sh -> deploy_tracked_app -> rails_runtime_check)\n# already precompiles as part of its check. The default ci+sync path\n# (vps_ci.sh) only runs tests against a throwaway test DB and never\n# touches production assets -- without this, public/assets/ is simply\n# never created for a plain `vps-deploy $app` run, so Propshaft 404s\n# every asset_path() and the page renders with no CSS at all. Safe to\n# call unconditionally: Propshaft precompile is idempotent.\nset_stage precompile\nprint \"vps-deploy: $app (assets precompile)\"\nsettle_boots\nrails_assets_precompile_as_app \"$app\" \"/home/${app}/app\" || exit 1\n\nset_stage restart\nprint \"vps-deploy: $app (restart)\"\nhold_deploy_flag\nif ! admin_service_retry restart \"$app\"; then\n  log \"restart failed after retries\"\n  write_stamp \"$app\" failed\n  exit 1\nfi\nif ! retry 5 2 admin_service check \"$app\"; then\n  log \"service check failed after restart\"\n  write_stamp \"$app\" failed\n  exit 1\nfi\napp_stopped=0\nset_stage health\nport=$(\"$RUBY\" -ryaml -e 'puts YAML.safe_load(File.read(\"'\"$repo\"'/RAILS/apps.yml\")).dig(\"apps\",\"'\"$app\"'\",\"port\")')\nup_ok=0\nfor i in {1..150}; do\n  curl -fsS \"http://127.0.0.1:${port}/up\" >/dev/null 2>&1 && { up_ok=1; break; }\n  sleep 2\ndone\n[[ $up_ok -eq 1 ]] || { write_stamp \"$app\" failed; print -u2 \"vps-deploy: $app /up not ready on :${port}\"; exit 1; }\n\n# /up answers before Propshaft is involved, so it says nothing about whether the\n# page a visitor gets is styled. On 2026-08-14 brgen served every page with a\n# <link> to a stylesheet that 404'd — public/assets held zero files — and /up,\n# rcctl check, the port probe and the public TLS check all passed. The site was\n# plainly broken and the only thing that noticed was a person looking at it.\n#\n# So ask for the stylesheet the page actually links, through the Host that owns\n# it: these apps are domain-constrained and a bare-IP request is a 403, which\n# would make this check pass by never finding a link to test.\ndomain=$(\"$RUBY\" -ryaml -e 'puts YAML.safe_load(File.read(\"'\"$repo\"'/RAILS/apps.yml\")).dig(\"apps\",\"'\"$app\"'\",\"domain\")')\n# The page and the link are two reads, because under set -e and pipefail one\n# pipeline that finds no link fails the assignment and ends the deploy with no\n# stamp and no message. A home page that does not answer is a failed deploy; a\n# home page that links no stylesheet is the else branch below.\nhome_page=$(curl -fsS --max-time 30 -H \"Host: ${domain}\" \"http://127.0.0.1:${port}/\" 2>/dev/null) || {\n  write_stamp \"$app\" failed\n  print -u2 \"vps-deploy: $app / did not answer on :${port} with Host ${domain}\"\n  exit 1\n}\ncss_href=$(print -r -- \"$home_page\" | \"$RUBY\" -e 'print $stdin.binmode.read[%r{/assets/[^\"]+\\.css}n].to_s')\nif [[ -n $css_href ]]; then\n  if curl -fsS -o /dev/null --max-time 30 -H \"Host: ${domain}\" \"http://127.0.0.1:${port}${css_href}\" 2>/dev/null; then\n    print \"vps-deploy: $app stylesheet ok (${css_href})\"\n  else\n    write_stamp \"$app\" failed\n    print -u2 \"vps-deploy: $app serves ${css_href} as 404 - precompiled assets missing, the site is unstyled\"\n    exit 1\n  fi\nelse\n  print \"vps-deploy: $app no /assets stylesheet linked on / - nothing to verify\"\nfi\n\nset_stage solid_queue\nqueue_schema=\"$repo/RAILS/${app}/db/queue_schema.rb\"\n# An app whose production config runs jobs in-process has no worker to prove;\n# demanding one, or an hourly drain, would be meaningless.\n# UTF-8 explicitly: the box runs this with no LANG, and production.rb has em dashes.\nqueue_async=0\n\"$RUBY\" -e 'exit File.read(ARGV[0], encoding: \"UTF-8\").match?(/queue_adapter\\s*=\\s*:async\\b/)' \\\n  \"$repo/RAILS/${app}/config/environments/production.rb\" 2>/dev/null && queue_async=1\nqueue_empty=0\n[[ -f $queue_schema ]] && \"$RUBY\" -e 'exit File.read(ARGV[0]).include?(\"define(version: 0)\")' \\\n  \"$queue_schema\" && queue_empty=1\nif (( ! queue_async && ! queue_empty )) && [[ -f $queue_schema ]]; then\n  print \"vps-deploy: $app (solid_queue proof)\"\n  admin_request solid-queue-proof \"$app\" \\\n    || { write_stamp \"$app\" failed; print -u2 \"vps-deploy: solid_queue proof failed for $app\"; exit 1; }\nfi\n\n# Restore whatever the guard shed while we were deploying.\n#\n# resource_guard sheds its OPTIONAL list — bsdports, then amber — under\n# sustained pressure, and a deploy is exactly that. relayd keeps answering TLS\n# with their ports closed, so the outage reads as a hang rather than a 5xx and\n# nothing reports it: on 2026-08-23 amber and bsdports were restarted by hand\n# three times in one session, each time discovered by a person running curl.\n#\n# `vps-deploy all` folds the restore into its ordering; this pass gives a\n# single-app deploy, the common case, the same restore. Reported, not silent — a\n# service that needed restoring is a fact about the box worth seeing.\nset_stage restore-shed\nrestore_failed=0\nrestore_started || restore_failed=1\n\n# The checks, on the deployed thing.\n#\n# Outside this pass the checks run only when a person types a command, so the\n# half that can only be measured against a running app would be measured on a\n# laptop where it means less. A user journey is real only here: production\n# answers 403 to a request without the right Host header, which development\n# never does.\n#\n# This set is the cheap half: source checks plus the live probes against the\n# loopback. It runs in about forty seconds on this one core with the fleet up,\n# which is what makes it affordable on every deploy.\n#\n# The browser half — rendered_suite, layout_suite, visual_contract — is NOT here\n# and is opt-in through PUB5_DEPLOY_BROWSER_CHECKS=1. Chrome is installed, but\n# this box is 1GB with four apps resident and resource_guard sheds amber and\n# bsdports under exactly that pressure. A deploy that takes the site down to\n# check the site has failed at the only thing it was for.\n#\n# CHECK_AUTOFIX=0 because a check must not rewrite the tree it is judging, least\n# of all the one that is deployed.\n#\n# CHECK_REQUIRE_LIVE=1 because this is the one host where a live check has no\n# excuse. It turns \"port closed, skipping\" into a failure, and half this set —\n# flow_journey, page_simulation, port_inventory — measures nothing without a\n# listening app. The fleet was just restarted and the shed apps restored a few\n# lines above, so a closed port here is a deploy that did not come up, not a\n# laptop. bin/check-rails takes the same position on the box through\n# Environment.on_vps?, and a second caller on the same host must not quietly\n# accept a skip.\ncheck_names=(production port_inventory generated_asset schema_migration\n            phantom_foreign_keys frontend_production stimulus_components\n            flow_journey page_simulation)\nif [[ -n ${PUB5_DEPLOY_BROWSER_CHECKS:-} ]]; then\n  check_names+=(rendered_suite layout_suite)\nfi\nset_stage checks\nprint \"vps-deploy: $app (checks: ${check_names})\"\nhold_deploy_flag\nif ! FLOW_APP=\"$app\" MASTER_STRICT=deploy CHECK_AUTOFIX=0 \"$RUBY\" \"$repo/MASTER/checks/runner.rb\" \"${check_names[@]}\"; then\n  write_stamp \"$app\" failed\n  print -u2 \"vps-deploy: $app checks failed - the app is deployed and serving, and something\"\n  print -u2 \"            it is now serving does not hold. Fix or revert; do not deploy over it.\"\n  exit 1\nfi\n\nset_stage final-check\n# The app deployed last has to be the app still running when the deploy says ok.\n# Every check above ran minutes ago, and a deploy whose own app is down on exit\n# is the one outcome the stamp must never call ok.\nif ! admin_service check \"$app\" >/dev/null 2>&1; then\n  write_stamp \"$app\" failed\n  print -u2 \"vps-deploy: $app is down at the end of its own deploy - /usr/local/sbin/pub5-admin service restart $app, then read /var/log/daemon\"\n  exit 1\nfi\n\n# The local canary is a release gate, not an advisory log line.\nif ! \"$RUBY\" \"$repo/OPENBSD/bin/canary\" --local --app \"$app\"; then\n  write_stamp \"$app\" failed\n  print -u2 \"vps-deploy: $app local canary failed - see the failed checks above\"\n  exit 1\nfi\n\nif (( restore_failed )); then\n  write_stamp \"$app\" failed\n  print -u2 \"vps-deploy: $app is not complete; a service that was running before is still down\"\n  exit 1\nfi\n\nprint \"vps-deploy: $app ok\"\nwrite_stamp \"$app\" ok\nexit 0\n" to include "def write_deploy_run_receipt".
       
       83 runs, 720 assertions, 4 failures, 0 errors, 0 skips
ok     0.1s  OPENBSD/test/test_ci_mirror_contract.rb
ok     2.8s  OPENBSD/test/test_ruby_select.rb
ok     0.9s  RAILS/test/require_relative_targets_test.rb
ok     0.9s  RAILS/test/no_top_level_test_calls_test.rb
ok     0.5s  RAILS/test/coverage_layout_test.rb
ok     0.4s  RAILS/test/erb_container_balance_test.rb
ok     0.2s  RAILS/test/test_path_idioms_test.rb
ok     0.1s  RAILS/test/typography_lint_test.rb
rehearsal: 1 of 16 failed - fix these before vps-deploy

--- rehearsal_exit=1 ---

Embed this paste

Install this app

  1. Tap the Share button
  2. Scroll down and tap "Add to Home Screen"
  3. Tap "Add" in the top right

Please confirm