Protect root upgrade throttle state

docs/adr/0002-remote-worker-upgrade.md

@@ -53,6 +53,13 @@ GitHub is therefore accepted as the trust root. The residual risk is explicit: a
 
 ### D6. Fail-closed, throttled, rollback-guarded, with a terminal status
 
+The throttle lock and timestamp live in the root upgrade unit's persistent
+`StateDirectory=rigforge-control-upgrade` with mode `0700`, separate from the receiver's
+state (#627). The verb requires a root-owned, non-symlink directory and rejects existing
+throttle objects unless they are root-owned regular files with mode `0600` and one link.
+Locking failure, timestamp I/O errors, or invalid timestamps refuse the upgrade. Receiver-owned legacy files are never
+opened or migrated; the new directory starts a fresh cooldown on the first attempt.
+
 `control-upgrade` is fail-closed and every step is rollback-guarded, reusing the upgrade flow's rebuild decision, including the pinned version/commit, compiled-minimum marker and binary checksum checks. A **pre-dial throttle stamp** on the rig bounds how often it will reach out to GitHub, so a looping or hostile consumer cannot turn the fleet into a beacon or a request amplifier. Outcomes use the same `/status` surface as control-apply, extended with a **`failed`** terminal (with reason) alongside `applied` / `rolled_back`, so a pre-apply refusal (target outside the allowed bound, throttled, fetch/build error) surfaces a clear terminal result to the consumer's poll rather than hanging in a non-terminal state.
 
 *Amended (#320, consumer feedback from pithead #597):* the status vocabulary grew three additive members so a poller never has to string-match `reason`. A non-terminal **`started`** is written the moment the oneshot claims the intent (D8 move done), so "mid-run" and "oneshot died mid-run" are distinguishable from "queued" (since #535 a run that dies after the claim records a terminal `failed` saying so, and only a `SIGKILL` or power loss leaves `started` behind); **`noop`** replaces `failed` for the already-on-target refusal (idempotent, not an error); **`throttled`** replaces `failed` for the D6 throttle refusal (retry-later, not an error). `applied`'s `reason` echoes the landed version. All other refusals stay `failed`.

docs/operations.md

@@ -715,6 +715,12 @@ ADR 0002's ~10-minute estimate — twice over, since a failed forward attempt re
 in the same run — so systemd's implicit 90s default would kill it mid-build. Nothing is left
 unbounded by it: the D6 throttle is what bounds how often this path fetches and builds at all.
 
+The root upgrade unit keeps its throttle lock and timestamp in
+`/var/lib/rigforge-control-upgrade`, owned by root with mode `0700`, separate from the
+receiver's writable state. Unsafe ownership, permissions, links, file types, or timestamp contents
+fail the upgrade before any fetch or build. Existing throttle files in the receiver's state are
+left untouched and never migrated, so the first upgrade after this change starts a fresh cooldown.
+
 `"control": "disabled"` + `apply` removes the units cleanly; `uninstall` removes them too. Linux-only.
 The design and the decisions behind it are recorded in
 [ADR 0001](../docs/adr/0001-writable-worker-config-control-path.md).

rigforge.sh

@@ -4691,35 +4691,32 @@ control_apply() {
     return 0
 }
 
-# Pre-dial throttle for the remote-upgrade path (#308, ADR 0002 D6): a persisted stamp under a flock
-# bounds how often a rig will fetch from GitHub / rebuild, so a looping or hostile trigger can't turn
-# the fleet into a beacon or thrash the miner with rebuilds. Returns 0 if allowed (and stamps the
-# attempt), 1 if still inside the window, 2 if the throttle state itself is unusable (fail-closed,
-# #321). Default 6h; override with CONTROL_UPGRADE_MIN_INTERVAL (s).
-_control_upgrade_throttle_ok() { # <state-dir>
-    local dir="$1" stamp="$1/upgrade-last" now min="${CONTROL_UPGRADE_MIN_INTERVAL:-21600}" last=0
-    now=$(date +%s)
-    mkdir -p "$dir" 2>/dev/null || true
-    # Can't open the lock (unwritable/full state dir)? Fail CLOSED (#321): this guard exists to bound
-    # how often a compromised trigger can make the rig dial out, and a degraded state dir is exactly
-    # what an attacker might arrange — fail-open would invert the posture right there. An operator
-    # with a broken state dir has bigger problems than a delayed upgrade. rc 2 (not 1) so the caller
-    # reports the real cause instead of "throttled — retry later".
-    exec 9>"$dir/.upgrade-throttle.lock" 2>/dev/null || {
-        warn "control-upgrade: couldn't take the throttle lock under $dir — refusing the upgrade (fail-closed; check the state dir)."
-        return 2
-    }
-    flock 9 2>/dev/null || true
-    [ -f "$stamp" ] && last=$(cat "$stamp" 2>/dev/null || echo 0)
-    case "$last" in '' | *[!0-9]*) last=0 ;; esac
+# A root-only parent prevents receiver swaps between validation and opening (#627).
+_control_upgrade_throttle_ok() (
+    local dir="$1" lock="$1/.upgrade-throttle.lock" stamp="$1/upgrade-last" now tmp min="${CONTROL_UPGRADE_MIN_INTERVAL:-21600}" last=0
+    [ ! -L "$dir" ] && [ -d "$dir" ] && [ "$(stat -c '%u:%g:%a' "$dir" 2>/dev/null)" = "0:0:700" ] || return 2
+    if [ -e "$lock" ] || [ -L "$lock" ]; then
+        [ ! -L "$lock" ] && [ -f "$lock" ] && [ "$(stat -c '%u:%g:%a:%h' "$lock" 2>/dev/null)" = "0:0:600:1" ] || return 2
+    fi
+    umask 077
+    exec 9>>"$lock" 2>/dev/null || return 2
+    flock -x 9 2>/dev/null || return 2
+    if [ -e "$stamp" ] || [ -L "$stamp" ]; then
+        [ ! -L "$stamp" ] && [ -f "$stamp" ] && [ "$(stat -c '%u:%g:%a:%h' "$stamp" 2>/dev/null)" = "0:0:600:1" ] || return 2
+        last=$(cat "$stamp" 2>/dev/null) || return 2
+    fi
+    case "$last" in '' | *[!0-9]*) return 2 ;; esac
+    now=$(date +%s) || return 2
     if [ $((now - last)) -lt "$min" ] 2>/dev/null; then
-        exec 9>&-
         return 1
     fi
-    printf '%s\n' "$now" >"$stamp" 2>/dev/null || true
-    exec 9>&-
+    tmp=$(mktemp "$dir/.upgrade-last.XXXXXX") || return 2
+    if ! printf '%s\n' "$now" >"$tmp" || ! mv -f "$tmp" "$stamp"; then
+        rm -f "$tmp"
+        return 2
+    fi
     return 0
-}
+)
 
 # The fetch + reachability + checkout + rebuild half of control-upgrade, split out so the orchestration
 # is unit-testable with this stubbed (like control-apply's _control_do_apply). Returns 0 on a successful
@@ -4812,10 +4809,10 @@ control_upgrade() {
     fi
     # D6 throttle: bound how often a rig fetches/rebuilds. rc 2 = throttle state unavailable — a
     # fail-closed refusal (#321) that must read as a broken rig, not as "throttled — retry later".
-    local thr_rc=0
-    _control_upgrade_throttle_ok "$state" || thr_rc=$?
+    local thr_rc=0 throttle="${RIGFORGE_CONTROL_UPGRADE_STATE:-/var/lib/rigforge-control-upgrade}"
+    _control_upgrade_throttle_ok "$throttle" || thr_rc=$?
     if [ "$thr_rc" -eq 2 ]; then
-        _control_status "$status" failed "$cid" version "throttle state unavailable under $state — refused (fail-closed)" ""
+        _control_status "$status" failed "$cid" version "throttle state unavailable under $throttle — refused (fail-closed)" ""
         warn "control-upgrade: throttle state unavailable — refused."
         return 0
     elif [ "$thr_rc" -ne 0 ]; then

systemd/rigforge-control-upgrade.service.template

@@ -19,6 +19,8 @@ Environment=RIGFORGE_APPLIANCE=$RIGFORGE_APPLIANCE
 RuntimeDirectory=rigforge-control
 RuntimeDirectoryMode=0700
 RuntimeDirectoryPreserve=yes
+StateDirectory=rigforge-control-upgrade
+StateDirectoryMode=0700
 ExecStart=$SCRIPT_DIR/rigforge.sh control-upgrade
 Nice=19
 IOSchedulingClass=idle

tests/e2e-real.sh

@@ -50,7 +50,8 @@ CTL_CONTROL_ACTIVE=0 CTL_CONTROL_ENABLED=0
 # E2E_UPGRADE_TARGET override, which stays deliberately PERMANENT). Same script-global reasoning as
 # above — _upgrade_cleanup must see it from a late trap fire too.
 UPG_ORIG_REF="" UPG_RESTORE_BUILD=0
-UPG_STAMP="/var/lib/rigforge-control/upgrade-last"
+UPG_STAMP="/var/lib/rigforge-control-upgrade/upgrade-last"
+UPG_LEGACY_STAMP="/var/lib/rigforge-control/upgrade-last" UPG_LEGACY_LOCK="/var/lib/rigforge-control/.upgrade-throttle.lock"
 # #349: script-global watchdog state remains visible to a late EXIT trap.
 WD_SAVED_CFG="" WD_CLEANUP_DONE=0 WD_WORKER_ROOT="" WD_WAS_ACTIVE=0
 E2E_EXIT_RC=0
@@ -903,11 +904,14 @@ _upg_post_and_poll() { # <token> <port> <vX.Y.Z> <timeout-s> -> terminal status
 
 # The upgrade-phase leftovers on top of _control_cleanup (which restores the snapshot, forces both
 # control flags off, re-applies, and checks the miner comes back). Idempotent like its parts.
+_upgrade_clear_throttles() {
+    rm -f "$UPG_STAMP" "$UPG_LEGACY_STAMP" "$UPG_LEGACY_LOCK" 2>/dev/null || return 1
+    [ ! -e "$UPG_STAMP" ] && [ ! -L "$UPG_STAMP" ] && [ ! -e "$UPG_LEGACY_STAMP" ] && [ ! -L "$UPG_LEGACY_STAMP" ] && [ ! -e "$UPG_LEGACY_LOCK" ] && [ ! -L "$UPG_LEGACY_LOCK" ]
+}
 _upgrade_cleanup() {
     local cleanup_ok=1
     _hgit tag -d v99.99.99 >/dev/null 2>&1 || ! _hgit show-ref --verify --quiet refs/tags/v99.99.99 || cleanup_ok=0
-    rm -f "$UPG_STAMP" 2>/dev/null
-    [ ! -e "$UPG_STAMP" ] || cleanup_ok=0
+    _upgrade_clear_throttles || cleanup_ok=0
     # #350: the auto-derived forward leg rewinds the checkout to a real previous tag to prove the
     # forward step for real, then must land back on the exact ref this phase started from — on ANY
     # exit, success or a hard abort mid-leg. A release gate must never leave the rig pinned to an
@@ -1049,8 +1053,12 @@ upgrade() {
             UPG_RESTORE_BUILD=1 # Returning to this ref alone does not restore its build artifacts.
             if _hgit checkout --quiet --force "$prev_tag" 2>/dev/null && "$RIGFORGE" upgrade >/tmp/e2e-upgrade-rewind.log 2>&1; then
                 ok "rewound the checkout to $prev_tag (a real prior release, to prove the forward leg from)"
-                rm -f "$stamp" 2>/dev/null || true # the rollback leg stamped; this is a fresh attempt
-                st=$(_upg_post_and_poll "$tok" "$control_port" "$cur_tag" 600)
+                if _upgrade_clear_throttles; then # the old upgrader uses receiver throttle state
+                    st=$(_upg_post_and_poll "$tok" "$control_port" "$cur_tag" 600)
+                else
+                    bad "could not clear throttle state before running the old upgrader"
+                    continue_or_cleanup _upgrade_cleanup upgrade
+                fi
                 [ "$st" = applied ] &&
                     ok "forward upgrade $prev_tag -> $cur_tag reached 'applied' (real fetch + build + health gate)" ||
                     bad "forward leg ended '$st' (expected applied, $prev_tag -> $cur_tag)"

tests/run.sh

@@ -119,7 +119,7 @@ echo "${STUB_NPROC:-4}"
 EOF
     cat >"$bin/stat" <<'EOF'
 #!/usr/bin/env bash
-case "$*" in *"%u:%g:%a"*) echo 0:0:700 ;; *) /usr/bin/stat "$@" ;; esac
+case "$*" in *"%u:%g:%a:%h"*) echo 0:0:600:1 ;; *"%u:%g:%a"*) echo 0:0:700 ;; *) /usr/bin/stat "$@" ;; esac
 EOF
     cat >"$bin/hostname" <<'EOF'
 #!/usr/bin/env bash
@@ -203,11 +203,11 @@ EOF
 
 STUBS="$SANDBOX/stubs"
 make_stubs "$STUBS"
-RIGFORGE_CONTROL_PROCESSING="$SANDBOX/control-processing"
-mkdir -m 700 "$RIGFORGE_CONTROL_PROCESSING"
-export RIGFORGE_CONTROL_PROCESSING
-assert_rc "privileged consumers share a lock and refuse a planted processing symlink (#479)" "$(
-    bash "$ROOT/tests/test-control-hardening.sh" "$SCRIPT"
+RIGFORGE_CONTROL_PROCESSING="$SANDBOX/control-processing" RIGFORGE_CONTROL_UPGRADE_STATE="$SANDBOX/upgrade-state"
+mkdir -m 700 "$RIGFORGE_CONTROL_PROCESSING" "$RIGFORGE_CONTROL_UPGRADE_STATE"
+export RIGFORGE_CONTROL_PROCESSING RIGFORGE_CONTROL_UPGRADE_STATE
+assert_rc "privileged consumers reject unsafe processing and throttle state (#479/#627)" "$(
+    bash "$ROOT/tests/test-control-hardening.sh" "$SCRIPT" && bash "$ROOT/tests/test-upgrade-throttle-security.sh" "$SCRIPT" >/dev/null
     echo $?
 )" "0"
 assert_rc "appliance pool policy rejects edits and preserves tuning (#583)" "$(
@@ -8851,7 +8851,7 @@ assert_eq "the sidecar records the change_id that caused it (#435, #254)" "$(J "
 # The backup the rollback branch would need: a `null` here fails this as surely as a wrong path does.
 assert_eq "the recorded rollback backup really exists on disk (#435)" "$([ -f "$(caast backup)" ] && echo y || echo n)" "y"
 
-source "$ROOT/tests/test-control-upgrade.sh"
+PATH="$STUBS:$PATH" source "$ROOT/tests/test-control-upgrade.sh"
 
 # Same failure, a second invocation shape, found on the v1.16.0 gate (#401). git exempts a repo from
 # its dubious-ownership check when the repo is owned by $SUDO_UID, so a plain

tests/test-control-upgrade.sh

@@ -5,6 +5,7 @@ cu_run() { # <staged-json|""> <installed-version> <do:ok|fail|down|buildfail|har
     local d
     d=$(mktemp -d "$SANDBOX/cu.XXXXXX")
     mkdir -p "$d/state/spool"
+    mkdir -m 700 "$d/upgrade-state"
     printf '%s' "$2" >"$d/VERSION"
     printf '{"pools":[{"url":"h:3333"}]}\n' >"$d/config.json"
     [ -n "$1" ] && printf '%s\n' "$1" >"$d/state/spool/upgrade-abc123def4567890.json"
@@ -14,6 +15,7 @@ cu_run() { # <staged-json|""> <installed-version> <do:ok|fail|down|buildfail|har
         SCRIPT_DIR="$d"
         CONFIG_JSON="$d/config.json"
         RIGFORGE_CONTROL_STATE="$d/state"
+        RIGFORGE_CONTROL_UPGRADE_STATE="$d/upgrade-state"
         CONTROL_UPGRADE_MIN_INTERVAL=0
         DO="$3"
         WML=0
@@ -119,7 +121,7 @@ assert_eq "throttle: zero interval always allowed" "$?" "0"
 # caller can report the real cause instead of "throttled — retry later". The dangling symlink into
 # a missing dir makes the open fail for ANY uid (root included), unlike a chmod-based setup.
 roThr="$SANDBOX/thr-ro"
-mkdir -p "$roThr"
+mkdir -m 700 "$roThr"
 ln -s "$roThr/no-such-dir/lock" "$roThr/.upgrade-throttle.lock"
 (
     source "$SCRIPT"
@@ -169,16 +171,19 @@ assert_eq "D10 guard does not consult origin/HEAD (#318)" \
 
 cuThr=$(mktemp -d "$SANDBOX/cuthr.XXXXXX")
 mkdir -p "$cuThr/state/spool"
+mkdir -m 700 "$cuThr/upgrade-state"
 printf '1.0.0' >"$cuThr/VERSION"
 printf '{"pools":[{"url":"h:3333"}]}\n' >"$cuThr/config.json"
 printf '{"version":"v9.9.9"}\n' >"$cuThr/state/spool/upgrade-abc123def4567890.json"
-date +%s >"$cuThr/state/upgrade-last"
+date +%s >"$cuThr/upgrade-state/upgrade-last"
+chmod 600 "$cuThr/upgrade-state/upgrade-last"
 sThr="$(
     source "$SCRIPT"
     OS_TYPE=Linux
     SCRIPT_DIR="$cuThr"
     CONFIG_JSON="$cuThr/config.json"
     RIGFORGE_CONTROL_STATE="$cuThr/state"
+    RIGFORGE_CONTROL_UPGRADE_STATE="$cuThr/upgrade-state"
     CONTROL_UPGRADE_MIN_INTERVAL=3600
     _control_upgrade_do() { return 0; }
     _wait_miner_live() { return 0; }
@@ -192,16 +197,18 @@ assert_contains "throttled reason says why" "$sThr" "too soon"
 
 cuTs=$(mktemp -d "$SANDBOX/cuts.XXXXXX")
 mkdir -p "$cuTs/state/spool"
+mkdir -m 700 "$cuTs/upgrade-state"
 printf '1.0.0' >"$cuTs/VERSION"
 printf '{"pools":[{"url":"h:3333"}]}\n' >"$cuTs/config.json"
 printf '{"version":"v9.9.9"}\n' >"$cuTs/state/spool/upgrade-abc123def4567890.json"
-ln -s "$cuTs/state/no-such-dir/lock" "$cuTs/state/.upgrade-throttle.lock"
+ln -s "$cuTs/no-such-dir/lock" "$cuTs/upgrade-state/.upgrade-throttle.lock"
 sTs="$(
     source "$SCRIPT"
     OS_TYPE=Linux
     SCRIPT_DIR="$cuTs"
     CONFIG_JSON="$cuTs/config.json"
     RIGFORGE_CONTROL_STATE="$cuTs/state"
+    RIGFORGE_CONTROL_UPGRADE_STATE="$cuTs/upgrade-state"
     CONTROL_UPGRADE_MIN_INTERVAL=3600
     _control_upgrade_do() { return 0; }
     _wait_miner_live() { return 0; }
@@ -215,6 +222,7 @@ assert_contains "fail-closed reason names the throttle state, not 'throttled'" "
 
 cuSt=$(mktemp -d "$SANDBOX/cust.XXXXXX")
 mkdir -p "$cuSt/state/spool"
+mkdir -m 700 "$cuSt/upgrade-state"
 printf '1.0.0' >"$cuSt/VERSION"
 printf '{"pools":[{"url":"h:3333"}]}\n' >"$cuSt/config.json"
 printf '{"version":"v9.9.9"}\n' >"$cuSt/state/spool/upgrade-abc123def4567890.json"
@@ -224,6 +232,7 @@ printf '{"version":"v9.9.9"}\n' >"$cuSt/state/spool/upgrade-abc123def4567890.jso
     SCRIPT_DIR="$cuSt"
     CONFIG_JSON="$cuSt/config.json"
     RIGFORGE_CONTROL_STATE="$cuSt/state"
+    RIGFORGE_CONTROL_UPGRADE_STATE="$cuSt/upgrade-state"
     CONTROL_UPGRADE_MIN_INTERVAL=0
     _control_upgrade_do() {
         cp "$cuSt/state/status.json" "$cuSt/mid-status.json" 2>/dev/null

tests/test-e2e-gate-fail-closed.sh

@@ -220,10 +220,12 @@ control_retry_case() {
 assert_eq "failed control cleanup retries instead of false-passing" "$(control_retry_case)" "1:1:0:0"
 
 REAL_UPG="$(sed -n '/^_upgrade_cleanup()/,/^}/p' "$ROOT/tests/e2e-real.sh")"
+REAL_UPG_CLEAR="$(sed -n '/^_upgrade_clear_throttles()/,/^}/p' "$ROOT/tests/e2e-real.sh")"
 upgrade_cleanup_case() { # checkout succeeds, rebuild fails twice
     (
+        eval "$REAL_UPG_CLEAR"
         eval "$REAL_UPG"
-        UPG_ORIG_REF=original UPG_RESTORE_BUILD=0 UPG_STAMP="$T491/retry-stamp" HERE="$T491" RIGFORGE=false
+        UPG_ORIG_REF=original UPG_RESTORE_BUILD=0 UPG_STAMP="$T491/retry-stamp" UPG_LEGACY_STAMP="$T491/retry-legacy-stamp" UPG_LEGACY_LOCK="$T491/retry-legacy-lock" HERE="$T491" RIGFORGE=false
         _hgit() { case "$1" in rev-parse) printf '%s\n' "${head:-old}" ;; checkout) head=original ;; esac }
         _control_cleanup() { :; }
         _upgrade_cleanup >/dev/null 2>&1
@@ -236,8 +238,9 @@ assert_eq "upgrade cleanup retries rebuild and retains source ref until it succe
 UPG_ARM_BLOCK="$(sed -n '/^            UPG_ORIG_REF=.*rev-parse HEAD/,/^            if _hgit checkout/p' "$ROOT/tests/e2e-real.sh" | sed '$d')"
 upgrade_same_head_case491() {
     (
+        eval "$REAL_UPG_CLEAR"
         eval "$REAL_UPG"
-        UPG_ORIG_REF="" UPG_RESTORE_BUILD=0 UPG_STAMP="$T491/same-head-stamp" calls=0
+        UPG_ORIG_REF="" UPG_RESTORE_BUILD=0 UPG_STAMP="$T491/same-head-stamp" UPG_LEGACY_STAMP="$T491/same-head-legacy-stamp" UPG_LEGACY_LOCK="$T491/same-head-legacy-lock" calls=0
         RIGFORGE=rebuild491
         _hgit() { [ "$1" != rev-parse ] || printf original; }
         _control_cleanup() { :; }
@@ -253,7 +256,7 @@ upgrade_same_head_case491() {
     )
 }
 assert_eq "upgrade cleanup rebuilds stale artifacts even when HEAD is already restored" "$(upgrade_same_head_case491)" "1:original:1:0:2:cleared:0"
-assert_eq "upgrade checkpoints guard setup, VERSION, noop, probe, and rollback" "$(grep -c 'continue_or_cleanup _upgrade_cleanup upgrade' "$ROOT/tests/e2e-real.sh")" "6"
+assert_eq "upgrade checkpoints guard setup, VERSION, noop, probe, rollback, and old-release throttle cleanup" "$(grep -c 'continue_or_cleanup _upgrade_cleanup upgrade' "$ROOT/tests/e2e-real.sh")" "7"
 
 UPG_VERSION_BLOCK="$(sed -n '/^    installed=$(tr /,/^    phase "upgrade — rollback/p' "$ROOT/tests/e2e-real.sh")"
 out="$( (
@@ -279,8 +282,9 @@ assert_eq "missing VERSION fires no later upgrade mutation" "$out" "posts=0"
 
 upgrade_artifact_case() {
     (
+        eval "$REAL_UPG_CLEAR"
         eval "$REAL_UPG"
-        UPG_ORIG_REF="" UPG_RESTORE_BUILD=0 UPG_STAMP="$T491/retained-stamp"
+        UPG_ORIG_REF="" UPG_RESTORE_BUILD=0 UPG_STAMP="$T491/retained-stamp" UPG_LEGACY_STAMP="$T491/retained-legacy-stamp" UPG_LEGACY_LOCK="$T491/retained-legacy-lock"
         : >"$UPG_STAMP"
         _hgit() { case "$1 $2" in 'tag -d') return 1 ;; 'show-ref --verify') return 0 ;; esac }
         rm() { return 1; }