ci: run release and notify-hub hosted and compare glibc with the v1.10.0 archive

.github/actionlint.yaml

@@ -1,5 +1,4 @@
-# The unchanged release workflow retains its protected runner labels.
+# Release keeps one protected runner label, on its x86_64 prebuilt cell.
 self-hosted-runner:
   labels:
-    - gkp-od-4cpu-8gb-amd64
     - gkp-od-16cpu-32gb-amd64-128gb

.github/workflows/release.yml

@@ -77,7 +77,7 @@ env:
 
 jobs:
   release:
-    runs-on: gkp-od-4cpu-8gb-amd64
+    runs-on: ubuntu-24.04
     environment: release
 
     outputs:
@@ -485,7 +485,32 @@ jobs:
           python3 scripts/release_version.py check-install \
             --prefix "${STAGE_DIR}" --expect "${NEXT}"
 
-      # Both dry-run and publication inspect every installed .a/.so/.so.*.
+      # The glibc comparison baseline is this triple's archive from the v1.10.0
+      # release: the real released bytes, pinned by sha256 and unpacked here.
+      # The checker below only reads the unpacked libraries.
+      - name: Fetch the v1.10.0 baseline archive for the glibc comparison
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          TRIPLE: ${{ matrix.triple }}
+        run: |
+          set -euo pipefail
+          case "${TRIPLE}" in
+            linux-x86_64) digest=cb412fbae9e8ba515de41dc31026c77d6c5e309693247bee26bb93c67d3c3543 ;;
+            linux-aarch64) digest=982b16da2d5f5ee7c274615895484fce36bd740052a14d59b66576dca496d7bb ;;
+            *) echo "::error::no v1.10.0 baseline archive for ${TRIPLE}" >&2; exit 1 ;;
+          esac
+          archive="pineforge-v1.10.0-${TRIPLE}.tar.gz"
+          root="${RUNNER_TEMP}/glibc-baseline-${TRIPLE}"
+          mkdir "${root}" "${root}/download" "${root}/tree"
+          gh release download v1.10.0 --repo "${GITHUB_REPOSITORY}" \
+            --pattern "${archive}" --dir "${root}/download"
+          echo "${digest}  ${root}/download/${archive}" | sha256sum -c -
+          tar -xzf "${root}/download/${archive}" -C "${root}/tree" \
+            --no-same-owner --no-same-permissions
+          echo "BASELINE_DIR=${root}/tree" >> "${GITHUB_ENV}"
+
+      # Both dry-run and publication inspect every installed .a/.so/.so.* and
+      # compare their highest glibc tag with the baseline fetched above.
       # Keep raw checker output and its actual exit status, including setup errors.
       - name: Check every staged library's glibc floor
         env:
@@ -494,7 +519,7 @@ jobs:
           set -euo pipefail
           log="${RUNNER_TEMP}/prebuilt-glibc-${TRIPLE}"
           mkdir -p "${log}"
-          if python3 scripts/check_prebuilt_glibc.py "${STAGE_DIR}" > "${log}/stdout.jsonl" 2> "${log}/stderr.txt"; then
+          if python3 scripts/check_prebuilt_glibc.py "${STAGE_DIR}" --baseline-root "${BASELINE_DIR}" > "${log}/stdout.jsonl" 2> "${log}/stderr.txt"; then
             checker_rc=0
           else checker_rc=$?; fi
           printf '%s\n' "${checker_rc}" > "${log}/exitcode.txt"
@@ -565,7 +590,7 @@ jobs:
   notify-hub:
     needs: [release, prebuilt]
     if: ${{ !inputs.dry_run }}
-    runs-on: gkp-od-4cpu-8gb-amd64
+    runs-on: ubuntu-24.04
     environment: release
     steps:
       - name: Mint org App token (hub dispatch)

docs/ci.md

@@ -113,7 +113,7 @@ Python refuse a system-wide install (PEP 668), and `.venv/` is ignored. The
 and fails, never skips, without it.
 
 Actionlint only accepts declared custom self-hosted runner labels; preflight
-checks the declarations in `.github/actionlint.yaml`, which now declare only the two protected `gkp-od-*` labels of `release.yml`, including its x86_64 release matrix cell. Hosted workflows declare nothing. Before a workflow moves onto a `gke-*` label, add its ref to `.github/runner-group-allowlist.txt` (group 14, currently empty because no workflow uses one); protected `gkp-*` refs belong in `.github/runner-group-15-allowlist.txt` (group 15, currently only `release.yml`). Both mirrors are binding; a maintainer must confirm the corresponding live org workflow/ref entries before enabling the routes.
+checks the declarations in `.github/actionlint.yaml`, which now declare only the protected `gkp-od-*` label of `release.yml`'s x86_64 release matrix cell. Hosted workflows declare nothing. Before a workflow moves onto a `gke-*` label, add its ref to `.github/runner-group-allowlist.txt` (group 14, currently empty because no workflow uses one); protected `gkp-*` refs belong in `.github/runner-group-15-allowlist.txt` (group 15, currently only `release.yml`). Both mirrors are binding; a maintainer must confirm the corresponding live org workflow/ref entries before enabling the routes.
 
 ORG-ACTIONPINS is enforced by preflight's `ci-workflow-contract`: every
 third-party `uses:` in `.github/workflows/*.yml` (also `.yaml`) and nested
@@ -1644,7 +1644,7 @@ The native matrix is exactly `1..4` with `--ctest-shard I/4`; live sanitizers ar
 | `corpus-parity.yml` | Schedule, dispatch, post CI callers | `authz`, full parity and subset: `ubuntu-24.04` |
 | `native-live.yml` | Dispatch and post CI callers on `main` | `authz`, both aggregates and the native shards: `ubuntu-24.04`; `matrix.runner`: live-sanitizers and live-tsan both `ubuntu-24.04` |
 | `docs-post.yml` | Push to `main`, dispatch | `authz` and credentialed `build`: `ubuntu-24.04` |
-| `release.yml` | Dispatch from `main` | `release` and `notify-hub`: `gkp-od-4cpu-8gb-amd64`; x86_64 `prebuilt`: `gkp-od-16cpu-32gb-amd64-128gb`; arm64 `prebuilt`: hosted `ubuntu-22.04-arm` |
+| `release.yml` | Dispatch from `main` | `release` and `notify-hub`: hosted `ubuntu-24.04`; x86_64 `prebuilt`: `gkp-od-16cpu-32gb-amd64-128gb`; arm64 `prebuilt`: hosted `ubuntu-22.04-arm` |
 | `promote-baseline.yml` | Push to `main`, dispatch from `main` | `promote-guard` and `promote`: `ubuntu-24.04` |
 | `release-ruleset-probe.yml` | Dispatch from `main` | `authz` and `probe`: `ubuntu-24.04` |
 
@@ -1668,15 +1668,15 @@ Trusted post workflows and PR proof legs build cold; no trusted job restores an
 `HOSTED_JOB_RUNNERS` pins the hosted x86 and arm64 jobs and the native profile runner matrix; `GKP_JOB_RUNNERS` separately pins the protected release jobs and sizes. Every pinned job runs directly on its hosted image. PR checks are
 `build`, `build-debug`, `corpus-parity-subset`, `legs` and `result` in `gke-pr.yml`. With the arm leg, the full set is `build`, `build-debug`, `build-arm`, `corpus-parity-subset`, `legs` and `result` in `gke-pr.yml`.
 
-Release uses protected GKE runner group 15:
-`release` and `notify-hub` use `gkp-od-4cpu-8gb-amd64`;
+Release uses protected GKE runner group 15 for its x64 artifact only:
+`release` and `notify-hub` run on hosted `ubuntu-24.04` and keep their `release` environment;
 Linux x64 `prebuilt` uses `gkp-od-16cpu-32gb-amd64-128gb`; baseline `promote` uses hosted `ubuntu-24.04`.
 The release ruleset probe runs `authz` and `probe` on hosted `ubuntu-24.04`.
 Release's ARM artifact cell retains hosted `ubuntu-22.04-arm`;
 1.8.0 has no macOS test or artifact cells, and
 promotion runs on a push to `main` plus manual dispatch from `main`.
 
-Release and notify-hub use the on-demand 4 CPU/8 GB set to prevent preemption after tag push.
+Release and notify-hub run on hosted `ubuntu-24.04`, which no scale-set preemption can interrupt after tag push.
 Release computes versions, stamps JSON catalogs and publishes with git/gh; notify-hub
 mints an App token and dispatches an API request. The protected 2 CPU/4 GB set
 runs promotion's API/tree checks, Node campaign tooling and Cloud SQL proxy; the

docs/reference/ci-runner-routing.md

@@ -5,7 +5,7 @@ parity verdict; an outside contributor cannot post them. GitHub Actions is advis
 The PR build (`ci.yml`) runs only the `fast`-labelled CTest rows (`ctest -L '^fast$'`); the post-merge legs (`ci-post.yml`) run
 every row, and a documentation-only change runs preflight alone among verification jobs.
 The seven PR/post-merge verification and documentation workflows start with the canon `authz` job and every other job in them needs it; release uses its status gate and baseline promotion its `promote-guard`. Pull-request
-proof workflows and post jobs use exact hosted pins: `ubuntu-24.04` for x86 and `ubuntu-24.04-arm` for the arm64 leg. Promotion, the ruleset probe and docs deployment also run on hosted `ubuntu-24.04`, and keep their `release`/`release-bot` environments. Only `release.yml` still names protected scale sets: `release` and `notify-hub` use `gkp-od-4cpu-8gb-amd64` in protected runner group 15. Hosted PR light jobs and `docs.yml`'s `build` retain `ubuntu-24.04`; post guards and aggregates use the same pin.
+proof workflows and post jobs use exact hosted pins: `ubuntu-24.04` for x86 and `ubuntu-24.04-arm` for the arm64 leg. Promotion, the ruleset probe and docs deployment also run on hosted `ubuntu-24.04`, and keep their `release`/`release-bot` environments. Only `release.yml` still names a protected scale set, in protected runner group 15: its x86_64 `prebuilt` cell. `release` and `notify-hub` run on hosted `ubuntu-24.04` and keep their `release` environment. Hosted PR light jobs and `docs.yml`'s `build` retain `ubuntu-24.04`; post guards and aggregates use the same pin.
 Hosted fork PRs receive no secrets, so they need no VM-only fork-refusal hook. Retired main/post VM labels remain refused. The retired `gke-*` scale sets and paid larger-runner labels are refused too. Group 14's mirror `.github/runner-group-allowlist.txt` is empty because no workflow uses a group-14 label; group 15's required ref (`release.yml`) is in `.github/runner-group-15-allowlist.txt`, with live membership confirmed by a maintainer.
 Release's x86_64 prebuilt cell uses `gkp-od-16cpu-32gb-amd64-128gb`, installs Eigen and asserts exactly 3.4.0 before CMake; the arm64 cell retains hosted `ubuntu-22.04-arm`. App-token jobs keep their exact `release`/`release-bot` environments. The `build-gate` job in `ci-post.yml` and `native-live.yml`'s `native-live-gate` always run and refuse a non-success authz result first.
 See `docs/ci.md` for the class table and docs-only exception. The commands below run the full sets.

scripts/check_prebuilt_glibc.py

@@ -6,6 +6,18 @@
 GNU ar/readelf must already be installed in the OS default executable search
 path. Never extract archives, load libraries, execute payloads, or write files.
 
+--baseline-root DIR adds the released-baseline comparison. DIR is an already
+unpacked released archive (the workflow checks its digest and unpacks it; this
+tool still extracts nothing). The same strict inspection reads every library
+under DIR, and the highest numeric GLIBC_ tag over all staged libraries may not
+exceed the highest over all baseline libraries: equal or lower passes, higher
+fails. A tree with no numeric tag ranks below any tag. The baseline is read, not
+judged: the fixed ceiling and the __isoc23_* rule apply to the staging tree
+only, since a released archive may predate them. A missing, empty or partly
+uninspectable baseline, or one overlapping the staging root, fails closed. No
+expected value is configured anywhere: the verdict is a function of the two
+inspected trees.
+
 All symlinks (including directory links), protected/key/config/.env* paths,
 unreadable entries, thin archives, non-ELF members, and wrong ELF types fail.
 Archives must contain REL objects with inspectable symbol tables; shared objects
@@ -65,8 +77,13 @@ class InspectionError(Exception):
     pass
 
 
+# Fields added to every record while the released baseline is being read.
+EVENT_CONTEXT: dict[str, str] = {}
+
+
 def emit(event: str, **fields) -> None:
-    print(json.dumps({"event": event, **fields}, ensure_ascii=True), flush=True)
+    print(json.dumps({"event": event, **EVENT_CONTEXT, **fields}, ensure_ascii=True),
+          flush=True)
 
 
 def issue(issues: list[dict], path: Path, cause: str, **fields) -> None:
@@ -282,6 +299,42 @@ def maximum_version(names: list[str]) -> str | None:
     return name[len("GLIBC_"):]
 
 
+def tree_maximum(rows: list[dict]) -> tuple[int, ...] | None:
+    """Highest numeric GLIBC_ tag over every inspected library of one tree."""
+    highest = maximum_version(sorted({name for row in rows for name in row["versions"]}))
+    return None if highest is None else tuple(int(part) for part in highest.split("."))
+
+
+def version_text(version: tuple[int, ...] | None) -> str | None:
+    return None if version is None else ".".join(str(part) for part in version)
+
+
+def glibc_label(version: tuple[int, ...] | None) -> str:
+    return "none" if version is None else "GLIBC_" + version_text(version)
+
+
+def exceeds_baseline(candidate: tuple[int, ...] | None,
+                     baseline: tuple[int, ...] | None) -> bool:
+    """True when the staged tree's highest numeric tag is above the baseline's.
+
+    Components compare with zero padding, so 2.36 equals 2.36.0. No numeric tag
+    at all ranks below every tag: a staged tag above a tagless baseline exceeds
+    it, and a tagless staged tree never does.
+    """
+    if candidate is None:
+        return False
+    if baseline is None:
+        return True
+    width = max(len(candidate), len(baseline))
+    return (candidate + (0,) * (width - len(candidate))
+            > baseline + (0,) * (width - len(baseline)))
+
+
+def roots_overlap(left: Path, right: Path) -> bool:
+    first, second = (Path(os.path.abspath(path)) for path in (left, right))
+    return first == second or first in second.parents or second in first.parents
+
+
 def version_sections(lines: list[str], dynamic_count: int | None) -> set[str]:
     headers = [(index, VERSION_HEADER.fullmatch(line))
                for index, line in enumerate(lines) if line.startswith("Version ")]
@@ -493,7 +546,7 @@ def policy_issues(path: Path, member: dict, issues: list[dict]) -> None:
 
 
 def inspect_target(root_fd: int, target: dict, directories: dict,
-                   tools: dict[str, str], issues: list[dict]) -> dict:
+                   tools: dict[str, str], issues: list[dict], policy: bool = True) -> dict:
     path = target["path"]
     row = {
         "path": str(path), "kind": target["kind"], "identity": target["identity"],
@@ -580,7 +633,8 @@ def inspect_target(root_fd: int, target: dict, directories: dict,
             try:
                 member.update(parse_elf(lines, "REL" if expected_members is not None else "DYN"))
                 member["inspected"] = True
-                policy_issues(path, member, issues)
+                if policy:
+                    policy_issues(path, member, issues)
             except (InspectionError, ValueError) as error:
                 issue(issues, path, str(error), member=name, ordinal=ordinal)
                 complete = False
@@ -613,16 +667,12 @@ def inspect_target(root_fd: int, target: dict, directories: dict,
     return row
 
 
-def main(argv: list[str] | None = None) -> int:
-    parser = argparse.ArgumentParser(description=__doc__)
-    parser.add_argument("staging_root", type=Path, help="explicit installed prebuilt staging root")
-    args = parser.parse_args(argv)
-    issues = []
+def inspect_tree(root: Path, issues: list[dict], policy: bool) -> tuple[list[dict], list[dict]]:
+    """Inventory and inspect one explicit root; the fixed policy is for staging only."""
     targets = []
     results = []
     root_fd = None
-    root = args.staging_root
-    emit("root", path=str(root), fixed_glibc_ceiling="2.36")
+    emit("root", path=str(root), **({"fixed_glibc_ceiling": "2.36"} if policy else {}))
     try:
         if sys.platform != "linux":
             raise InspectionError("Linux descriptor-based ELF inspection is required")
@@ -638,7 +688,7 @@ def main(argv: list[str] | None = None) -> int:
             issue(issues, root, "no shipped .so/.so.*/.a libraries in staging inventory")
         tools = installed_tools(root, issues) if targets else {}
         for target in targets:
-            results.append(inspect_target(root_fd, target, directories, tools, issues))
+            results.append(inspect_target(root_fd, target, directories, tools, issues, policy))
         # Re-open the path and re-inventory metadata to catch late additions,
         # removals, renames, or a replacement root; payloads are never re-read.
         final_fd = open_root(root)
@@ -657,12 +707,59 @@ def main(argv: list[str] | None = None) -> int:
     finally:
         if root_fd is not None:
             os.close(root_fd)
+    return targets, results
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("staging_root", type=Path, help="explicit installed prebuilt staging root")
+    parser.add_argument("--baseline-root", default=None, metavar="DIR",
+                        help="explicit unpacked released archive; the staging tree's highest "
+                             "numeric GLIBC_ tag may not exceed the highest found there")
+    args = parser.parse_args(argv)
+    if args.baseline_root is not None and not args.baseline_root:
+        parser.error("--baseline-root must name a directory")
+    issues = []
+    targets, results = inspect_tree(args.staging_root, issues, True)
     inspected = sum(row["inspected"] for row in results)
-    success = bool(targets) and inspected == len(targets) and not issues
+    complete = bool(targets) and inspected == len(targets)
+    baseline_fields = {}
+    if args.baseline_root is not None:
+        baseline_root = Path(args.baseline_root)
+        baseline_targets = []
+        baseline_results = []
+        if roots_overlap(args.staging_root, baseline_root):
+            issue(issues, baseline_root.absolute(),
+                  "baseline root must be separate from the staging root")
+        else:
+            EVENT_CONTEXT["role"] = "baseline"
+            try:
+                baseline_targets, baseline_results = inspect_tree(baseline_root, issues, False)
+            finally:
+                EVENT_CONTEXT.clear()
+        baseline_inspected = sum(row["inspected"] for row in baseline_results)
+        baseline_fields = {"baseline_discovered": len(baseline_targets),
+                           "baseline_inspected": baseline_inspected}
+        complete = (complete and bool(baseline_targets)
+                    and baseline_inspected == len(baseline_targets))
+        if complete:
+            staged_highest = tree_maximum(results)
+            released_highest = tree_maximum(baseline_results)
+            above = exceeds_baseline(staged_highest, released_highest)
+            emit("baseline_comparison", staged_max_glibc=version_text(staged_highest),
+                 baseline_max_glibc=version_text(released_highest), exceeds=above)
+            if above:
+                issue(issues, args.staging_root.absolute(),
+                      f"staged highest {glibc_label(staged_highest)} exceeds released baseline "
+                      f"highest {glibc_label(released_highest)}",
+                      staged_max_glibc=version_text(staged_highest),
+                      baseline_max_glibc=version_text(released_highest),
+                      baseline_root=str(baseline_root.absolute()))
+    success = complete and not issues
     emit("summary", verdict="PASS" if success else "FAIL", discovered=len(targets),
          inspected=inspected, archive_members=sum(
              sum(member["inspected"] for member in row["members"]) for row in results),
-         issues=len(issues))
+         issues=len(issues), **baseline_fields)
     return 0 if success else 1