From 655e60834f9aaec371a6c3860c16da5e2cb500c6 Mon Sep 17 00:00:00 2001 From: Brendan Clement Date: Tue, 20 Oct 2026 10:33:07 -0700 Subject: [PATCH 01/19] refactor: run every assembler stage through one function The edition builder needs to assemble a site exactly as `make assemble` does. `assemble_site` runs the stages in order, and `main` now calls it, so a stage added later reaches both. --- scripts/assemble.py | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/scripts/assemble.py b/scripts/assemble.py index 43562171..2a141e2b 100644 --- a/scripts/assemble.py +++ b/scripts/assemble.py @@ -1016,6 +1016,22 @@ def emit( return written + 1 +def assemble_site(config: Config) -> tuple[int, list[str]]: + """Run every stage on `config` and write its output. + + Returns the number of files written and the warnings. `main` and the + edition builder (`edition.py`) both assemble through here, so a saved + edition is built exactly the way the site is. + """ + resolved = resolve(config) + docs_json, nav_raw = assemble_nav(resolved) + warnings = validate(config, resolved, docs_json) + merge(resolved) + inline_examples(resolved) + annotate(resolved, docs_json) + return emit(config, resolved, docs_json, nav_raw), warnings + + def revision(path: Path) -> str: """The commit a root was read from, so every build names its inputs. @@ -1133,13 +1149,7 @@ def main() -> int: print(f"openapi spec {'updated from' if changed else 'already matches'} {tag}") return 0 - resolved = resolve(config) - docs_json, nav_raw = assemble_nav(resolved) - warnings = validate(config, resolved, docs_json) - merge(resolved) - inline_examples(resolved) - annotate(resolved, docs_json) - count = emit(config, resolved, docs_json, nav_raw) + count, warnings = assemble_site(config) except AssembleError as exc: print(f"assemble: {exc}", file=sys.stderr) return 1 From 0b2b9d3ca7021ca3e39fbfd83dd6b4ca03f0586e Mon Sep 17 00:00:00 2001 From: Brendan Clement Date: Tue, 20 Oct 2026 10:33:07 -0700 Subject: [PATCH 02/19] feat: prepare saved documentation editions from release records `scripts/edition.py prepare` builds one release's edition with the assembler and saves it under `editions///`, with a release record in `releases/` naming the release and the commit of every root. Each root is read from Git at its commit, only from its docs folder. An Enterprise release must be qualified, and its shared pages come from the LanceDB commit it bundles. An OSS edition names its Enterprise source explicitly. A saved edition is never replaced: preparing the same release again changes nothing, even after a correction, and different inputs stop. A failed preparation saves nothing. `check` pairs records with editions and requires changed editions to list their corrections. --- scripts/edition.py | 606 ++++++++++++++++++++++++++++++++++ scripts/tests/test_edition.py | 534 ++++++++++++++++++++++++++++++ 2 files changed, 1140 insertions(+) create mode 100644 scripts/edition.py create mode 100644 scripts/tests/test_edition.py diff --git a/scripts/edition.py b/scripts/edition.py new file mode 100644 index 00000000..3a60d242 --- /dev/null +++ b/scripts/edition.py @@ -0,0 +1,606 @@ +""" +Prepare a saved documentation edition from one product release, and check the +saved editions. + +An edition is the site the assembler builds from one release's own sources, +saved under `editions///`. Its release record, +`releases/-.yaml`, names the release and the commit every +root was read from. Later builds compose the saved editions; nothing fetches +fresh content into one. See VERSIONING.md. + + prepare oss VERSION the lancedb/lancedb tag vVERSION, with the + Enterprise pages from a lancedb/sophon tag or + commit named explicitly with --enterprise + prepare enterprise VERSION the lancedb/sophon tag vVERSION, which must be + qualified, with the shared pages from the + LanceDB commit that release bundles + check pair every record with its edition, and require + an edition changed since it was prepared to + list its corrections + +Each root is read from Git objects at its commit, never from a working tree, +and only from its documentation folder: `docs/web` of lancedb/lancedb and +lancedb/sophon, and `docs` of this repository at HEAD, which also supplies +assemble.yaml and the assembler. + +A saved edition is never replaced. Preparing a release that already has one +changes nothing when the release inputs match its record, even after the +edition has been corrected, and stops when they differ. A preparation that +fails leaves neither an edition nor a record behind. + +Usage: + + python scripts/edition.py prepare oss VERSION --enterprise REF [--lancedb REPO] [--sophon REPO] + python scripts/edition.py prepare enterprise VERSION [--bundled-lancedb COMMIT] + [--lancedb REPO] [--sophon REPO] + python scripts/edition.py check +""" + +from __future__ import annotations + +import argparse +import os +import re +import shutil +import subprocess +import sys +import tempfile +from pathlib import Path, PurePosixPath + +import yaml + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +import assemble # noqa: E402 +import candidate # noqa: E402 + +REPO_ROOT = Path(__file__).resolve().parent.parent +FORMAT = 1 +PRODUCTS = ("oss", "enterprise") +# Editions are of stable releases. Prereleases need a policy first. +VERSION_RE = re.compile(r"(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)") +COMMIT_RE = candidate.COMMIT_RE +# Where each root in assemble.yaml is read from: repository and folder. A root +# added there without a source here stops preparation, so nothing reaches an +# edition without its provenance in the record. +SOURCES = { + "lancedb": ("lancedb/lancedb", "docs/web"), + "enterprise": ("lancedb/sophon", "docs/web"), + "build": ("lancedb/docs", "docs"), +} +# An Enterprise release bundles LanceDB as this gitlink, and a release that +# passed qualification has an annotated tag of this name on its commit. +BUNDLED_LANCEDB = "src/lancedb" +QUALIFIED_TAG = "release-qualified/v{version}" +# Files that would change how Git stores the edition's own files. +GIT_CONTROL = frozenset({".gitignore", ".gitattributes"}) +HEADER = """\ +# Release record, written by scripts/edition.py prepare. `release` and `sources` +# say what the edition was built from, and `prepared` describes the snapshot as +# first saved. Neither changes afterwards. A reviewed correction to the saved +# edition adds an entry to `corrections`; see VERSIONING.md. +""" + + +class EditionError(Exception): + """Inputs or state that must stop preparation before anything is saved.""" + + +# --------------------------------------------------------------------------- # +# git +# --------------------------------------------------------------------------- # + + +def git(repo: Path, *args: str, data: bytes | None = None) -> bytes: + result = subprocess.run( + ["git", "-C", str(repo), *args], input=data, capture_output=True + ) + if result.returncode: + message = result.stderr.decode(errors="replace").strip() + raise EditionError(f"git {args[0]} in {repo} failed: {message}") + return result.stdout + + +def text(repo: Path, *args: str) -> str: + return git(repo, *args).decode().strip() + + +def commit_of(repo: Path, ref: str, what: str) -> str: + try: + return text(repo, "rev-parse", "--verify", "--quiet", f"{ref}^{{commit}}") + except EditionError: + raise EditionError(f"{what}: {ref} is not a commit in {repo}; fetch it first") from None + + +def tag(repo: Path, name: str, what: str) -> tuple[str, str]: + """The object a tag points at and the commit it names.""" + ref = f"refs/tags/{name}" + try: + obj = text(repo, "rev-parse", "--verify", "--quiet", ref) + except EditionError: + raise EditionError(f"{what}: {repo} has no tag {name}; fetch the release tags first") from None + return obj, commit_of(repo, ref, what) + + +def pinned(repo: Path, ref: str, what: str) -> str: + """A tag or a full commit hash; a branch can move, so it is not accepted.""" + if COMMIT_RE.fullmatch(ref): + return commit_of(repo, ref, what) + if not text(repo, "tag", "--list", ref): + raise EditionError( + f"{what}: {ref} is neither a tag in {repo} nor a full commit hash; " + "name a pinned revision, not a branch" + ) + return tag(repo, ref, what)[1] + + +def bundled_lancedb(sophon: Path, commit: str, what: str) -> str: + """The LanceDB commit an Enterprise release bundles, from its gitlink.""" + listing = git(sophon, "ls-tree", "-z", commit, "--", BUNDLED_LANCEDB).decode() + meta = listing.split("\t", 1)[0].split() + if len(meta) != 3 or meta[0] != "160000": + raise EditionError( + f"{what} records no bundled LanceDB: {BUNDLED_LANCEDB} is not a gitlink " + f"at {commit}" + ) + return meta[2] + + +def export(repo: Path, commit: str, folder: str, dest: Path, what: str) -> None: + """Write the files under `folder` at `commit` into `dest`, byte for byte.""" + listing = git(repo, "ls-tree", "-r", "-z", "--full-tree", commit, "--", folder) + paths, blobs = [], [] + for entry in listing.decode().split("\0"): + if not entry: + continue + meta, path = entry.split("\t", 1) + mode, kind, blob = meta.split() + if not path.startswith(folder + "/"): + continue + if any(part in ("", ".", "..") for part in path.split("/")): + raise EditionError(f"{what}: {path!r} is not a plain path inside {folder}") + if mode == "120000" or kind != "blob": + raise EditionError( + f"{what}: {path} is a {'symlink' if mode == '120000' else kind}; " + "an edition reads only regular files" + ) + paths.append(path[len(folder) + 1 :]) + blobs.append(blob) + if not paths: + raise EditionError( + f"{what} has no {folder}: an edition is built only from a release's own " + "documentation sources" + ) + out = git(repo, "cat-file", "--batch", data="".join(f"{b}\n" for b in blobs).encode()) + offset = 0 + for path, blob in zip(paths, blobs): + header_end = out.index(b"\n", offset) + header = out[offset:header_end].split() + if len(header) != 3: + raise EditionError(f"{what}: {repo} lacks {folder}/{path} ({blob}); fetch it first") + size = int(header[2]) + target = dest / path + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(out[header_end + 1 : header_end + 1 + size]) + offset = header_end + 1 + size + 1 + + +# --------------------------------------------------------------------------- # +# release inputs +# --------------------------------------------------------------------------- # + + +def source(name: str, commit: str) -> dict: + repository, folder = SOURCES[name] + return {"repository": repository, "path": folder, "commit": commit} + + +def docs_commit(repo: Path) -> str: + """This repository's HEAD, which must hold everything the build reads.""" + changed = text(repo, "status", "--porcelain", "--", "assemble.yaml", "scripts", SOURCES["build"][1]) + if changed: + raise EditionError( + "commit or stash the changes to assemble.yaml, scripts/ and docs/ first; " + "the record names this checkout's HEAD:\n" + changed + ) + return commit_of(repo, "HEAD", "lancedb/docs") + + +def resolve_release(args: argparse.Namespace, repo: Path) -> dict: + """The record a preparation would write, before anything is built.""" + product, version = args.product, args.version + if not VERSION_RE.fullmatch(version): + raise EditionError(f"{version!r} is not a stable release version such as 1.2.3") + lancedb, sophon = args.lancedb.resolve(), args.sophon.resolve() + if product == "oss": + if args.bundled_lancedb: + raise EditionError("--bundled-lancedb applies to Enterprise releases only") + if not args.enterprise: + raise EditionError( + "an OSS edition includes the Enterprise pages, so name the lancedb/sophon " + "tag or commit they come from with --enterprise" + ) + release_tag = f"v{version}" + commit = tag(lancedb, release_tag, f"lancedb/lancedb {release_tag}")[1] + enterprise = pinned(sophon, args.enterprise, "lancedb/sophon --enterprise") + release = { + "repository": SOURCES["lancedb"][0], + "tag": release_tag, + "commit": commit, + "enterprise_pages": {"ref": args.enterprise, "commit": enterprise}, + } + shared = commit + else: + if args.enterprise: + raise EditionError( + "--enterprise applies to OSS releases; an Enterprise release reads its own pages" + ) + release_tag = f"v{version}" + what = f"lancedb/sophon {release_tag}" + enterprise = tag(sophon, release_tag, what)[1] + qualified = QUALIFIED_TAG.format(version=version) + try: + qualification, qualified_commit = tag(sophon, qualified, what) + except EditionError: + raise EditionError( + f"{what} is not qualified: {sophon} has no tag {qualified}" + ) from None + if text(sophon, "cat-file", "-t", qualification) != "tag": + raise EditionError(f"{what}: {qualified} is not an annotated tag") + if qualified_commit != enterprise: + raise EditionError( + f"{what} is {enterprise}, but {qualified} qualifies {qualified_commit}" + ) + shared = bundled_lancedb(sophon, enterprise, what) + if args.bundled_lancedb and args.bundled_lancedb != shared: + raise EditionError( + f"--bundled-lancedb {args.bundled_lancedb} disagrees with {what}, which " + f"bundles LanceDB {shared} at {BUNDLED_LANCEDB}" + ) + commit_of(lancedb, shared, f"the LanceDB that {what} bundles") + release = { + "repository": SOURCES["enterprise"][0], + "tag": release_tag, + "commit": enterprise, + "qualification": {"tag": qualified, "object": qualification}, + "bundled_lancedb": shared, + } + return { + "format": FORMAT, + "product": product, + "version": version, + "edition": f"editions/{product}/{version}", + "release": release, + "sources": { + "lancedb": source("lancedb", shared), + "enterprise": source("enterprise", enterprise), + "build": source("build", docs_commit(repo)), + }, + } + + +def identity(record: dict) -> dict: + """What makes two preparations the same release: commits, not ref names. + + This repository's own commit is left out: re-running a release from a later + HEAD is still the same release, and its saved edition is kept as it is. + """ + release = record["release"] + return { + "release": release.get("commit"), + "qualification": (release.get("qualification") or {}).get("object"), + "bundled_lancedb": release.get("bundled_lancedb"), + "enterprise_pages": (release.get("enterprise_pages") or {}).get("commit"), + "lancedb": record["sources"]["lancedb"]["commit"], + "enterprise": record["sources"]["enterprise"]["commit"], + } + + +# --------------------------------------------------------------------------- # +# records +# --------------------------------------------------------------------------- # + + +def record_path(repo: Path, product: str, version: str) -> Path: + return repo / "releases" / f"{product}-{version}.yaml" + + +def load_record(path: Path) -> dict: + """A release record, checked enough to compare and to verify its edition.""" + try: + record = yaml.safe_load(path.read_text(encoding="utf-8")) + except (OSError, yaml.YAMLError) as exc: + raise EditionError(f"{path.name}: unreadable: {exc}") from None + if not isinstance(record, dict) or record.get("format") != FORMAT: + raise EditionError(f"{path.name}: not a format {FORMAT} release record") + product, version = record.get("product"), record.get("version") + if product not in PRODUCTS or not isinstance(version, str) or not VERSION_RE.fullmatch(version): + raise EditionError(f"{path.name}: product and version are not a release") + if path.name != f"{product}-{version}.yaml": + raise EditionError(f"{path.name}: names {product} {version}") + if record.get("edition") != f"editions/{product}/{version}": + raise EditionError(f"{path.name}: edition must be editions/{product}/{version}") + prepared = record.get("prepared") + if not isinstance(prepared, dict) or not candidate.CHECKSUM_RE.fullmatch( + str(prepared.get("sha256")) + ): + raise EditionError(f"{path.name}: `prepared` has no snapshot checksum") + try: + identity(record) + except (AttributeError, KeyError, TypeError): + raise EditionError(f"{path.name}: `release` or `sources` is incomplete") from None + corrections = record.get("corrections") + if not isinstance(corrections, list) or not all( + isinstance(c, dict) and isinstance(c.get("summary"), str) and c["summary"].strip() + for c in corrections + ): + raise EditionError(f"{path.name}: every correction needs a one-line `summary`") + return record + + +def saved_hashes(repo: Path, edition: str) -> dict[str, str]: + """The edition's files as Git would save them: ignored local files, such as + `.DS_Store`, are not part of it. Preparation refuses an edition that has + any ignored path of its own.""" + hashes = candidate.file_hashes(repo / edition) + listing = "".join(f"{edition}/{p}\0" for p in hashes) + ignored = ignored_paths(repo, listing, index=True) + return {p: h for p, h in hashes.items() if f"{edition}/{p}" not in ignored} + + +def ignored_paths(repo: Path, listing: str, index: bool = False) -> set[str]: + """The NUL-separated paths in `listing` that .gitignore leaves out. + + With `index`, files Git already tracks are never ignored, as for `git add`. + """ + result = subprocess.run( + ["git", "-C", str(repo), "check-ignore", "-z", "--stdin", *([] if index else ["--no-index"])], + input=listing.encode(), capture_output=True, + ) + if result.returncode not in (0, 1): + raise EditionError(f"git check-ignore failed: {result.stderr.decode().strip()}") + return {name for name in result.stdout.decode().split("\0") if name} + + +def snapshot_state(repo: Path, record: dict) -> str: + """Whether the saved edition is still the prepared snapshot.""" + current = candidate.site_checksum(saved_hashes(repo, record["edition"])) + if current == record["prepared"]["sha256"]: + return "as prepared" + count = len(record["corrections"]) + if not count: + raise EditionError( + f"{record['edition']} differs from its prepared snapshot, and " + f"releases/{record['product']}-{record['version']}.yaml lists no correction" + ) + return f"corrected ({count} correction{'s' if count > 1 else ''})" + + +# --------------------------------------------------------------------------- # +# prepare +# --------------------------------------------------------------------------- # + + +def existing(repo: Path, requested: dict) -> str: + """What to say about a release that already has a record or an edition.""" + product, version = requested["product"], requested["version"] + path = record_path(repo, product, version) + folder = repo / requested["edition"] + if not path.exists(): + raise EditionError( + f"{requested['edition']} exists without releases/{path.name}; an edition " + "is never replaced, so remove an incomplete one in a reviewed change first" + ) + record = load_record(path) + if not folder.is_dir(): + raise EditionError( + f"releases/{path.name} exists, but {requested['edition']} does not; restore " + "it from Git history rather than preparing it again" + ) + recorded, wanted = identity(record), identity(requested) + if recorded != wanted: + changes = "; ".join( + f"{key} recorded {recorded[key]}, requested {wanted[key]}" + for key in recorded + if recorded[key] != wanted[key] + ) + raise EditionError( + f"{product} {version} is already saved from different inputs ({changes}); " + "an edition is never replaced, so review the difference instead" + ) + try: + state = snapshot_state(repo, record) + except EditionError as exc: + raise EditionError(f"{product} {version} is already saved; nothing changed, but {exc}") from None + return ( + f"{product} {version} is already saved from these release inputs: " + f"{requested['edition']} is {state}; nothing changed" + ) + + +def build_config(raw: dict, roots: dict[str, Path], output: Path, path: Path) -> Path: + """assemble.yaml as committed, reading the exported roots into `output`.""" + raw = dict(raw, roots=[dict(entry) for entry in raw.get("roots", [])]) + names = [entry.get("name") for entry in raw.get("roots", [])] + if sorted(names) != sorted(SOURCES): + raise EditionError( + f"assemble.yaml has roots {names}, but edition.py records the sources of " + f"{sorted(SOURCES)}; add every root to SOURCES" + ) + for entry in raw["roots"]: + entry["path"] = str(roots[entry["name"]]) + raw["output"] = str(output) + if raw.get("openapi"): + # The spec is a file of the docs root; check the exported copy. + dest = PurePosixPath(raw["openapi"]["dest"]).relative_to(SOURCES["build"][1]) + raw["openapi"] = dict(raw["openapi"], dest=str(roots["build"] / dest)) + path.write_text(yaml.safe_dump(raw, sort_keys=False), encoding="utf-8") + return path + + +def openapi_source(raw: dict, hashes: dict[str, str]) -> dict | None: + """The released spec the docs root ships, and the hash of the saved copy.""" + spec = raw.get("openapi") + if not spec: + return None + rel = PurePosixPath(spec["dest"]).relative_to(SOURCES["build"][1]).as_posix() + return { + "repository": spec["repo"], + "release": spec["release"], + "path": spec["source"], + "sha256": hashes.get(rel), + } + + +def place(staging: Path, site: Path, record: str, repo: Path, edition: Path, path: Path) -> None: + """Move the edition into place, then create its record; undo both on failure.""" + created = [d for d in (edition.parent.parent, edition.parent, path.parent) if not d.exists()] + moved = claimed = False + try: + for directory in (edition.parent, path.parent): + directory.mkdir(parents=True, exist_ok=True) + if os.path.lexists(edition) or os.path.lexists(path): + raise EditionError(f"{edition.relative_to(repo)} appeared while preparing; nothing saved") + os.rename(site, edition) + moved = True + # Claim the record's name, then put the whole record there at once. + os.close(os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL)) + claimed = True + draft = staging / "record.yaml" + draft.write_text(record, encoding="utf-8") + os.replace(draft, path) + except BaseException: + if claimed: + path.unlink() + if moved: + os.rename(edition, site) + for directory in reversed(created): + if directory.exists() and not any(directory.iterdir()): + directory.rmdir() + raise + + +def prepare(args: argparse.Namespace, repo: Path = REPO_ROOT) -> str: + requested = resolve_release(args, repo) + edition = repo / requested["edition"] + path = record_path(repo, requested["product"], requested["version"]) + if os.path.lexists(edition) or os.path.lexists(path): + return existing(repo, requested) + + (repo / "build").mkdir(exist_ok=True) + staging = Path(tempfile.mkdtemp(prefix="edition-", dir=repo / "build")).resolve() + try: + roots = {} + for name, (repository, folder) in SOURCES.items(): + roots[name] = staging / "roots" / name + commit = requested["sources"][name]["commit"] + source_repo = {"lancedb": args.lancedb, "enterprise": args.sophon}.get(name, repo) + export(source_repo.resolve(), commit, folder, roots[name], f"{repository} {commit}") + site = staging / "site" + raw = yaml.safe_load(git(repo, "show", "HEAD:assemble.yaml").decode()) + config = assemble.load_config(build_config(raw, roots, site, staging / "assemble.yaml")) + _count, warnings = assemble.assemble_site(config) + for warning in warnings: + print(f"warning: {warning}", file=sys.stderr) + + hashes = candidate.file_hashes(site) + control = sorted(p for p in hashes if PurePosixPath(p).name in GIT_CONTROL) + if control: + raise EditionError(f"the edition would hold {control}, which change how Git stores it") + listing = "".join(f"{requested['edition']}/{p}\0" for p in hashes) + f"releases/{path.name}\0" + ignored = sorted(ignored_paths(repo, listing)) + if ignored: + raise EditionError(f".gitignore would leave out {ignored[:5]}; the edition must be saved whole") + + openapi = openapi_source(raw, hashes) + record = { + **requested, + "sources": {**requested["sources"], **({"openapi": openapi} if openapi else {})}, + "prepared": {"files": len(hashes), "sha256": candidate.site_checksum(hashes)}, + "corrections": [], + } + place(staging, site, HEADER + yaml.safe_dump(record, sort_keys=False), repo, edition, path) + except (assemble.AssembleError, candidate.CandidateError) as exc: + raise EditionError(str(exc)) from None + finally: + shutil.rmtree(staging, ignore_errors=True) + try: + (repo / "build").rmdir() + except OSError: + pass + sources = " ".join(f"{k}={v['commit']}" for k, v in requested["sources"].items()) + return ( + f"prepared {requested['product']} {requested['version']}: {len(hashes)} files, " + f"sha256 {record['prepared']['sha256']}, in {requested['edition']} with " + f"releases/{path.name}\nsources: {sources}" + ) + + +# --------------------------------------------------------------------------- # +# check +# --------------------------------------------------------------------------- # + + +def check(repo: Path = REPO_ROOT) -> tuple[list[str], list[str]]: + """Each edition's state, and every problem with records and editions.""" + lines, problems, paired = [], [], set() + for path in sorted((repo / "releases").glob("*.yaml")): + try: + record = load_record(path) + if not (repo / record["edition"]).is_dir(): + raise EditionError(f"{path.name}: {record['edition']} is missing") + paired.add(record["edition"]) + lines.append(f"{record['product']} {record['version']}: {snapshot_state(repo, record)}") + except (EditionError, candidate.CandidateError) as exc: + problems.append(str(exc)) + editions = repo / "editions" + entries = sorted([*editions.glob("*"), *editions.glob("*/*")]) if editions.is_dir() else [] + names = [entry.relative_to(repo).as_posix() for entry in entries] + ignored = ignored_paths(repo, "".join(f"{name}\0" for name in names), index=True) + for entry, rel in zip(entries, names): + if rel in ignored: + continue + if entry.parent == editions: + if entry.name not in PRODUCTS or not entry.is_dir(): + problems.append(f"{rel} is not a product folder") + elif rel not in paired: + problems.append(f"{rel} has no release record") + return lines, problems + + +def parse(argv: list[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__.split("\n")[1]) + commands = parser.add_subparsers(dest="command", required=True) + prep = commands.add_parser("prepare", help="save one release's edition") + prep.add_argument("product", choices=PRODUCTS) + prep.add_argument("version", help="the release version, such as 1.2.3") + prep.add_argument("--enterprise", metavar="REF", + help="OSS only: the lancedb/sophon tag or commit for the Enterprise pages") + prep.add_argument("--bundled-lancedb", metavar="COMMIT", + help="Enterprise only: the LanceDB commit the release is expected to bundle") + prep.add_argument("--lancedb", type=Path, default=REPO_ROOT.parent / "lancedb", + help="a lancedb/lancedb repository (default: ../lancedb)") + prep.add_argument("--sophon", type=Path, default=REPO_ROOT.parent / "sophon", + help="a lancedb/sophon repository (default: ../sophon)") + commands.add_parser("check", help="check the saved editions against their records") + return parser.parse_args(argv) + + +def main(argv: list[str] | None = None) -> int: + args = parse(argv) + try: + if args.command == "prepare": + print(prepare(args)) + return 0 + lines, problems = check() + except (EditionError, candidate.CandidateError) as exc: + print(f"edition: {exc}", file=sys.stderr) + return 1 + print("\n".join(lines) or "no saved editions") + for problem in problems: + print(f"edition: {problem}", file=sys.stderr) + return 1 if problems else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/tests/test_edition.py b/scripts/tests/test_edition.py new file mode 100644 index 00000000..32f9f512 --- /dev/null +++ b/scripts/tests/test_edition.py @@ -0,0 +1,534 @@ +"""Tests for preparing saved editions from release records. + +Disposable repositories stand in for lancedb/lancedb, lancedb/sophon and this +repository. Their versions are fixtures, not releases: OSS 0.95.0 to 0.99.0 and +Enterprise 8.1.0 to 8.9.0, each built to exercise one case. The docs fixture +uses this repository's own assemble.yaml and .gitignore, and every edition is +built by the real assembler. + +Run with `make test-assemble`. +""" + +import hashlib +import json +import os +import shutil +import subprocess +import sys +from dataclasses import dataclass +from pathlib import Path + +import pytest +import yaml + +SCRIPTS = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(SCRIPTS)) + +import edition # noqa: E402 + +NAVIGATION = { + "navigation": { + "tabs": [ + { + "tab": "Documentation", + "groups": [ + {"group": "Start", "pages": ["index", "guide"]}, + {"group": "Enterprise", "pages": ["enterprise/security"]}, + ], + } + ] + } +} +GENEVA_NAV = {"tabs": [{"tab": "Geneva", "groups": [{"group": "Geneva", "pages": ["geneva/index"]}]}]} +MARKER = bytes(range(256)) +INTERNAL = b"Internal engineering notes\n" + + +@dataclass(frozen=True) +class Gitlink: + commit: str + + +@dataclass(frozen=True) +class Symlink: + target: str + + +def environment() -> dict[str, str]: + env = {k: v for k, v in os.environ.items() if not k.startswith("GIT_")} + env.update( + GIT_AUTHOR_NAME="Test", + GIT_AUTHOR_EMAIL="test@example.com", + GIT_COMMITTER_NAME="Test", + GIT_COMMITTER_EMAIL="test@example.com", + GIT_CONFIG_GLOBAL=os.devnull, + GIT_CONFIG_NOSYSTEM="1", + GIT_TERMINAL_PROMPT="0", + ) + return env + + +@pytest.fixture(autouse=True) +def isolated_git(monkeypatch): + """The builder's own git calls see no user configuration either.""" + for key in [k for k in os.environ if k.startswith("GIT_")]: + monkeypatch.delenv(key) + for key, value in environment().items(): + if key.startswith("GIT_"): + monkeypatch.setenv(key, value) + + +def git(cwd: Path, *args: str, data: bytes | None = None) -> str: + return subprocess.run( + ["git", *args], cwd=cwd, env=environment(), input=data, check=True, capture_output=True + ).stdout.decode().strip() + + +def snapshot(repo: Path, files: dict, message: str) -> str: + """Commit exactly `files` on main, whatever the working tree holds.""" + lines = [] + for path, content in sorted(files.items()): + if isinstance(content, Gitlink): + lines.append(f"160000 commit {content.commit}\t{path}") + continue + data = content.target.encode() if isinstance(content, Symlink) else content + blob = git(repo, "hash-object", "-w", "--stdin", data=data) + lines.append(f"{'120000' if isinstance(content, Symlink) else '100644'} blob {blob}\t{path}") + index = repo / ".git" / "fixture-index" + env = {**environment(), "GIT_INDEX_FILE": str(index)} + subprocess.run(["git", "read-tree", "--empty"], cwd=repo, env=env, check=True) + subprocess.run(["git", "update-index", "--index-info"], cwd=repo, env=env, check=True, + input="".join(f"{line}\n" for line in lines).encode()) + tree = subprocess.run(["git", "write-tree"], cwd=repo, env=env, check=True, + capture_output=True).stdout.decode().strip() + index.unlink() + parent = subprocess.run(["git", "rev-parse", "--verify", "--quiet", "HEAD"], cwd=repo, + env=environment(), capture_output=True).stdout.decode().strip() + commit = git(repo, "commit-tree", tree, *(["-p", parent] if parent else []), "-m", message) + git(repo, "update-ref", "refs/heads/main", commit) + return commit + + +def annotated(repo: Path, name: str, commit: str) -> str: + git(repo, "tag", "-a", "-m", name, name, commit) + return git(repo, "rev-parse", f"refs/tags/{name}") + + +def new_repo(path: Path) -> Path: + path.mkdir(parents=True) + git(path, "init", "-q", "-b", "main") + return path + + +def oss_pages(label: str) -> dict: + """lancedb's docs/web as of `label`: navigation, a page with an example and a + draft section, a draft page, an asset, and the Enterprise placeholder the + current assembler needs.""" + web = "docs/web/" + guide = ( + "---\ntitle: Guide\n---\n\n" + 'import { PyGuide } from "/snippets/guide.mdx";\n\n' + f"Shared guide as of {label}.\n\n" + '\n{PyGuide}\n\n\n' + "{/*\nDraft for the next release.\n\n## Coming soon {#coming-soon}\n*/}\n" + ) + return { + "README.md": b"Not documentation\n", + "python/lancedb.py": b"print('library code')\n", + web + "docs.json": json.dumps(NAVIGATION).encode(), + web + "index.mdx": f"# LanceDB {label}\n".encode(), + web + "guide.mdx": guide.encode(), + web + "snippets/guide.mdx": f"export const PyGuide = {json.dumps(f'print({label!r})' + chr(10))};\n".encode(), + web + "static/assets/marker.svg": MARKER, + web + "enterprise/security.mdx": b"Open-source security placeholder\n", + web + ".mintignore": b"draft.mdx\n", + web + "draft.mdx": b"# Not released yet\n", + } + + +def enterprise_pages(label: str, bundled: str | None, **extra) -> dict: + files = { + "README.md": b"Not documentation\n", + "docs/ci.md": INTERNAL, + "docs/internal/notes.mdx": INTERNAL, + "docs/web/enterprise/security.mdx": f"Enterprise security for {label}\n".encode(), + **extra, + } + if bundled: + files["src/lancedb"] = Gitlink(bundled) + return files + + +@pytest.fixture(scope="module") +def sources(tmp_path_factory) -> dict: + """lancedb and sophon repositories with tagged fixture releases. Shared, so + tests only read them.""" + base = tmp_path_factory.mktemp("sources").resolve() + lancedb = new_repo(base / "lancedb") + c = {} + c["no-docs"] = snapshot(lancedb, {"README.md": b"Before docs/web\n"}, "no docs yet") + git(lancedb, "tag", "v0.97.0", c["no-docs"]) + c["bundled"] = snapshot(lancedb, oss_pages("the bundled revision"), "bundled by 8.1") + c["ignored"] = snapshot(lancedb, {**oss_pages("0.95.0"), "docs/web/scratch/notes.mdx": b"x\n"}, "0.95") + git(lancedb, "tag", "v0.95.0", c["ignored"]) + c["0.98.0"] = snapshot(lancedb, oss_pages("0.98.0"), "0.98") + annotated(lancedb, "v0.98.0", c["0.98.0"]) + c["0.99.0"] = snapshot(lancedb, oss_pages("0.99.0"), "0.99") + annotated(lancedb, "v0.99.0", c["0.99.0"]) + c["main"] = snapshot(lancedb, oss_pages("unreleased main"), "main moves on") + # A working tree that differs from every commit. + (lancedb / "docs/web").mkdir(parents=True) + (lancedb / "docs/web/guide.mdx").write_text("Uncommitted text\n") + + sophon = new_repo(base / "sophon") + e = {} + + def release(version: str, files: dict, qualified: str | None = "self") -> str: + commit = snapshot(sophon, files, version) + annotated(sophon, f"v{version}", commit) + if qualified: + annotated(sophon, f"release-qualified/v{version}", commit if qualified == "self" else qualified) + e[version] = commit + return commit + + release("8.1.0", enterprise_pages("8.1", c["bundled"])) + release("8.2.0", enterprise_pages("8.2", c["0.99.0"])) + release("8.3.0", enterprise_pages("8.3", c["0.99.0"]), qualified=None) + release("8.4.0", enterprise_pages("8.4", c["0.99.0"]), qualified=e["8.3.0"]) + release("8.5.0", enterprise_pages("8.5", c["no-docs"])) + release("8.6.0", enterprise_pages("8.6", None)) + release("8.7.0", enterprise_pages("8.7", c["0.99.0"], **{"docs/web/.notes.mdx": INTERNAL})) + release("8.8.0", enterprise_pages("8.8", c["0.99.0"], **{"docs/web/enterprise/ci.md": Symlink("../../ci.md")})) + release("8.9.0", enterprise_pages("8.9", "1" * 40)) + git(sophon, "tag", "v8.1.0-lightweight", e["8.1.0"]) + e["main"] = snapshot(sophon, enterprise_pages("unreleased main", c["main"]), "main moves on") + return {"lancedb": lancedb, "sophon": sophon, "oss": c, "enterprise": e} + + +@pytest.fixture +def docs(tmp_path) -> Path: + """A lancedb/docs checkout: this repository's assemble.yaml and .gitignore, + and docs-owned pages, a navigation fragment and the OpenAPI spec.""" + repo = new_repo(tmp_path.resolve() / "docs") + for name in ("assemble.yaml", ".gitignore"): + shutil.copy(SCRIPTS.parent / name, repo / name) + files = { + "docs/docs.nav.json": json.dumps(GENEVA_NAV), + "docs/geneva/index.mdx": "# Geneva\n", + "docs/api-reference/rest/openapi.yml": "openapi: 3.0.0\n", + } + for rel, content in files.items(): + (repo / rel).parent.mkdir(parents=True, exist_ok=True) + (repo / rel).write_text(content) + git(repo, "add", "-A") + git(repo, "commit", "-q", "-m", "docs") + return repo + + +def prepare(docs: Path, sources: dict, *argv: str, lancedb: Path | None = None) -> str: + args = edition.parse([ + "prepare", *argv, + "--lancedb", str(lancedb or sources["lancedb"]), "--sophon", str(sources["sophon"]), + ]) + return edition.prepare(args, repo=docs) + + +def saved(root: Path) -> dict[str, bytes]: + """Every file under `root`, hidden ones included.""" + return { + p.relative_to(root).as_posix(): p.read_bytes() + for p in sorted(root.rglob("*")) + if p.is_file() + } + + +def checksum(files: dict[str, bytes]) -> str: + listing = "".join(f"{hashlib.sha256(files[p]).hexdigest()} {p}\n" for p in sorted(files)) + return hashlib.sha256(listing.encode()).hexdigest() + + +def record(docs: Path, name: str) -> dict: + return yaml.safe_load((docs / "releases" / f"{name}.yaml").read_text()) + + +def state(docs: Path) -> dict[str, bytes | None]: + """What preparation may change: editions, records and leftovers in build/, + directories included.""" + found = {} + for top in ("editions", "releases", "build"): + root = docs / top + if root.exists(): + found[top] = None + for path in sorted(root.rglob("*")): + found[f"{top}/{path.relative_to(root).as_posix()}"] = ( + path.read_bytes() if path.is_file() else None + ) + return found + + +def correct(docs: Path, name: str, page: str, text: str) -> None: + """A reviewed correction: edit the saved page and say so in the record.""" + product, version = name.split("-") + (docs / "editions" / product / version / page).write_text(text) + path = docs / "releases" / f"{name}.yaml" + path.write_text(path.read_text().replace( + "corrections: []", f"corrections:\n- summary: Fix {page}.\n files: [{page}]" + )) + + +# --------------------------------------------------------------------------- # +# preparing editions +# --------------------------------------------------------------------------- # + + +def test_oss_edition_is_built_from_its_tag_and_named_enterprise_pages(docs, sources): + out = prepare(docs, sources, "oss", "0.98.0", "--enterprise", "v8.1.0") + + files = saved(docs / "editions/oss/0.98.0") + assert sorted(files) == [ + ".mintignore", "api-reference/rest/openapi.yml", "docs.json", "draft.mdx", + "enterprise/security.mdx", "geneva/index.mdx", "guide.mdx", "index.mdx", + "static/assets/marker.svg", + ] + guide = files["guide.mdx"].decode() + # The tag's own page, with its example written in, its draft section and + # draft page kept for Mint, and its asset byte for byte. + assert "Shared guide as of 0.98.0." in guide + assert "unreleased main" not in guide and "Uncommitted" not in guide + assert "```python Python icon=\"python\"\nprint('0.98.0')\n```" in guide + assert "/snippets/" not in guide + assert "{/*\nDraft for the next release." in guide + assert files[".mintignore"] == b"draft.mdx\n" + assert files["static/assets/marker.svg"] == MARKER + assert files["enterprise/security.mdx"] == b"Enterprise security for 8.1\n" + assert json.loads(files["docs.json"])["navigation"]["tabs"][1]["tab"] == "Geneva" + + oss, ent = sources["oss"], sources["enterprise"] + rec = record(docs, "oss-0.98.0") + assert rec["release"] == { + "repository": "lancedb/lancedb", + "tag": "v0.98.0", + "commit": oss["0.98.0"], + "enterprise_pages": {"ref": "v8.1.0", "commit": ent["8.1.0"]}, + } + assert rec["sources"] == { + "lancedb": {"repository": "lancedb/lancedb", "path": "docs/web", "commit": oss["0.98.0"]}, + "enterprise": {"repository": "lancedb/sophon", "path": "docs/web", "commit": ent["8.1.0"]}, + "build": {"repository": "lancedb/docs", "path": "docs", "commit": git(docs, "rev-parse", "HEAD")}, + "openapi": { + "repository": "lance-format/lance-namespace", + "release": yaml.safe_load((docs / "assemble.yaml").read_text())["openapi"]["release"], + "path": "docs/src/spec.yaml", + "sha256": hashlib.sha256(b"openapi: 3.0.0\n").hexdigest(), + }, + } + assert rec["prepared"] == {"files": len(files), "sha256": checksum(files)} + assert rec["corrections"] == [] + assert "prepared oss 0.98.0: 9 files" in out + assert not (docs / "build").exists() + + +def test_enterprise_edition_reads_the_lancedb_it_bundles(docs, sources): + bundled = sources["oss"]["bundled"] + prepare(docs, sources, "enterprise", "8.1.0", "--bundled-lancedb", bundled) + + files = saved(docs / "editions/enterprise/8.1.0") + # The shared pages are the bundled revision's, which no OSS release tags, + # and no OSS edition is needed for them. + assert "Shared guide as of the bundled revision." in files["guide.mdx"].decode() + assert files["enterprise/security.mdx"] == b"Enterprise security for 8.1\n" + assert not (docs / "editions/oss").exists() + assert not any(INTERNAL in data for data in files.values()) + + rec = record(docs, "enterprise-8.1.0") + sophon = sources["sophon"] + assert rec["release"] == { + "repository": "lancedb/sophon", + "tag": "v8.1.0", + "commit": sources["enterprise"]["8.1.0"], + "qualification": { + "tag": "release-qualified/v8.1.0", + "object": git(sophon, "rev-parse", "refs/tags/release-qualified/v8.1.0"), + }, + "bundled_lancedb": bundled, + } + assert rec["sources"]["lancedb"]["commit"] == bundled + assert rec["sources"]["enterprise"]["commit"] == sources["enterprise"]["8.1.0"] + + +@pytest.mark.parametrize( + "argv, error", + [ + (["oss", "0.97.0", "--enterprise", "v8.1.0"], "has no docs/web"), + (["oss", "0.96.0", "--enterprise", "v8.1.0"], "has no tag v0.96.0"), + (["oss", "1.0.0-beta.1", "--enterprise", "v8.1.0"], "not a stable release version"), + (["oss", "0.98.0"], "name the lancedb/sophon tag or commit"), + (["oss", "0.98.0", "--enterprise", "main"], "not a branch"), + (["oss", "0.98.0", "--enterprise", "v8.1.0", "--bundled-lancedb", "1" * 40], "Enterprise releases only"), + (["oss", "0.95.0", "--enterprise", "v8.1.0"], ".gitignore would leave out"), + (["enterprise", "8.1.0", "--enterprise", "v8.2.0"], "applies to OSS releases"), + (["enterprise", "8.3.0"], "is not qualified"), + (["enterprise", "8.4.0"], "but release-qualified/v8.4.0 qualifies"), + (["enterprise", "8.5.0"], "has no docs/web"), + (["enterprise", "8.6.0"], "records no bundled LanceDB"), + (["enterprise", "8.1.0", "--bundled-lancedb", "1" * 40], "disagrees with lancedb/sophon v8.1.0"), + (["enterprise", "8.9.0"], "fetch it first"), + (["enterprise", "8.7.0"], r"\.notes\.mdx is hidden"), + (["enterprise", "8.8.0"], "ci.md is a symlink"), + ], +) +def test_preparation_stops_and_saves_nothing(docs, sources, argv, error): + with pytest.raises(edition.EditionError, match=error): + prepare(docs, sources, *argv) + + assert state(docs) == {} + + +# --------------------------------------------------------------------------- # +# protecting saved editions +# --------------------------------------------------------------------------- # + + +def test_conflicting_inputs_for_a_saved_release_stop(docs, sources, tmp_path): + prepare(docs, sources, "oss", "0.98.0", "--enterprise", "v8.1.0") + before = state(docs) + + with pytest.raises(edition.EditionError, match="enterprise_pages recorded .*, requested"): + prepare(docs, sources, "oss", "0.98.0", "--enterprise", "v8.2.0") + + # The release tag moved after the edition was saved. + retagged = tmp_path / "retagged" + git(tmp_path, "clone", "-q", str(sources["lancedb"]), str(retagged)) + git(retagged, "tag", "-f", "-a", "-m", "moved", "v0.98.0", sources["oss"]["0.99.0"]) + with pytest.raises(edition.EditionError, match="already saved from different inputs .*release recorded"): + prepare(docs, sources, "oss", "0.98.0", "--enterprise", "v8.1.0", lancedb=retagged) + + assert state(docs) == before + + +def test_duplicate_preparation_changes_nothing_even_after_a_correction(docs, sources): + prepare(docs, sources, "oss", "0.98.0", "--enterprise", "v8.1.0") + out = prepare(docs, sources, "oss", "0.98.0", "--enterprise", "v8.1.0") + assert "editions/oss/0.98.0 is as prepared; nothing changed" in out + + correct(docs, "oss-0.98.0", "guide.mdx", "Corrected guide\n") + # This repository moves on, and the same release is prepared again, its + # Enterprise pages named by commit this time. + (docs / "docs/geneva/index.mdx").write_text("# Geneva, later\n") + git(docs, "commit", "-q", "-am", "later") + before = state(docs) + out = prepare(docs, sources, "oss", "0.98.0", "--enterprise", sources["enterprise"]["8.1.0"]) + + assert "is corrected (1 correction); nothing changed" in out + assert state(docs) == before + assert (docs / "editions/oss/0.98.0/guide.mdx").read_text() == "Corrected guide\n" + + +def test_a_newer_edition_leaves_a_corrected_older_one_alone(docs, sources): + prepare(docs, sources, "oss", "0.98.0", "--enterprise", "v8.1.0") + correct(docs, "oss-0.98.0", "guide.mdx", "Corrected guide\n") + before = state(docs) + + prepare(docs, sources, "oss", "0.99.0", "--enterprise", "v8.2.0") + prepare(docs, sources, "enterprise", "8.2.0") + + after = state(docs) + assert {k: v for k, v in after.items() if k in before} == before + assert "Shared guide as of 0.99.0." in (docs / "editions/oss/0.99.0/guide.mdx").read_text() + assert "Shared guide as of 0.99.0." in (docs / "editions/enterprise/8.2.0/guide.mdx").read_text() + + +def test_a_failed_preparation_leaves_saved_editions_intact(docs, sources, monkeypatch): + prepare(docs, sources, "oss", "0.98.0", "--enterprise", "v8.1.0") + before = state(docs) + + with pytest.raises(edition.EditionError, match="has no docs/web"): + prepare(docs, sources, "enterprise", "8.5.0") + assert state(docs) == before + + def fail(*_args): + raise OSError("disk full") + + # The edition cannot be moved into place, after its folders were made. + with monkeypatch.context() as patch: + patch.setattr(edition.os, "rename", fail) + with pytest.raises(OSError, match="disk full"): + prepare(docs, sources, "enterprise", "8.1.0") + assert state(docs) == before + + # The last step fails: the record cannot be put in place after the edition + # has been moved there. + with monkeypatch.context() as patch: + patch.setattr(edition.os, "replace", fail) + with pytest.raises(OSError, match="disk full"): + prepare(docs, sources, "enterprise", "8.1.0") + assert state(docs) == before + + +def test_an_edition_or_a_record_without_its_pair_is_refused(docs, sources): + orphan = docs / "editions/oss/0.99.0" + orphan.mkdir(parents=True) + (orphan / "index.mdx").write_text("half made\n") + with pytest.raises(edition.EditionError, match="exists without releases/oss-0.99.0.yaml"): + prepare(docs, sources, "oss", "0.99.0", "--enterprise", "v8.2.0") + + prepare(docs, sources, "oss", "0.98.0", "--enterprise", "v8.1.0") + shutil.rmtree(docs / "editions/oss/0.98.0") + with pytest.raises(edition.EditionError, match="restore it from Git history"): + prepare(docs, sources, "oss", "0.98.0", "--enterprise", "v8.1.0") + assert not (docs / "editions/oss/0.98.0").exists() + + +# --------------------------------------------------------------------------- # +# checking saved editions +# --------------------------------------------------------------------------- # + + +def test_check_pairs_records_and_requires_corrections_to_be_listed(docs, sources): + assert edition.check(docs) == ([], []) + + prepare(docs, sources, "oss", "0.98.0", "--enterprise", "v8.1.0") + # Finder files are not part of what Git saves. + for folder in ("editions", "editions/oss", "editions/oss/0.98.0"): + (docs / folder / ".DS_Store").write_bytes(b"\0") + assert edition.check(docs) == (["oss 0.98.0: as prepared"], []) + + (docs / "editions/oss/0.98.0/guide.mdx").write_text("Edited without a record\n") + lines, problems = edition.check(docs) + assert lines == [] and "lists no correction" in problems[0] + + correct(docs, "oss-0.98.0", "guide.mdx", "Corrected guide\n") + (docs / "editions/enterprise/8.1.0").mkdir(parents=True) + lines, problems = edition.check(docs) + assert lines == ["oss 0.98.0: corrected (1 correction)"] + assert problems == ["editions/enterprise/8.1.0 has no release record"] + + +def test_command_records_only_committed_build_inputs(docs, sources): + """End to end, as a person runs it: the scripts are part of the checkout.""" + (docs / "scripts").mkdir() + for name in ("assemble.py", "candidate.py", "edition.py"): + shutil.copy(SCRIPTS / name, docs / "scripts" / name) + git(docs, "add", "-A") + git(docs, "commit", "-q", "-m", "scripts") + + def run(*argv: str) -> subprocess.CompletedProcess: + return subprocess.run( + [sys.executable, "-B", "scripts/edition.py", *argv], + cwd=docs, env=environment(), capture_output=True, text=True, + ) + + repos = ["--lancedb", str(sources["lancedb"]), "--sophon", str(sources["sophon"])] + result = run("prepare", "oss", "0.98.0", "--enterprise", "v8.1.0", *repos) + assert result.returncode == 0, result.stderr + assert record(docs, "oss-0.98.0")["sources"]["build"]["commit"] == git(docs, "rev-parse", "HEAD") + + with (docs / "scripts/assemble.py").open("a") as script: + script.write("# a local change\n") + result = run("prepare", "oss", "0.99.0", "--enterprise", "v8.2.0", *repos) + assert result.returncode == 1 + assert "commit or stash the changes" in result.stderr + assert not (docs / "editions/oss/0.99.0").exists() + + result = run("check") + assert (result.returncode, result.stdout) == (0, "oss 0.98.0: as prepared\n") From 034e2c1699e6a0b6b840ce46008800dcdd6c5071 Mon Sep 17 00:00:00 2001 From: Brendan Clement Date: Tue, 20 Oct 2026 10:33:07 -0700 Subject: [PATCH 03/19] docs: describe versioned editions and how to correct them VERSIONING.md carries the approved design: decisions, components, records, corrections, planned reader and publication behavior, and the open questions. The README explains how to prepare and correct an edition, with a sample record. --- AGENTS.md | 7 +- CLAUDE.md | 7 +- README.md | 122 ++++++++++++++++++++++++++++ VERSIONING.md | 215 ++++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 349 insertions(+), 2 deletions(-) create mode 100644 VERSIONING.md diff --git a/AGENTS.md b/AGENTS.md index 4c5d5e9c..555707c3 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -11,4 +11,9 @@ This is a documentation site for [LanceDB](https://docs.lancedb.com). ## Snippets - Code snippets are generated in `lancedb/lancedb` from the tests under `docs/web-tests/`, not in this repository. See "Code snippets" in README.md for the command. -- The four Geneva snippets in `docs/snippets/` are frozen copies; do not regenerate or edit them. \ No newline at end of file +- The four Geneva snippets in `docs/snippets/` are frozen copies; do not regenerate or edit them. + +## Saved editions + +- `editions/` holds saved documentation editions and `releases/` their records. Create an edition only with `scripts/edition.py prepare`, and change one only as a reviewed correction listed in its record. See "Saved editions" in README.md. +- Write new documentation in `lancedb/lancedb` or `lancedb/sophon`, not in a saved edition. diff --git a/CLAUDE.md b/CLAUDE.md index 4c5d5e9c..555707c3 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -11,4 +11,9 @@ This is a documentation site for [LanceDB](https://docs.lancedb.com). ## Snippets - Code snippets are generated in `lancedb/lancedb` from the tests under `docs/web-tests/`, not in this repository. See "Code snippets" in README.md for the command. -- The four Geneva snippets in `docs/snippets/` are frozen copies; do not regenerate or edit them. \ No newline at end of file +- The four Geneva snippets in `docs/snippets/` are frozen copies; do not regenerate or edit them. + +## Saved editions + +- `editions/` holds saved documentation editions and `releases/` their records. Create an edition only with `scripts/edition.py prepare`, and change one only as a reviewed correction listed in its record. See "Saved editions" in README.md. +- Write new documentation in `lancedb/lancedb` or `lancedb/sophon`, not in a saved edition. diff --git a/README.md b/README.md index 7de2d1a3..a3e6981b 100644 --- a/README.md +++ b/README.md @@ -202,6 +202,128 @@ set. It cannot see whether the environment requires reviewers and admits only `main`, or whether the ruleset refuses every other writer: verify those separately before the first production publication. +## Saved editions + +Each published documentation edition is saved in this repository: the site the +assembler built from one product release's own sources, in +`editions///`, with a release record in +`releases/-.yaml` that names every source it was read from. +New documentation is still written in `lancedb/lancedb` and `lancedb/sophon`. +[VERSIONING.md](VERSIONING.md) describes the design. Composing the saved +editions into the site, and publishing them, are later steps; for now the site +is assembled and published as above. + +### Preparing an edition + +Fetch the release tags into the lancedb and sophon checkouts, then run the +command from a checkout of this repository with no uncommitted changes to +`assemble.yaml`, `scripts/` or `docs/`: + +```bash +uv run --no-project --with pyyaml scripts/edition.py prepare enterprise 1.10.0 --lancedb ../lancedb --sophon ../sophon +uv run --no-project --with pyyaml scripts/edition.py prepare oss 0.41.0 --enterprise v1.10.0 --lancedb ../lancedb --sophon ../sophon +``` + +- **Enterprise** reads sophon's tag `v1.10.0`, which must have a + `release-qualified/v1.10.0` tag on the same commit. The shared pages come + from the LanceDB commit that release bundles at `src/lancedb`, which may be + one no OSS release tags: fetch it with `git -C ../lancedb fetch origin + `. `--bundled-lancedb ` stops preparation if the release + bundles a different one. +- **OSS** reads lancedb's tag `v0.41.0`. The Enterprise pages come from the + sophon tag or full commit hash named with `--enterprise`; a branch is not + accepted. Which Enterprise release to name is not settled yet; see + [Open questions](VERSIONING.md#open-questions). +- The lancedb and sophon roots are read from Git at their commits, never from a + working tree, and only from `docs/web`. This repository's `docs/`, + `assemble.yaml` and assembler come from HEAD. Only stable versions, such as + `1.2.3`, have editions. + +Preparation saves nothing, and says why, when a tag or commit is missing, a +release has no `docs/web`, an Enterprise release is not qualified or bundles +another LanceDB, a root holds a symlink, the Enterprise root holds a hidden +file, the assembler stops, or `.gitignore` would leave out one of the +edition's files. + +Preparing a saved release again changes nothing when its release commits match +the record, even after the edition has been corrected, and stops when they +differ. It never replaces a saved edition. + +Check the new edition as a site, check every edition against its record, and +commit the edition and its record together: + +```bash +(cd editions/enterprise/1.10.0 && mint validate && mint broken-links --check-anchors --check-redirects) +uv run --no-project --with pyyaml scripts/edition.py check +``` + +A record looks like this, with shortened commits and checksums: + +```yaml +format: 1 +product: enterprise +version: 1.10.0 +edition: editions/enterprise/1.10.0 +release: + repository: lancedb/sophon + tag: v1.10.0 + commit: 5a1f0c9e… + qualification: + tag: release-qualified/v1.10.0 + object: 9c3e27d4… + bundled_lancedb: 7d0b4a12… +sources: + lancedb: + repository: lancedb/lancedb + path: docs/web + commit: 7d0b4a12… + enterprise: + repository: lancedb/sophon + path: docs/web + commit: 5a1f0c9e… + build: + repository: lancedb/docs + path: docs + commit: e2c4b8f0… + openapi: + repository: lance-format/lance-namespace + release: v0.12.0 + path: docs/src/spec.yaml + sha256: 41ab93c7… +prepared: + files: 312 + sha256: b07d5e61… +corrections: [] +``` + +`release` and `sources` say what the edition was built from, and `prepared` is +the snapshot as first saved; neither changes afterwards. An OSS record has +`enterprise_pages`, the `--enterprise` ref and its commit, in place of +`qualification` and `bundled_lancedb`. + +### Correcting a saved edition + +A saved edition can be corrected after review, at the same URL: + +1. Edit the affected files in each edition that has the error, and in no other. +2. In each edition's record, add an entry under `corrections`. Leave the rest + of the record as it is. + + ```yaml + corrections: + - summary: Fix the default retry count in the merge-insert example. + files: [tables/merge-insert.mdx] + ``` + +3. Run the checks above. `edition.py check` fails when an edition differs from + its prepared snapshot without a correction listed. Check a corrected example + the way that example needs; a typo needs no test run. +4. If the error is still in the current source, in `lancedb/lancedb`, + `lancedb/sophon` or `docs/` here, fix it there as well. +5. Open one pull request that names every edition it corrects. + +Preparing that release again later keeps the correction. + ## Code snippets The code examples on the open-source pages are tested programs in diff --git a/VERSIONING.md b/VERSIONING.md new file mode 100644 index 00000000..d5412f84 --- /dev/null +++ b/VERSIONING.md @@ -0,0 +1,215 @@ +# Versioned documentation + +Readers should find the documentation for the product version they use. +Engineers keep writing documentation beside the product code, in +`lancedb/lancedb` and `lancedb/sophon`. Release automation prepares each +version's documentation, and people review it before it is merged and before +it reaches production. + +This is the design that work follows. It is built in four steps: + +1. **Release records and saved editions.** A record names one release's + sources, and a manual command saves the edition built from them. This step + is in place: see [Saved editions](README.md#saved-editions). +2. **Composing editions into the site:** version paths, links, navigation, + selectors, redirects and an "About this version" page for each edition. +3. **Supervised publication:** merged editions and corrections go to staging + automatically, and a person approves promoting that same candidate to + production. +4. **Release discovery:** each product's own qualification signal starts the + same preparation, and a bot opens the documentation pull request. + +Until steps 2 and 3 are in place and a cutover is approved, the site is built +and published as it is today, and production stays where it is. + +## Decisions + +| Topic | Decision | +| --- | --- | +| Hosting | Mintlify, with its native product and version selectors. | +| Default | The latest approved, published Enterprise edition. | +| Unversioned links | Resolve to the same page in that edition. Versioned links stay in their edition. | +| Navigation | Every edition keeps the full documentation set, with OSS and Enterprise badges. | +| Release cadence | OSS and Enterprise editions publish independently. An Enterprise release needs no new OSS release. | +| Enterprise sources | An Enterprise edition uses its own pages and the shared pages of the exact LanceDB it bundles, which may be an older release or a commit no release tags. | +| History | No backfill. Editions start with releases that have their documentation sources. | +| Storage | One folder per edition, all on one branch of this repository. | +| Corrections | A reviewed edit to a saved edition is allowed for a historical correction. Fix the current source too when the error is still there. | +| Automation | Preparation, checks, pull requests and staging are automated. People review before merging and approve production promotion. | +| Missing page on version switch | Mintlify's native fallback, with no custom switching logic. | + +"Latest" means the approved preferred edition of a product, not whichever job +finished last. A later maintenance release must not replace a newer preferred +release line. Selecting the latest Enterprise edition does not select the +latest OSS edition: the LanceDB it bundles can be older. + +## Components + +```mermaid +flowchart TD + L["lancedb/lancedb
Shared pages and examples"] --> B + S["lancedb/sophon
Enterprise pages and bundled LanceDB"] --> B + D["This repository
Docs-owned pages"] --> B + R["Release record
Version and exact source commits"] --> B + B["Edition builder
Saves a new edition"] --> E + E["Saved editions"] + C["Reviewed correction"] --> E + E --> A["Site composition
Version paths, links and navigation"] + N["Shared site configuration
Branding and preferred editions"] --> A + A --> V["Checked site candidate"] + V --> T["Staging"] + T --> H["Production approval"] + H --> P["Promote the same candidate"] +``` + +These are parts of the existing tooling, not new services. + +- **Edition builder** (`scripts/edition.py`) reads the recorded commits, + builds the edition with the existing assembler, and saves it. The assembler + stays where pages are composed: overlays, examples written into pages, + availability badges and native drafts all come from it. +- **Saved edition**: the pages, assets, navigation and draft controls of one + release. Later builds never fetch fresh content into it. +- **Site composition** gives each saved edition its version path and builds + the navigation and redirects. It does not rebuild editions. +- **Publisher** puts the checked candidate on staging and, after approval, the + same candidate on production. + +Global branding and Mintlify's renderer change independently of editions, so +a saved edition keeps its content, not its exact pixels. + +## Editions and their records + +```text +releases/ + oss-.yaml + enterprise-.yaml +editions/ + oss// + enterprise// +``` + +A record names the product and version, the release tag and commit, the +Enterprise qualification tag, the LanceDB an Enterprise release bundles, and +the commit of every root the assembler read, including this repository's own +pages (Geneva, the dataset cards and the REST API specification) and the +specification's upstream release. Older editions never inherit today's shared +pages. + +Saved editions are kept by default but are not immutable. A reviewed +correction changes an edition's files at the same URL. The record keeps the +original sources and the checksum of the snapshot as prepared, lists each +correction, and Git history holds the correction pull requests. Publication +checksums describe the corrected candidate. + +Preparation never overwrites a saved edition: + +- The same release prepared again from the same release commits changes + nothing, even after the edition has been corrected. A later commit of this + repository does not make it a different release. +- Different commits for a saved release, such as a moved tag or another + Enterprise source, stop preparation for review. +- A failed preparation leaves no edition and no record. + +## Preparing a release + +The manual command and later automatic discovery run the same preparation. +Nobody assembles files by hand. + +**Enterprise.** Preparation takes the `lancedb/sophon` release tag, which must +be qualified, reads the LanceDB commit that release bundles, and takes the +shared pages from that commit. A standalone OSS edition is not needed. + +**OSS.** Preparation takes the `lancedb/lancedb` release tag. The full +navigation includes the Enterprise pages, so the `lancedb/sophon` tag or commit +they come from is named explicitly. Nothing reads Sophon's `main` or infers a +compatibility relationship. + +A missing tag, absent documentation, an unqualified or inconsistently tagged +Enterprise release, or private-source files that may not be published stop +preparation with the reason. Nothing is made up from today's sources. + +The bundled LanceDB is not a client SDK compatibility matrix. Client minimum +versions belong beside the examples they apply to, when evidence supports +them, and the tested client version is not a claimed minimum. + +## Corrections + +```mermaid +flowchart LR + E["Error in a published edition"] --> F["Edit the saved files"] + F --> PR["Correction PR names the editions"] + PR --> C["Checks and review"] + C --> S["Corrected candidate on staging"] + S --> A["Approve production promotion"] + F --> U["Fix the current source too
if the error is still there"] +``` + +A correction to a saved Enterprise edition edits its folder and keeps its +version and URL. It needs no product release and no historical source branch. + +Corrections are explicit. If three editions are wrong, the pull request names +and edits all three. Checking a corrected example fits the example; a typo +needs no SDK test suite. Preparing a release again never rebuilds a corrected +edition from its original sources. + +## Reader behavior + +Site composition will provide this: + +- First-time readers land on the latest published Enterprise edition. +- Unversioned links go to the same page in that edition. Redirects come from + the actual page map and keep valid anchors; paths are not blindly prefixed. +- Versioned links stay in their edition. +- Readers can choose the newest OSS edition independently. +- Selectors use short labels. An "About this version" page shows the full + release identity and its sources. +- Version switching is Mintlify's own. In a hosted trial, switching from a + page that only a newer version has landed on another page of the older + version, without explanation. That is Mintlify's fallback, not a promise. +- Test versions and probe pages never ship as editions. + +An old unversioned URL with no matching page still needs a policy. A +catch-all redirect that hides missing content is not one. + +## Publication and discovery + +At first the preparation bot opens pull requests and does not merge them. A +merged edition or correction goes to staging. Production needs a person's +approval of the candidate that was reviewed. A stale approval cannot promote a +different candidate, and an older queued run cannot overwrite a newer +approved publication or change the preferred edition. Promotion reuses the +staged artifact. + +Missing qualification, absent sources, ambiguous inputs or failed checks stop +the workflow with a report. Duplicate events are harmless. + +Rolling back changes the preferred edition and keeps published folders. +Restoring an earlier whole-site build would remove newer editions, so it needs +a deliberate decision. + +Merging and production promotion can later become automatic, separately. +That changes approval policy, not the builder or the publisher. + +## Open questions + +These are settled where they first matter, before production: + +1. Each product's qualification signal for discovery. Enterprise releases + carry a qualification tag, which preparation already requires. For OSS, + the stable-release completion signal, beta and release-candidate handling, + and the release owner's approval are still to be confirmed. +2. Which Enterprise source supplies the Enterprise pages of an OSS edition. + Preparation takes it explicitly until this is decided. +3. Missing targets for legacy URLs. +4. Approval enforcement in the repository, and the supported Mintlify staging + setup. +5. Preferred-release order across stable and maintenance lines. + +**Enterprise pages without placeholders.** The assembler currently needs an +OSS page at the same path for every Enterprise page. Removing that +requirement, with its navigation and draft handling, is a separate change that +must land before the first real editions. Editions save whatever the +assembler builds, so they need no change for it. Once it lands, prepare test +editions again and check Enterprise-only pages, intentional overlays, +navigation and hidden drafts together. From 2a56a426612353d34032d70e23b34fa466e2fc72 Mon Sep 17 00:00:00 2001 From: Brendan Clement Date: Tue, 20 Oct 2026 11:13:54 -0700 Subject: [PATCH 04/19] fix: validate the provenance fields of release records A record that lost its docs source, carried a malformed commit or dropped an Enterprise release's qualification or bundled LanceDB still loaded, because only a few nested keys were read. Records are now checked against what `prepare` writes: every release and source field, product-specific fields, commit and checksum formats, and agreement between each source and the release field that selected it. Each problem names its record and field. The check is offline and does not require a tag for a bundled LanceDB commit. --- README.md | 5 +- scripts/edition.py | 178 ++++++++++++++++++++++++++++------ scripts/tests/test_edition.py | 161 +++++++++++++++++++++++++++++- 3 files changed, 314 insertions(+), 30 deletions(-) diff --git a/README.md b/README.md index a3e6981b..7cdb212e 100644 --- a/README.md +++ b/README.md @@ -299,7 +299,10 @@ corrections: [] `release` and `sources` say what the edition was built from, and `prepared` is the snapshot as first saved; neither changes afterwards. An OSS record has `enterprise_pages`, the `--enterprise` ref and its commit, in place of -`qualification` and `bundled_lancedb`. +`qualification` and `bundled_lancedb`. `edition.py check` refuses a record +that lacks any of these fields, or whose commits are malformed or disagree, +such as an Enterprise record whose `sources.lancedb.commit` is not its +`release.bundled_lancedb`. ### Correcting a saved edition diff --git a/scripts/edition.py b/scripts/edition.py index 3a60d242..4b7241e6 100644 --- a/scripts/edition.py +++ b/scripts/edition.py @@ -306,38 +306,158 @@ def record_path(repo: Path, product: str, version: str) -> Path: def load_record(path: Path) -> dict: - """A release record, checked enough to compare and to verify its edition.""" + """A release record that holds everything `prepare` writes.""" + name = f"releases/{path.name}" try: record = yaml.safe_load(path.read_text(encoding="utf-8")) except (OSError, yaml.YAMLError) as exc: - raise EditionError(f"{path.name}: unreadable: {exc}") from None - if not isinstance(record, dict) or record.get("format") != FORMAT: - raise EditionError(f"{path.name}: not a format {FORMAT} release record") - product, version = record.get("product"), record.get("version") - if product not in PRODUCTS or not isinstance(version, str) or not VERSION_RE.fullmatch(version): - raise EditionError(f"{path.name}: product and version are not a release") - if path.name != f"{product}-{version}.yaml": - raise EditionError(f"{path.name}: names {product} {version}") - if record.get("edition") != f"editions/{product}/{version}": - raise EditionError(f"{path.name}: edition must be editions/{product}/{version}") - prepared = record.get("prepared") - if not isinstance(prepared, dict) or not candidate.CHECKSUM_RE.fullmatch( - str(prepared.get("sha256")) - ): - raise EditionError(f"{path.name}: `prepared` has no snapshot checksum") - try: - identity(record) - except (AttributeError, KeyError, TypeError): - raise EditionError(f"{path.name}: `release` or `sources` is incomplete") from None - corrections = record.get("corrections") - if not isinstance(corrections, list) or not all( - isinstance(c, dict) and isinstance(c.get("summary"), str) and c["summary"].strip() - for c in corrections - ): - raise EditionError(f"{path.name}: every correction needs a one-line `summary`") + raise EditionError(f"{name}: unreadable: {exc}") from None + problems = record_problems(record, path.name) + if problems: + raise EditionError(f"{name}: " + "; ".join(problems)) return record +MISSING = object() + + +def record_problems(record: object, filename: str) -> list[str]: + """Each field of `record` that differs from what `prepare` writes. + + Formats are checked first; a field that disagrees with another is + reported only when both are well formed, so each problem names the field + that is wrong. Nothing is fetched: the commits are checked for form and + agreement, not against the repositories. + """ + problems: list[str] = [] + + def lookup(path: str) -> object: + value = record + for key in path.split("."): + if not isinstance(value, dict) or key not in value: + return MISSING + value = value[key] + return value + + def fields(path: str, required: tuple[str, ...], optional: tuple[str, ...] = ()) -> bool: + """Whether `path` is a mapping; reports any field it should not have.""" + value = lookup(path) if path else record + if value is MISSING: + problems.append(f"{path} is missing") + return False + if not isinstance(value, dict): + problems.append(f"{path or 'the record'} is not a mapping") + return False + for key in sorted(set(value) - set(required) - set(optional), key=str): + problems.append(f"{path + '.' if path else ''}{key} is not a field here") + return True + + def check(path: str, valid, expected: str) -> object: + """The value at `path` if `valid` accepts it, else None and a problem.""" + value = lookup(path) + if value is MISSING: + problems.append(f"{path} is missing") + elif not valid(value): + problems.append(f"{path} is {value!r}, not {expected}") + else: + return value + return None + + def commit(value: object) -> bool: + return isinstance(value, str) and COMMIT_RE.fullmatch(value) is not None + + def checksum(value: object) -> bool: + return isinstance(value, str) and candidate.CHECKSUM_RE.fullmatch(value) is not None + + def line(value: object) -> bool: + return isinstance(value, str) and bool(value.strip()) and "\n" not in value.strip() + + top = ("format", "product", "version", "edition", "release", "sources", "prepared", "corrections") + if not fields("", top): + return problems + if check("format", lambda v: v == FORMAT and not isinstance(v, bool), str(FORMAT)) is None: + return problems + product = check("product", lambda v: v in PRODUCTS, " or ".join(PRODUCTS)) + version = check( + "version", lambda v: isinstance(v, str) and VERSION_RE.fullmatch(v), "a version such as 1.2.3" + ) + if product and version: + check("edition", lambda v: v == f"editions/{product}/{version}", f"editions/{product}/{version}") + if filename != f"{product}-{version}.yaml": + problems.append(f"product and version name {product}-{version}.yaml, not {filename}") + + # The release, and the commits only it can name. + own = "lancedb" if product == "oss" else "enterprise" + extra = ("enterprise_pages",) if product == "oss" else ("qualification", "bundled_lancedb") + if product and fields("release", ("repository", "tag", "commit", *extra)): + check("release.repository", lambda v: v == SOURCES[own][0], SOURCES[own][0]) + if version: + check("release.tag", lambda v: v == f"v{version}", f"v{version}") + check("release.commit", commit, "a commit hash") + if product == "oss": + if fields("release.enterprise_pages", ("ref", "commit")): + check("release.enterprise_pages.ref", line, "the tag or commit named with --enterprise") + check("release.enterprise_pages.commit", commit, "a commit hash") + else: + if fields("release.qualification", ("tag", "object")): + if version: + tag_name = QUALIFIED_TAG.format(version=version) + check("release.qualification.tag", lambda v: v == tag_name, tag_name) + check("release.qualification.object", commit, "a tag object hash") + check("release.bundled_lancedb", commit, "a commit hash") + + # Every root the assembler read, and the spec the docs root ships. + if fields("sources", tuple(SOURCES), ("openapi",)): + for name, (repository, folder) in SOURCES.items(): + if fields(f"sources.{name}", ("repository", "path", "commit")): + check(f"sources.{name}.repository", lambda v, r=repository: v == r, repository) + check(f"sources.{name}.path", lambda v, f=folder: v == f, folder) + check(f"sources.{name}.commit", commit, "a commit hash") + if lookup("sources.openapi") is not MISSING and fields( + "sources.openapi", ("repository", "release", "path", "sha256") + ): + for key in ("repository", "release", "path"): + check(f"sources.openapi.{key}", line, "a name") + check("sources.openapi.sha256", checksum, "a SHA-256 checksum") + + # A source and the release field that selected it name the same commit. + if product: + pairs = ( + [("sources.lancedb.commit", "release.commit"), + ("sources.enterprise.commit", "release.enterprise_pages.commit")] + if product == "oss" + else [("sources.enterprise.commit", "release.commit"), + ("sources.lancedb.commit", "release.bundled_lancedb")] + ) + for source_path, release_path in pairs: + a, b = lookup(source_path), lookup(release_path) + if commit(a) and commit(b) and a != b: + problems.append(f"{source_path} is {a}, but {release_path} is {b}") + + if fields("prepared", ("files", "sha256")): + check( + "prepared.files", + lambda v: isinstance(v, int) and not isinstance(v, bool) and v > 0, + "a number of files", + ) + check("prepared.sha256", checksum, "a SHA-256 checksum") + + corrections = check("corrections", lambda v: isinstance(v, list), "a list") + for index, entry in enumerate(corrections or []): + where = f"corrections[{index}]" + if not isinstance(entry, dict): + problems.append(f"{where} is not a mapping") + continue + for key in sorted(set(entry) - {"summary", "files"}, key=str): + problems.append(f"{where}.{key} is not a field here") + if not line(entry.get("summary")): + problems.append(f"{where}.summary must be one line of text") + files = entry.get("files", []) + if not isinstance(files, list) or not all(isinstance(f, str) and f for f in files): + problems.append(f"{where}.files must be a list of the edition's paths") + return problems + + def saved_hashes(repo: Path, edition: str) -> dict[str, str]: """The edition's files as Git would save them: ignored local files, such as `.DS_Store`, are not part of it. Preparation refuses an edition that has @@ -545,11 +665,13 @@ def check(repo: Path = REPO_ROOT) -> tuple[list[str], list[str]]: """Each edition's state, and every problem with records and editions.""" lines, problems, paired = [], [], set() for path in sorted((repo / "releases").glob("*.yaml")): + # The file name says which edition is this record's, even when the + # record itself is malformed. + paired.add("editions/" + path.stem.replace("-", "/", 1)) try: record = load_record(path) if not (repo / record["edition"]).is_dir(): - raise EditionError(f"{path.name}: {record['edition']} is missing") - paired.add(record["edition"]) + raise EditionError(f"releases/{path.name}: {record['edition']} is missing") lines.append(f"{record['product']} {record['version']}: {snapshot_state(repo, record)}") except (EditionError, candidate.CandidateError) as exc: problems.append(str(exc)) diff --git a/scripts/tests/test_edition.py b/scripts/tests/test_edition.py index 32f9f512..5e0b5f3c 100644 --- a/scripts/tests/test_edition.py +++ b/scripts/tests/test_edition.py @@ -9,6 +9,7 @@ Run with `make test-assemble`. """ +import copy import hashlib import json import os @@ -207,9 +208,13 @@ def release(version: str, files: dict, qualified: str | None = "self") -> str: @pytest.fixture def docs(tmp_path) -> Path: + return docs_checkout(tmp_path.resolve() / "docs") + + +def docs_checkout(path: Path) -> Path: """A lancedb/docs checkout: this repository's assemble.yaml and .gitignore, and docs-owned pages, a navigation fragment and the OpenAPI spec.""" - repo = new_repo(tmp_path.resolve() / "docs") + repo = new_repo(path) for name in ("assemble.yaml", ".gitignore"): shutil.copy(SCRIPTS.parent / name, repo / name) files = { @@ -532,3 +537,157 @@ def run(*argv: str) -> subprocess.CompletedProcess: result = run("check") assert (result.returncode, result.stdout) == (0, "oss 0.98.0: as prepared\n") + + +# --------------------------------------------------------------------------- # +# release records keep their provenance +# --------------------------------------------------------------------------- # + + +def rewrite(docs: Path, name: str, change) -> None: + path = docs / "releases" / f"{name}.yaml" + rec = yaml.safe_load(path.read_text()) + change(rec) + path.write_text(yaml.safe_dump(rec)) + + +def test_records_without_their_pinned_sources_fail(docs, sources): + """The independent review's two cases, through `check`.""" + prepare(docs, sources, "oss", "0.98.0", "--enterprise", "v8.1.0") + prepare(docs, sources, "enterprise", "8.1.0") + + def oss(rec): + del rec["sources"]["build"] + rec["sources"]["lancedb"]["commit"] = "not-a-commit" + + def enterprise(rec): + del rec["release"]["qualification"] + del rec["release"]["bundled_lancedb"] + + rewrite(docs, "oss-0.98.0", oss) + rewrite(docs, "enterprise-8.1.0", enterprise) + + assert edition.check(docs) == ([], [ + "releases/enterprise-8.1.0.yaml: release.qualification is missing; " + "release.bundled_lancedb is missing", + "releases/oss-0.98.0.yaml: sources.lancedb.commit is 'not-a-commit', not a commit " + "hash; sources.build is missing", + ]) + + +@pytest.fixture(scope="module") +def records(tmp_path_factory, sources) -> dict[str, dict]: + """The records the builder writes, which are the contract.""" + docs = docs_checkout(tmp_path_factory.mktemp("records").resolve() / "docs") + prepare(docs, sources, "oss", "0.98.0", "--enterprise", "v8.1.0") + prepare(docs, sources, "enterprise", "8.1.0") + return {name: record(docs, name) for name in ("oss-0.98.0", "enterprise-8.1.0")} + + +def change(path: str, value=KeyError): + """Set the field at a dotted path, or delete it.""" + def apply(rec): + *parents, key = path.split(".") + for parent in parents: + rec = rec[parent] + if value is KeyError: + del rec[key] + else: + rec[key] = value + return apply + + +C = "2" * 40 + +# One change each, and the one problem it must cause. {release}, {pages}, +# {bundled} and {prepared} stand for the record's own values. +MALFORMED = [ + ("oss-0.98.0", change("format", 2), "format is 2, not 1"), + ("oss-0.98.0", change("product", "cloud"), "product is 'cloud', not oss or enterprise"), + ("oss-0.98.0", change("version", "0.98"), "version is '0.98', not a version such as 1.2.3"), + ("oss-0.98.0", change("edition", "editions/oss/0.99.0"), + "edition is 'editions/oss/0.99.0', not editions/oss/0.98.0"), + ("oss-0.98.0", change("notes", "x"), "notes is not a field here"), + ("oss-0.98.0", change("release.repository", "lancedb/sophon"), + "release.repository is 'lancedb/sophon', not lancedb/lancedb"), + ("oss-0.98.0", change("release.tag", "v0.99.0"), "release.tag is 'v0.99.0', not v0.98.0"), + ("oss-0.98.0", change("release.commit", "abc123"), "release.commit is 'abc123', not a commit hash"), + ("oss-0.98.0", change("release.commit", C), + "sources.lancedb.commit is {release}, but release.commit is " + C), + ("oss-0.98.0", change("release.enterprise_pages"), "release.enterprise_pages is missing"), + ("oss-0.98.0", change("release.enterprise_pages.ref", ""), + "release.enterprise_pages.ref is '', not the tag or commit named with --enterprise"), + ("oss-0.98.0", change("release.enterprise_pages.commit", C), + "sources.enterprise.commit is {pages}, but release.enterprise_pages.commit is " + C), + ("oss-0.98.0", change("release.bundled_lancedb", C), "release.bundled_lancedb is not a field here"), + ("oss-0.98.0", change("sources.build"), "sources.build is missing"), + ("oss-0.98.0", change("sources.build.commit", "HEAD"), "sources.build.commit is 'HEAD', not a commit hash"), + ("oss-0.98.0", change("sources.enterprise.repository", "lancedb/lancedb"), + "sources.enterprise.repository is 'lancedb/lancedb', not lancedb/sophon"), + ("oss-0.98.0", change("sources.lancedb.path", "docs"), "sources.lancedb.path is 'docs', not docs/web"), + ("oss-0.98.0", change("sources.geneva", {}), "sources.geneva is not a field here"), + ("oss-0.98.0", change("sources.openapi.sha256", "x"), + "sources.openapi.sha256 is 'x', not a SHA-256 checksum"), + ("oss-0.98.0", change("prepared"), "prepared is missing"), + ("oss-0.98.0", change("prepared.files", "9"), "prepared.files is '9', not a number of files"), + ("oss-0.98.0", change("prepared.sha256", "0" * 63), + f"prepared.sha256 is {'0' * 63!r}, not a SHA-256 checksum"), + ("oss-0.98.0", change("corrections", {}), "corrections is {{}}, not a list"), + ("oss-0.98.0", change("corrections", [{"files": ["guide.mdx"]}]), "corrections[0].summary must be one line of text"), + ("oss-0.98.0", change("corrections", [{"summary": "One\nTwo"}]), + "corrections[0].summary must be one line of text"), + ("oss-0.98.0", change("corrections", [{"summary": "Fix.", "files": "guide.mdx"}]), + "corrections[0].files must be a list of the edition's paths"), + ("oss-0.98.0", change("corrections", [{"summary": "Fix.", "approved": True}]), + "corrections[0].approved is not a field here"), + ("enterprise-8.1.0", change("release.qualification"), "release.qualification is missing"), + ("enterprise-8.1.0", change("release.qualification.tag", "release-qualified/v8.2.0"), + "release.qualification.tag is 'release-qualified/v8.2.0', not release-qualified/v8.1.0"), + ("enterprise-8.1.0", change("release.qualification.object", "annotated"), + "release.qualification.object is 'annotated', not a tag object hash"), + ("enterprise-8.1.0", change("release.bundled_lancedb"), "release.bundled_lancedb is missing"), + ("enterprise-8.1.0", change("release.bundled_lancedb", C), + "sources.lancedb.commit is {bundled}, but release.bundled_lancedb is " + C), + ("enterprise-8.1.0", change("release.commit", C), + "sources.enterprise.commit is {release}, but release.commit is " + C), + ("enterprise-8.1.0", change("release.repository", "lancedb/lancedb"), + "release.repository is 'lancedb/lancedb', not lancedb/sophon"), + ("enterprise-8.1.0", change("release.enterprise_pages", {"ref": "v8.1.0", "commit": C}), + "release.enterprise_pages is not a field here"), +] + + +@pytest.mark.parametrize( + "name, mutate, problem", MALFORMED, ids=[f"{n}:{p[:60]}" for n, _m, p in MALFORMED] +) +def test_a_malformed_record_names_the_field(records, tmp_path, name, mutate, problem): + rec = copy.deepcopy(records[name]) + values = { + "release": rec["release"]["commit"], + "pages": (rec["release"].get("enterprise_pages") or {}).get("commit"), + "bundled": rec["release"].get("bundled_lancedb"), + "prepared": rec["prepared"]["sha256"], + } + mutate(rec) + path = tmp_path / "releases" / f"{name}.yaml" + path.parent.mkdir() + path.write_text(yaml.safe_dump(rec)) + + with pytest.raises(edition.EditionError) as raised: + edition.load_record(path) + assert str(raised.value) == f"releases/{name}.yaml: " + problem.format(**values) + + +def test_records_the_builder_writes_are_valid(records, tmp_path): + """Including an Enterprise record whose bundled LanceDB no release tags, + and a record with two corrections.""" + corrected = copy.deepcopy(records["oss-0.98.0"]) + corrected["corrections"] = [ + {"summary": "Fix the guide.", "files": ["guide.mdx"]}, + {"summary": "Remove a page."}, + ] + for name, rec in [*records.items(), ("oss-0.98.0", corrected)]: + assert edition.record_problems(rec, f"{name}.yaml") == [] + assert edition.record_problems(records["enterprise-8.1.0"], "enterprise-8.2.0.yaml") == [ + "product and version name enterprise-8.1.0.yaml, not enterprise-8.2.0.yaml" + ] From 62675ccdd20920f7793ad289885765849d63ff61 Mon Sep 17 00:00:00 2001 From: Brendan Clement Date: Tue, 20 Oct 2026 11:14:19 -0700 Subject: [PATCH 05/19] fix: bind recorded corrections to the edition's checksum Once a record listed any correction, `check` accepted every later change to its edition, including an unrelated deleted page. Each correction now records the checksum of the edition it accepts, and the edition must match the latest one, or the prepared checksum when there is none. `check` prints the checksum to record, and the README's correction steps use it. Preparing a saved release again still keeps the edition and its corrections. --- AGENTS.md | 2 +- CLAUDE.md | 2 +- README.md | 59 ++++++++++------ STYLE.md | 2 +- VERSIONING.md | 23 ++++--- scripts/edition.py | 45 ++++++++---- scripts/tests/test_edition.py | 124 ++++++++++++++++++++++++++++++---- 7 files changed, 194 insertions(+), 63 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 555707c3..5989cd69 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -16,4 +16,4 @@ This is a documentation site for [LanceDB](https://docs.lancedb.com). ## Saved editions - `editions/` holds saved documentation editions and `releases/` their records. Create an edition only with `scripts/edition.py prepare`, and change one only as a reviewed correction listed in its record. See "Saved editions" in README.md. -- Write new documentation in `lancedb/lancedb` or `lancedb/sophon`, not in a saved edition. +- Write new documentation in `lancedb/lancedb` or the Enterprise documentation repository, not in a saved edition. diff --git a/CLAUDE.md b/CLAUDE.md index 555707c3..5989cd69 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -16,4 +16,4 @@ This is a documentation site for [LanceDB](https://docs.lancedb.com). ## Saved editions - `editions/` holds saved documentation editions and `releases/` their records. Create an edition only with `scripts/edition.py prepare`, and change one only as a reviewed correction listed in its record. See "Saved editions" in README.md. -- Write new documentation in `lancedb/lancedb` or `lancedb/sophon`, not in a saved edition. +- Write new documentation in `lancedb/lancedb` or the Enterprise documentation repository, not in a saved edition. diff --git a/README.md b/README.md index 7cdb212e..1d37c7b9 100644 --- a/README.md +++ b/README.md @@ -5,8 +5,8 @@ Home of the [LanceDB](https://lancedb.com/) documentation. Built using [Mintlify ## Development The published site is assembled from three roots, listed in `assemble.yaml`: -the open-source pages in `lancedb/lancedb` and the Enterprise pages in -`lancedb/sophon`, both under `docs/web`, and this repository's `docs/`. +the open-source pages in `lancedb/lancedb`, the Enterprise documentation, +both under `docs/web`, and this repository's `docs/`. This repository's `docs/` holds the pages it still owns: the Geneva pages, the dataset cards and the REST API reference. The four Geneva snippets @@ -187,11 +187,11 @@ Publishing relies on settings outside this repository: - Mintlify's Git settings: repository `lancedb/docs`, branch `assembled`, and the content directory above. Mintlify serves the branch its settings name, so publishing to `assembled` reaches production only once that is the branch. -- The LanceDB Docs Reader GitHub App installed on `lancedb/sophon` only, with +- The LanceDB Docs Reader GitHub App installed on the Enterprise documentation repository only, with Contents: read and the required Metadata: read permission. Set its Client ID in the `SOPHON_DOCS_APP_CLIENT_ID` repository variable and its PEM private key in the `SOPHON_DOCS_APP_PRIVATE_KEY` repository secret. The Assemble and Docs - Check workflows generate a short-lived token scoped to Sophon with + Check workflows generate a short-lived token scoped to that repository with Contents: read. Checkout does not persist it, and the token is revoked when the job ends. Fork pull requests receive no App secret and cannot run the combined build; the assembler guard tests still run without credentials. @@ -208,14 +208,15 @@ Each published documentation edition is saved in this repository: the site the assembler built from one product release's own sources, in `editions///`, with a release record in `releases/-.yaml` that names every source it was read from. -New documentation is still written in `lancedb/lancedb` and `lancedb/sophon`. +New documentation is still written in `lancedb/lancedb` and the Enterprise +documentation repository. [VERSIONING.md](VERSIONING.md) describes the design. Composing the saved editions into the site, and publishing them, are later steps; for now the site is assembled and published as above. ### Preparing an edition -Fetch the release tags into the lancedb and sophon checkouts, then run the +Fetch the release tags into the LanceDB and Enterprise checkouts, then run the command from a checkout of this repository with no uncommitted changes to `assemble.yaml`, `scripts/` or `docs/`: @@ -224,17 +225,17 @@ uv run --no-project --with pyyaml scripts/edition.py prepare enterprise 1.10.0 - uv run --no-project --with pyyaml scripts/edition.py prepare oss 0.41.0 --enterprise v1.10.0 --lancedb ../lancedb --sophon ../sophon ``` -- **Enterprise** reads sophon's tag `v1.10.0`, which must have a +- **Enterprise** reads the Enterprise release tag `v1.10.0`, which must have a `release-qualified/v1.10.0` tag on the same commit. The shared pages come from the LanceDB commit that release bundles at `src/lancedb`, which may be one no OSS release tags: fetch it with `git -C ../lancedb fetch origin `. `--bundled-lancedb ` stops preparation if the release bundles a different one. - **OSS** reads lancedb's tag `v0.41.0`. The Enterprise pages come from the - sophon tag or full commit hash named with `--enterprise`; a branch is not + Enterprise tag or full commit hash named with `--enterprise`; a branch is not accepted. Which Enterprise release to name is not settled yet; see [Open questions](VERSIONING.md#open-questions). -- The lancedb and sophon roots are read from Git at their commits, never from a +- The LanceDB and Enterprise roots are read from Git at their commits, never from a working tree, and only from `docs/web`. This repository's `docs/`, `assemble.yaml` and assembler come from HEAD. Only stable versions, such as `1.2.3`, have editions. @@ -257,7 +258,8 @@ commit the edition and its record together: uv run --no-project --with pyyaml scripts/edition.py check ``` -A record looks like this, with shortened commits and checksums: +A record looks like this, with a placeholder for the Enterprise repository +and shortened commits and checksums: ```yaml format: 1 @@ -265,7 +267,7 @@ product: enterprise version: 1.10.0 edition: editions/enterprise/1.10.0 release: - repository: lancedb/sophon + repository: tag: v1.10.0 commit: 5a1f0c9e… qualification: @@ -278,7 +280,7 @@ sources: path: docs/web commit: 7d0b4a12… enterprise: - repository: lancedb/sophon + repository: path: docs/web commit: 5a1f0c9e… build: @@ -309,23 +311,36 @@ such as an Enterprise record whose `sources.lancedb.commit` is not its A saved edition can be corrected after review, at the same URL: 1. Edit the affected files in each edition that has the error, and in no other. -2. In each edition's record, add an entry under `corrections`. Leave the rest - of the record as it is. +2. Run `edition.py check`. It fails, because the edition no longer matches its + record, and prints the corrected edition's checksum: + + ```text + edition: editions/enterprise/1.10.0 has changed since it was prepared; its files now have checksum 3f9c2a7e…. For a reviewed correction, add an entry under `corrections` in releases/enterprise-1.10.0.yaml with `sha256: 3f9c2a7e…` + ``` + +3. In each edition's record, add an entry at the end of `corrections` with a + one-line summary, the files it changes, and that checksum; for the first + correction, replace `corrections: []`. Leave the rest of the record as it is. ```yaml corrections: - summary: Fix the default retry count in the merge-insert example. files: [tables/merge-insert.mdx] + sha256: 3f9c2a7e… ``` -3. Run the checks above. `edition.py check` fails when an edition differs from - its prepared snapshot without a correction listed. Check a corrected example - the way that example needs; a typo needs no test run. -4. If the error is still in the current source, in `lancedb/lancedb`, - `lancedb/sophon` or `docs/` here, fix it there as well. -5. Open one pull request that names every edition it corrects. - -Preparing that release again later keeps the correction. +4. Run the checks above. `edition.py check` now passes: each edition must match + the checksum its record last accepted, the prepared one or the latest + correction's. Any later change to the edition, including a deleted file, + fails again until it is recorded the same way, as a further entry. Check a + corrected example the way that example needs; a typo needs no test run. +5. If the error is still in the current source, in `lancedb/lancedb`, + the Enterprise documentation repository or `docs/` here, fix it there as well. +6. Open one pull request that names every edition it corrects. Reviewers decide + whether each correction is right; the checksum shows only that the record + describes the files. + +Preparing that release again later keeps the edition and its corrections. ## Code snippets diff --git a/STYLE.md b/STYLE.md index e2a2599e..144f0f22 100644 --- a/STYLE.md +++ b/STYLE.md @@ -5,7 +5,7 @@ Be direct, approachable and technically precise. Give readers enough context to make a decision, then show them how to act on it. This guide covers prose on docs.lancedb.com: the pages in `lancedb/lancedb` -`docs/web`, the public Enterprise pages in `lancedb/sophon` `docs/web`, and the +`docs/web`, the public Enterprise pages under `docs/web`, and the Geneva, dataset and REST pages in this repository. [WRITING.md](WRITING.md) covers Mintlify mechanics. diff --git a/VERSIONING.md b/VERSIONING.md index d5412f84..f4df7d99 100644 --- a/VERSIONING.md +++ b/VERSIONING.md @@ -2,9 +2,9 @@ Readers should find the documentation for the product version they use. Engineers keep writing documentation beside the product code, in -`lancedb/lancedb` and `lancedb/sophon`. Release automation prepares each -version's documentation, and people review it before it is merged and before -it reaches production. +`lancedb/lancedb` and the Enterprise documentation repository. Release +automation prepares each version's documentation, and people review it before +it is merged and before it reaches production. This is the design that work follows. It is built in four steps: @@ -48,7 +48,7 @@ latest OSS edition: the LanceDB it bundles can be older. ```mermaid flowchart TD L["lancedb/lancedb
Shared pages and examples"] --> B - S["lancedb/sophon
Enterprise pages and bundled LanceDB"] --> B + S["Enterprise source
Enterprise pages and bundled LanceDB"] --> B D["This repository
Docs-owned pages"] --> B R["Release record
Version and exact source commits"] --> B B["Edition builder
Saves a new edition"] --> E @@ -98,9 +98,10 @@ pages. Saved editions are kept by default but are not immutable. A reviewed correction changes an edition's files at the same URL. The record keeps the -original sources and the checksum of the snapshot as prepared, lists each -correction, and Git history holds the correction pull requests. Publication -checksums describe the corrected candidate. +original sources and the checksum of the snapshot as prepared, and lists each +correction with the checksum of the edition it accepted; the saved edition +must match the latest one. Git history holds the correction pull requests. +Publication checksums describe the corrected candidate. Preparation never overwrites a saved edition: @@ -116,14 +117,14 @@ Preparation never overwrites a saved edition: The manual command and later automatic discovery run the same preparation. Nobody assembles files by hand. -**Enterprise.** Preparation takes the `lancedb/sophon` release tag, which must +**Enterprise.** Preparation takes the Enterprise release tag, which must be qualified, reads the LanceDB commit that release bundles, and takes the shared pages from that commit. A standalone OSS edition is not needed. **OSS.** Preparation takes the `lancedb/lancedb` release tag. The full -navigation includes the Enterprise pages, so the `lancedb/sophon` tag or commit -they come from is named explicitly. Nothing reads Sophon's `main` or infers a -compatibility relationship. +navigation includes the Enterprise pages, so the Enterprise tag or commit +they come from is named explicitly. Nothing reads the Enterprise repository's +`main` or infers a compatibility relationship. A missing tag, absent documentation, an unqualified or inconsistently tagged Enterprise release, or private-source files that may not be published stop diff --git a/scripts/edition.py b/scripts/edition.py index 4b7241e6..675fbb87 100644 --- a/scripts/edition.py +++ b/scripts/edition.py @@ -14,9 +14,10 @@ prepare enterprise VERSION the lancedb/sophon tag vVERSION, which must be qualified, with the shared pages from the LanceDB commit that release bundles - check pair every record with its edition, and require - an edition changed since it was prepared to - list its corrections + check check every record, pair it with its edition, + and require the edition to match the checksum + the record last accepted: the prepared one, or + its latest correction's Each root is read from Git objects at its commit, never from a working tree, and only from its documentation folder: `docs/web` of lancedb/lancedb and @@ -78,7 +79,8 @@ # Release record, written by scripts/edition.py prepare. `release` and `sources` # say what the edition was built from, and `prepared` describes the snapshot as # first saved. Neither changes afterwards. A reviewed correction to the saved -# edition adds an entry to `corrections`; see VERSIONING.md. +# edition adds an entry to `corrections` with the checksum of the corrected +# edition; see "Saved editions" in README.md. """ @@ -443,18 +445,29 @@ def line(value: object) -> bool: check("prepared.sha256", checksum, "a SHA-256 checksum") corrections = check("corrections", lambda v: isinstance(v, list), "a list") + accepted = lookup("prepared.sha256") for index, entry in enumerate(corrections or []): where = f"corrections[{index}]" if not isinstance(entry, dict): problems.append(f"{where} is not a mapping") + accepted = None continue - for key in sorted(set(entry) - {"summary", "files"}, key=str): + for key in sorted(set(entry) - {"summary", "files", "sha256"}, key=str): problems.append(f"{where}.{key} is not a field here") if not line(entry.get("summary")): problems.append(f"{where}.summary must be one line of text") files = entry.get("files", []) if not isinstance(files, list) or not all(isinstance(f, str) and f for f in files): problems.append(f"{where}.files must be a list of the edition's paths") + sha256 = entry.get("sha256") + if not checksum(sha256): + problems.append( + f"{where}.sha256 is {'missing' if sha256 is None else repr(sha256)}, not the " + "SHA-256 checksum of the corrected edition that `check` reports" + ) + elif sha256 == accepted: + problems.append(f"{where}.sha256 is the checksum before it; a correction changes the edition") + accepted = sha256 return problems @@ -483,16 +496,21 @@ def ignored_paths(repo: Path, listing: str, index: bool = False) -> set[str]: def snapshot_state(repo: Path, record: dict) -> str: - """Whether the saved edition is still the prepared snapshot.""" + """Whether the saved edition is the snapshot its record last accepted: + the one prepared, or the one its latest correction recorded.""" current = candidate.site_checksum(saved_hashes(repo, record["edition"])) - if current == record["prepared"]["sha256"]: - return "as prepared" - count = len(record["corrections"]) - if not count: + corrections = record["corrections"] + accepted = corrections[-1]["sha256"] if corrections else record["prepared"]["sha256"] + if current != accepted: + since = "its latest recorded correction" if corrections else "it was prepared" raise EditionError( - f"{record['edition']} differs from its prepared snapshot, and " - f"releases/{record['product']}-{record['version']}.yaml lists no correction" + f"{record['edition']} has changed since {since}; its files now have checksum " + f"{current}. For a reviewed correction, add an entry under `corrections` in " + f"releases/{record['product']}-{record['version']}.yaml with `sha256: {current}`" ) + if not corrections: + return "as prepared" + count = len(corrections) return f"corrected ({count} correction{'s' if count > 1 else ''})" @@ -718,7 +736,8 @@ def main(argv: list[str] | None = None) -> int: except (EditionError, candidate.CandidateError) as exc: print(f"edition: {exc}", file=sys.stderr) return 1 - print("\n".join(lines) or "no saved editions") + if lines or not problems: + print("\n".join(lines) or "no saved editions") for problem in problems: print(f"edition: {problem}", file=sys.stderr) return 1 if problems else 0 diff --git a/scripts/tests/test_edition.py b/scripts/tests/test_edition.py index 5e0b5f3c..73d1f9fe 100644 --- a/scripts/tests/test_edition.py +++ b/scripts/tests/test_edition.py @@ -13,6 +13,7 @@ import hashlib import json import os +import re import shutil import subprocess import sys @@ -271,14 +272,28 @@ def state(docs: Path) -> dict[str, bytes | None]: return found -def correct(docs: Path, name: str, page: str, text: str) -> None: - """A reviewed correction: edit the saved page and say so in the record.""" +def correct(docs: Path, name: str, page: str, text: str) -> str: + """A reviewed correction, as the README describes it: edit the saved page, + then record it.""" product, version = name.split("-") (docs / "editions" / product / version / page).write_text(text) + return record_correction(docs, name, page) + + +def record_correction(docs: Path, name: str, page: str) -> str: + """Add a correction entry with the checksum `check` reports for the edition.""" + product, version = name.split("-") + _lines, problems = edition.check(docs) + reported = [p for p in problems if p.startswith(f"editions/{product}/{version} has changed")] + sha256 = re.search(r"`sha256: ([0-9a-f]{64})`", reported[0])[1] + add_correction(docs, name, page, sha256) + return sha256 + + +def add_correction(docs: Path, name: str, page: str, sha256: str) -> None: path = docs / "releases" / f"{name}.yaml" - path.write_text(path.read_text().replace( - "corrections: []", f"corrections:\n- summary: Fix {page}.\n files: [{page}]" - )) + entry = f"- summary: Fix {page}.\n files: [{page}]\n sha256: {sha256}\n" + path.write_text(path.read_text().replace("corrections: []\n", "corrections:\n") + entry) # --------------------------------------------------------------------------- # @@ -500,7 +515,7 @@ def test_check_pairs_records_and_requires_corrections_to_be_listed(docs, sources (docs / "editions/oss/0.98.0/guide.mdx").write_text("Edited without a record\n") lines, problems = edition.check(docs) - assert lines == [] and "lists no correction" in problems[0] + assert lines == [] and "has changed since it was prepared" in problems[0] correct(docs, "oss-0.98.0", "guide.mdx", "Corrected guide\n") (docs / "editions/enterprise/8.1.0").mkdir(parents=True) @@ -538,6 +553,76 @@ def run(*argv: str) -> subprocess.CompletedProcess: result = run("check") assert (result.returncode, result.stdout) == (0, "oss 0.98.0: as prepared\n") + (docs / "editions/oss/0.98.0/index.mdx").write_text("A correction\n") + result = run("check") + assert (result.returncode, result.stdout) == (1, "") + assert re.fullmatch( + r"edition: editions/oss/0\.98\.0 has changed since it was prepared; its files now have " + r"checksum ([0-9a-f]{64})\. For a reviewed correction, add an entry under `corrections` in " + r"releases/oss-0\.98\.0\.yaml with `sha256: \1`\n", + result.stderr, + ) + + +# --------------------------------------------------------------------------- # +# corrections are bound to the edition's checksum +# --------------------------------------------------------------------------- # + + +def test_a_later_unrecorded_change_fails_after_a_recorded_correction(docs, sources): + """The independent review's case: a recorded correction to guide.mdx, then + an unrelated page deleted without updating the record.""" + prepare(docs, sources, "oss", "0.98.0", "--enterprise", "v8.1.0") + folder = docs / "editions/oss/0.98.0" + (folder / "guide.mdx").write_text("Reviewed correction\n") + add_correction(docs, "oss-0.98.0", "guide.mdx", checksum(saved(folder))) + assert edition.check(docs) == (["oss 0.98.0: corrected (1 correction)"], []) + (folder / "enterprise/security.mdx").unlink() + + lines, problems = edition.check(docs) + + assert lines == [] + assert len(problems) == 1 + assert problems[0].startswith( + "editions/oss/0.98.0 has changed since its latest recorded correction; its files now " + "have checksum " + ) + + +def test_each_correction_records_the_snapshot_it_accepts(docs, sources): + prepare(docs, sources, "oss", "0.98.0", "--enterprise", "v8.1.0") + prepared = record(docs, "oss-0.98.0")["prepared"] + first = correct(docs, "oss-0.98.0", "guide.mdx", "Corrected guide\n") + assert edition.check(docs) == (["oss 0.98.0: corrected (1 correction)"], []) + + # Later changes nobody recorded: an edit, then a deletion. + page = docs / "editions/oss/0.98.0/index.mdx" + page.write_text("Unreviewed edit\n") + assert "has changed since its latest recorded correction" in edition.check(docs)[1][0] + page.unlink() + assert "has changed since its latest recorded correction" in edition.check(docs)[1][0] + + # Recording the deletion, once reviewed, as a second correction accepts it. + second = record_correction(docs, "oss-0.98.0", "index.mdx") + assert edition.check(docs) == (["oss 0.98.0: corrected (2 corrections)"], []) + rec = record(docs, "oss-0.98.0") + assert rec["prepared"] == prepared + assert [c["sha256"] for c in rec["corrections"]] == [first, second] + assert second == checksum(saved(docs / "editions/oss/0.98.0")) + + # Preparing the release again keeps the edition and both corrections. + before = state(docs) + out = prepare(docs, sources, "oss", "0.98.0", "--enterprise", "v8.1.0") + assert out.endswith("editions/oss/0.98.0 is corrected (2 corrections); nothing changed") + assert state(docs) == before + + # It stays a no-op when a change has not been recorded, and says so. + (docs / "editions/oss/0.98.0/guide.mdx").write_text("Another unreviewed edit\n") + before = state(docs) + with pytest.raises(edition.EditionError, match="already saved; nothing changed, but .* has changed"): + prepare(docs, sources, "oss", "0.98.0", "--enterprise", "v8.1.0") + assert state(docs) == before + # --------------------------------------------------------------------------- # # release records keep their provenance @@ -597,7 +682,7 @@ def apply(rec): return apply -C = "2" * 40 +A, B, C = "a" * 64, "b" * 64, "2" * 40 # One change each, and the one problem it must cause. {release}, {pages}, # {bundled} and {prepared} stand for the record's own values. @@ -633,12 +718,19 @@ def apply(rec): ("oss-0.98.0", change("prepared.sha256", "0" * 63), f"prepared.sha256 is {'0' * 63!r}, not a SHA-256 checksum"), ("oss-0.98.0", change("corrections", {}), "corrections is {{}}, not a list"), - ("oss-0.98.0", change("corrections", [{"files": ["guide.mdx"]}]), "corrections[0].summary must be one line of text"), - ("oss-0.98.0", change("corrections", [{"summary": "One\nTwo"}]), + ("oss-0.98.0", change("corrections", [{"summary": "Fix.", "files": ["guide.mdx"]}]), + "corrections[0].sha256 is missing, not the SHA-256 checksum of the corrected edition that " + "`check` reports"), + ("oss-0.98.0", change("corrections", [{"summary": "Fix.", "sha256": "{prepared}"}]), + "corrections[0].sha256 is the checksum before it; a correction changes the edition"), + ("oss-0.98.0", change("corrections", [{"summary": "Fix.", "sha256": A}, {"summary": "Again.", "sha256": A}]), + "corrections[1].sha256 is the checksum before it; a correction changes the edition"), + ("oss-0.98.0", change("corrections", [{"sha256": A}]), "corrections[0].summary must be one line of text"), + ("oss-0.98.0", change("corrections", [{"summary": "One\nTwo", "sha256": A}]), "corrections[0].summary must be one line of text"), - ("oss-0.98.0", change("corrections", [{"summary": "Fix.", "files": "guide.mdx"}]), + ("oss-0.98.0", change("corrections", [{"summary": "Fix.", "files": "guide.mdx", "sha256": A}]), "corrections[0].files must be a list of the edition's paths"), - ("oss-0.98.0", change("corrections", [{"summary": "Fix.", "approved": True}]), + ("oss-0.98.0", change("corrections", [{"summary": "Fix.", "sha256": A, "approved": True}]), "corrections[0].approved is not a field here"), ("enterprise-8.1.0", change("release.qualification"), "release.qualification is missing"), ("enterprise-8.1.0", change("release.qualification.tag", "release-qualified/v8.2.0"), @@ -658,7 +750,8 @@ def apply(rec): @pytest.mark.parametrize( - "name, mutate, problem", MALFORMED, ids=[f"{n}:{p[:60]}" for n, _m, p in MALFORMED] + "name, mutate, problem", MALFORMED, + ids=[f"{n}:{p.replace('{{}}', '{}')[:60]}" for n, _m, p in MALFORMED], ) def test_a_malformed_record_names_the_field(records, tmp_path, name, mutate, problem): rec = copy.deepcopy(records[name]) @@ -669,6 +762,9 @@ def test_a_malformed_record_names_the_field(records, tmp_path, name, mutate, pro "prepared": rec["prepared"]["sha256"], } mutate(rec) + for entry in rec.get("corrections") if isinstance(rec.get("corrections"), list) else []: + if entry.get("sha256") == "{prepared}": + entry["sha256"] = values["prepared"] path = tmp_path / "releases" / f"{name}.yaml" path.parent.mkdir() path.write_text(yaml.safe_dump(rec)) @@ -683,8 +779,8 @@ def test_records_the_builder_writes_are_valid(records, tmp_path): and a record with two corrections.""" corrected = copy.deepcopy(records["oss-0.98.0"]) corrected["corrections"] = [ - {"summary": "Fix the guide.", "files": ["guide.mdx"]}, - {"summary": "Remove a page."}, + {"summary": "Fix the guide.", "files": ["guide.mdx"], "sha256": A}, + {"summary": "Remove a page.", "sha256": B}, ] for name, rec in [*records.items(), ("oss-0.98.0", corrected)]: assert edition.record_problems(rec, f"{name}.yaml") == [] From 8d7c9ac794b990de825a2fcd4918d99a89e94e5b Mon Sep 17 00:00:00 2001 From: Brendan Clement Date: Tue, 20 Oct 2026 13:47:28 -0700 Subject: [PATCH 06/19] feat: check saved editions against the pull request's base `check` compared each edition only with the record beside it, so an edition rewritten together with its prepared checksum, an earlier correction edited or removed, or an edition deleted with its record all passed. `check --base COMMIT` also compares the editions and records on that commit: each must remain, every record field but `corrections` must be unchanged, and existing corrections must stay as they were, with new ones only after them. Records are compared as parsed values, so formatting does not matter, and new editions are checked as before. The Assemble workflow's test job runs it offline, without credentials: a pull request against the base commit its event names, and a push against the commit the branch pointed at before it, each fetched by hash. A manual run checks every edition against its record only. --- .github/workflows/assemble.yml | 23 +++++- Makefile | 8 +- scripts/edition.py | 92 +++++++++++++++++++-- scripts/tests/test_edition.py | 147 ++++++++++++++++++++++++++++++++- 4 files changed, 257 insertions(+), 13 deletions(-) diff --git a/.github/workflows/assemble.yml b/.github/workflows/assemble.yml index ec664eeb..0024e48b 100644 --- a/.github/workflows/assemble.yml +++ b/.github/workflows/assemble.yml @@ -29,7 +29,8 @@ env: SOPHON_DOCS_ROOT: sophon/docs/web jobs: - # Forks run the guard tests without private-source credentials. + # Forks run the guard tests and the saved-edition check without + # private-source credentials. test: name: Test the assembler runs-on: ubuntu-latest @@ -42,6 +43,26 @@ jobs: - name: Test the overlay, navigation and private-root guards run: make test-assemble + # A pull request is compared with the base commit its event names, and a + # push with the commit the branch pointed at before it. That commit is + # fetched by hash, so the comparison never depends on a remote-tracking + # branch or on what the shallow checkout happens to hold. A manual run + # has no base and checks each edition against its record only. + - name: Check that saved editions and their records are kept + env: + EVENT: ${{ github.event_name }} + # Not BASE itself: make would read it from the environment. + BASE_SHA: ${{ github.event_name == 'pull_request' && github.event.pull_request.base.sha || github.event_name == 'push' && github.event.before || '' }} + run: | + set -euo pipefail + if [ -z "$BASE_SHA" ] || [ "$BASE_SHA" = 0000000000000000000000000000000000000000 ]; then + echo "No base commit for this $EVENT event; checking each edition against its record only." + make check-editions + else + git fetch --no-tags --depth=1 origin "$BASE_SHA" + make check-editions BASE="$BASE_SHA" + fi + assemble: runs-on: ubuntu-latest steps: diff --git a/Makefile b/Makefile index 667dc691..bfcae4b2 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,7 @@ ASSEMBLE_SCRIPT := scripts/assemble.py ASSEMBLE_RUN := uv run --no-project --with pyyaml # uv run automatically handles virtualenv, so no activation needed -.PHONY: py ts rs snippets hf-sync assemble test-assemble check-spec sync-spec +.PHONY: py ts rs snippets hf-sync assemble test-assemble check-editions check-spec sync-spec # Snippets are generated in lancedb/lancedb, beside the tests they come from; # these targets only say so. See "Code snippets" in README.md. @@ -32,6 +32,12 @@ assemble: test-assemble: @$(ASSEMBLE_RUN) --with pytest pytest scripts/tests -q +# Check every saved edition against its release record. With BASE set to a +# commit, also check that the editions and records on it are kept. See "Saved +# editions" in README.md. +check-editions: + @$(ASSEMBLE_RUN) scripts/edition.py check $(if $(BASE),--base $(BASE)) + # Fail if the tracked OpenAPI spec has drifted from the release pinned in # assemble.yaml. Run in CI so the pin cannot rot unnoticed. check-spec: diff --git a/scripts/edition.py b/scripts/edition.py index 675fbb87..7c2a43c5 100644 --- a/scripts/edition.py +++ b/scripts/edition.py @@ -18,6 +18,9 @@ and require the edition to match the checksum the record last accepted: the prepared one, or its latest correction's + check --base COMMIT also require every edition and record on + COMMIT to remain, each record as it was there + but for corrections added at the end Each root is read from Git objects at its commit, never from a working tree, and only from its documentation folder: `docs/web` of lancedb/lancedb and @@ -27,14 +30,16 @@ A saved edition is never replaced. Preparing a release that already has one changes nothing when the release inputs match its record, even after the edition has been corrected, and stops when they differ. A preparation that -fails leaves neither an edition nor a record behind. +fails leaves neither an edition nor a record behind. Pull requests run +`check --base` against their base commit, so a merged edition and its record +stay as they are, apart from recorded corrections. Usage: python scripts/edition.py prepare oss VERSION --enterprise REF [--lancedb REPO] [--sophon REPO] python scripts/edition.py prepare enterprise VERSION [--bundled-lancedb COMMIT] [--lancedb REPO] [--sophon REPO] - python scripts/edition.py check + python scripts/edition.py check [--base COMMIT] """ from __future__ import annotations @@ -78,9 +83,10 @@ HEADER = """\ # Release record, written by scripts/edition.py prepare. `release` and `sources` # say what the edition was built from, and `prepared` describes the snapshot as -# first saved. Neither changes afterwards. A reviewed correction to the saved -# edition adds an entry to `corrections` with the checksum of the corrected -# edition; see "Saved editions" in README.md. +# first saved. Once merged, neither changes; `edition.py check --base` refuses +# a change to them. A reviewed correction to the saved edition adds an entry at +# the end of `corrections` with the checksum of the corrected edition; see +# "Saved editions" in README.md. """ @@ -679,8 +685,71 @@ def prepare(args: argparse.Namespace, repo: Path = REPO_ROOT) -> str: # --------------------------------------------------------------------------- # -def check(repo: Path = REPO_ROOT) -> tuple[list[str], list[str]]: - """Each edition's state, and every problem with records and editions.""" +def differences(was: object, now: object, path: str) -> list[str]: + """How two parsed values differ, field by field.""" + if isinstance(was, dict) and isinstance(now, dict): + return [ + change + for key in sorted(set(was) | set(now), key=str) + for change in differences( + was.get(key, MISSING), now.get(key, MISSING), f"{path}.{key}" if path else str(key) + ) + ] + if type(was) is type(now) and was == now: + return [] + return [f"{path} {'was removed' if now is MISSING else 'was added' if was is MISSING else 'changed'}"] + + +def preserved(repo: Path, base: str) -> list[str]: + """What has happened since `base` to the editions and records it has. + + Each of them remains. A record keeps every field it had there, except that + its corrections may gain entries at the end. Records are compared as + parsed values, so a change of formatting is not a change. + """ + problems = [] + listing = git(repo, "ls-tree", "-z", "--name-only", base, "--", "releases/").decode() + for rel in sorted(name for name in listing.split("\0") if name.endswith(".yaml")): + path = repo / rel + if not path.is_file(): + problems.append(f"{rel} is on the base {base} but has been removed; a saved release keeps its record") + continue + try: + was = yaml.safe_load(git(repo, "show", f"{base}:{rel}")) + now = yaml.safe_load(path.read_text(encoding="utf-8")) + except (OSError, yaml.YAMLError): + was = now = None + if not isinstance(was, dict) or not isinstance(now, dict): + # `check` reports a malformed record by itself, and one on the base + # can only be repaired by changing it. + continue + changes = differences( + {k: v for k, v in was.items() if k != "corrections"}, + {k: v for k, v in now.items() if k != "corrections"}, + "", + ) + before, after = was.get("corrections"), now.get("corrections") + after = after if isinstance(after, list) else [] + for index, entry in enumerate(before if isinstance(before, list) else []): + changes += differences(entry, after[index] if index < len(after) else MISSING, f"corrections[{index}]") + if changes: + problems.append( + f"{rel} differs from the base {base}: {', '.join(changes)}; a saved release's " + "record only gains corrections, at the end" + ) + listing = git(repo, "ls-tree", "-z", "-d", "--name-only", base, "--", + *(f"editions/{product}/" for product in PRODUCTS)).decode() + for rel in sorted(name for name in listing.split("\0") if name): + _editions, product, version = rel.split("/") + # A missing edition whose record remains is reported with the record. + if not (repo / rel).is_dir() and not record_path(repo, product, version).is_file(): + problems.append(f"{rel} is on the base {base} but has been removed; a saved edition is never removed") + return problems + + +def check(repo: Path = REPO_ROOT, base: str | None = None) -> tuple[list[str], list[str]]: + """Each edition's state, and every problem with records and editions; with + `base`, also every change since that commit to the ones it has.""" lines, problems, paired = [], [], set() for path in sorted((repo / "releases").glob("*.yaml")): # The file name says which edition is this record's, even when the @@ -705,6 +774,8 @@ def check(repo: Path = REPO_ROOT) -> tuple[list[str], list[str]]: problems.append(f"{rel} is not a product folder") elif rel not in paired: problems.append(f"{rel} has no release record") + if base: + problems += preserved(repo, commit_of(repo, base, "--base")) return lines, problems @@ -722,7 +793,10 @@ def parse(argv: list[str] | None = None) -> argparse.Namespace: help="a lancedb/lancedb repository (default: ../lancedb)") prep.add_argument("--sophon", type=Path, default=REPO_ROOT.parent / "sophon", help="a lancedb/sophon repository (default: ../sophon)") - commands.add_parser("check", help="check the saved editions against their records") + checking = commands.add_parser("check", help="check the saved editions against their records") + checking.add_argument("--base", metavar="COMMIT", + help="also require the editions and records on COMMIT, such as a pull " + "request's base, to be kept") return parser.parse_args(argv) @@ -732,7 +806,7 @@ def main(argv: list[str] | None = None) -> int: if args.command == "prepare": print(prepare(args)) return 0 - lines, problems = check() + lines, problems = check(base=args.base) except (EditionError, candidate.CandidateError) as exc: print(f"edition: {exc}", file=sys.stderr) return 1 diff --git a/scripts/tests/test_edition.py b/scripts/tests/test_edition.py index 73d1f9fe..c3329a0c 100644 --- a/scripts/tests/test_edition.py +++ b/scripts/tests/test_edition.py @@ -296,6 +296,12 @@ def add_correction(docs: Path, name: str, page: str, sha256: str) -> None: path.write_text(path.read_text().replace("corrections: []\n", "corrections:\n") + entry) +def commit_all(docs: Path, message: str) -> str: + git(docs, "add", "-A") + git(docs, "commit", "-q", "-m", message) + return git(docs, "rev-parse", "HEAD") + + # --------------------------------------------------------------------------- # # preparing editions # --------------------------------------------------------------------------- # @@ -549,19 +555,33 @@ def run(*argv: str) -> subprocess.CompletedProcess: assert result.returncode == 1 assert "commit or stash the changes" in result.stderr assert not (docs / "editions/oss/0.99.0").exists() + git(docs, "checkout", "--", "scripts/assemble.py") result = run("check") assert (result.returncode, result.stdout) == (0, "oss 0.98.0: as prepared\n") + base = commit_all(docs, "oss 0.98.0") + assert run("check", "--base", base).returncode == 0 (docs / "editions/oss/0.98.0/index.mdx").write_text("A correction\n") result = run("check") assert (result.returncode, result.stdout) == (1, "") - assert re.fullmatch( + reported = re.fullmatch( r"edition: editions/oss/0\.98\.0 has changed since it was prepared; its files now have " r"checksum ([0-9a-f]{64})\. For a reviewed correction, add an entry under `corrections` in " r"releases/oss-0\.98\.0\.yaml with `sha256: \1`\n", result.stderr, - ) + )[1] + + # The prepared checksum rewritten to match: only the base shows the change. + rewrite(docs, "oss-0.98.0", change("prepared.sha256", reported)) + assert run("check").returncode == 0 + result = run("check", "--base", base) + assert (result.returncode, result.stdout) == (1, "oss 0.98.0: as prepared\n") + assert result.stderr == f"edition: {kept(base, 'releases/oss-0.98.0.yaml', 'prepared.sha256 changed')}\n" + + result = run("check", "--base", "1" * 40) + assert (result.returncode, result.stdout) == (1, "") + assert result.stderr.startswith(f"edition: --base: {'1' * 40} is not a commit in ") # --------------------------------------------------------------------------- # @@ -624,6 +644,129 @@ def test_each_correction_records_the_snapshot_it_accepts(docs, sources): assert state(docs) == before +# --------------------------------------------------------------------------- # +# merged editions and records are kept +# --------------------------------------------------------------------------- # + + +@pytest.fixture +def base(docs, sources) -> str: + """Main after OSS 0.98.0 and Enterprise 8.1.0 were merged, and then a + correction to OSS 0.98.0: the base a pull request is checked against.""" + prepare(docs, sources, "oss", "0.98.0", "--enterprise", "v8.1.0") + prepare(docs, sources, "enterprise", "8.1.0") + commit_all(docs, "editions") + correct(docs, "oss-0.98.0", "guide.mdx", "Corrected guide\n") + return commit_all(docs, "a correction") + + +KEPT = ["enterprise 8.1.0: as prepared", "oss 0.98.0: corrected (1 correction)"] + + +def kept(base: str, rel: str, *changes: str) -> str: + """The problem `check --base` reports for a record changed since `base`.""" + return ( + f"{rel} differs from the base {base}: {', '.join(changes)}; a saved release's " + "record only gains corrections, at the end" + ) + + +def test_unchanged_editions_and_a_new_one_pass_against_the_base(docs, sources, base): + assert edition.check(docs, base=base) == (KEPT, []) + + # A new edition, and a record written again with other formatting and no + # header comment. + prepare(docs, sources, "oss", "0.99.0", "--enterprise", "v8.2.0") + rewrite(docs, "enterprise-8.1.0", lambda rec: None) + assert edition.check(docs, base=base) == ([*KEPT, "oss 0.99.0: as prepared"], []) + + # A new edition is checked against its own record, as always. + (docs / "editions/oss/0.99.0/index.mdx").write_text("Edited without a record\n") + lines, problems = edition.check(docs, base=base) + assert lines == KEPT + assert len(problems) == 1 + assert problems[0].startswith("editions/oss/0.99.0 has changed since it was prepared") + + +def test_prepared_rewritten_with_the_snapshot_fails_against_the_base(docs, base): + folder = docs / "editions/enterprise/8.1.0" + (folder / "index.mdx").write_text("Edited\n") + (folder / "guide.mdx").unlink() + files = saved(folder) + rewrite(docs, "enterprise-8.1.0", change("prepared", {"files": len(files), "sha256": checksum(files)})) + + # The edition matches its record, so on its own the change passes. + assert edition.check(docs) == (KEPT, []) + assert edition.check(docs, base=base) == (KEPT, [ + kept(base, "releases/enterprise-8.1.0.yaml", "prepared.files changed", "prepared.sha256 changed"), + ]) + + +def test_editing_or_removing_a_recorded_correction_fails_against_the_base(docs, base): + folder = docs / "editions/oss/0.98.0" + # A page deleted, and the existing correction's checksum rewritten to match. + (folder / "enterprise/security.mdx").unlink() + + def rewritten(rec): + rec["corrections"][0]["sha256"] = checksum(saved(folder)) + + rewrite(docs, "oss-0.98.0", rewritten) + assert edition.check(docs) == (KEPT, []) + assert edition.check(docs, base=base) == (KEPT, [ + kept(base, "releases/oss-0.98.0.yaml", "corrections[0].sha256 changed"), + ]) + + # The correction undone and its entry removed: the edition is as prepared. + git(docs, "reset", "-q", "--hard", base) + git(docs, "checkout", "HEAD~1", "--", "editions/oss/0.98.0/guide.mdx") + rewrite(docs, "oss-0.98.0", change("corrections", [])) + lines = [KEPT[0], "oss 0.98.0: as prepared"] + assert edition.check(docs) == (lines, []) + assert edition.check(docs, base=base) == (lines, [ + kept(base, "releases/oss-0.98.0.yaml", "corrections[0] was removed"), + ]) + + +def test_removing_or_replacing_a_saved_release_fails_against_the_base(docs, sources, base): + shutil.rmtree(docs / "editions/enterprise/8.1.0") + (docs / "releases/enterprise-8.1.0.yaml").unlink() + assert edition.check(docs) == ([KEPT[1]], []) + assert edition.check(docs, base=base) == ([KEPT[1]], [ + f"releases/enterprise-8.1.0.yaml is on the base {base} but has been removed; a saved " + "release keeps its record", + f"editions/enterprise/8.1.0 is on the base {base} but has been removed; a saved edition " + "is never removed", + ]) + + # Only the edition removed: reported once, with its record. + git(docs, "reset", "-q", "--hard", base) + shutil.rmtree(docs / "editions/enterprise/8.1.0") + assert edition.check(docs, base=base) == ([KEPT[1]], [ + "releases/enterprise-8.1.0.yaml: editions/enterprise/8.1.0 is missing", + ]) + + # Removed with its record, then prepared again from other inputs. + git(docs, "reset", "-q", "--hard", base) + shutil.rmtree(docs / "editions/oss/0.98.0") + (docs / "releases/oss-0.98.0.yaml").unlink() + prepare(docs, sources, "oss", "0.98.0", "--enterprise", "v8.2.0") + assert edition.check(docs) == ([KEPT[0], "oss 0.98.0: as prepared"], []) + problems = edition.check(docs, base=base)[1] + assert len(problems) == 1 + assert problems[0].startswith(f"releases/oss-0.98.0.yaml differs from the base {base}: ") + for item in ("prepared.sha256 changed", "release.enterprise_pages.commit changed", + "sources.enterprise.commit changed", "corrections[0] was removed"): + assert item in problems[0] + + +def test_an_appended_correction_passes_against_the_base(docs, base): + rec = record(docs, "oss-0.98.0") + correct(docs, "oss-0.98.0", "index.mdx", "Corrected index\n") + + assert edition.check(docs, base=base) == ([KEPT[0], "oss 0.98.0: corrected (2 corrections)"], []) + assert record(docs, "oss-0.98.0")["corrections"][0] == rec["corrections"][0] + + # --------------------------------------------------------------------------- # # release records keep their provenance # --------------------------------------------------------------------------- # From 7f27b0776329c25d7623d6e7e28c00dd6f0e1a11 Mon Sep 17 00:00:00 2001 From: Brendan Clement Date: Tue, 20 Oct 2026 13:50:06 -0700 Subject: [PATCH 07/19] fix: build editions from a commit of docs main The record named this checkout's HEAD as the source of the docs-owned pages, assemble.yaml and the assembler. On a pull request's branch that commit gets a new hash when the branch is squash merged, and a second edition prepared after the first was committed named the unmerged edition commit. `docs/` and assemble.yaml are now read from Git at origin/main, or at an earlier commit of main named with `--docs`, and that is the commit the record names. Preparation stops when the commit is not on origin/main, or when this checkout's scripts/, the assembler that runs, are not that commit's. Preparing a saved release again is still a no-op that keeps its corrections. --- scripts/edition.py | 60 +++++++++++++++---- scripts/tests/test_edition.py | 106 +++++++++++++++++++++++++++++----- 2 files changed, 140 insertions(+), 26 deletions(-) diff --git a/scripts/edition.py b/scripts/edition.py index 7c2a43c5..89a04db9 100644 --- a/scripts/edition.py +++ b/scripts/edition.py @@ -24,8 +24,11 @@ Each root is read from Git objects at its commit, never from a working tree, and only from its documentation folder: `docs/web` of lancedb/lancedb and -lancedb/sophon, and `docs` of this repository at HEAD, which also supplies -assemble.yaml and the assembler. +lancedb/sophon, and `docs` of this repository at a commit of its main branch, +origin/main or one named with --docs, which also supplies assemble.yaml. A +branch's commits get new hashes when it is merged, so a record names only a +commit of main. The assembler that runs is this checkout's scripts/, which +must be that commit's. A saved edition is never replaced. Preparing a release that already has one changes nothing when the release inputs match its record, even after the @@ -37,8 +40,9 @@ Usage: python scripts/edition.py prepare oss VERSION --enterprise REF [--lancedb REPO] [--sophon REPO] + [--docs REF] python scripts/edition.py prepare enterprise VERSION [--bundled-lancedb COMMIT] - [--lancedb REPO] [--sophon REPO] + [--lancedb REPO] [--sophon REPO] [--docs REF] python scripts/edition.py check [--base COMMIT] """ @@ -74,6 +78,9 @@ "enterprise": ("lancedb/sophon", "docs/web"), "build": ("lancedb/docs", "docs"), } +# This repository's main as fetched. Squash and rebase merges give a branch's +# commits new hashes, so only a commit of main stays recoverable. +DOCS_MAIN = "origin/main" # An Enterprise release bundles LanceDB as this gitlink, and a release that # passed qualification has an annotated tag of this name on its commit. BUNDLED_LANCEDB = "src/lancedb" @@ -203,15 +210,39 @@ def source(name: str, commit: str) -> dict: return {"repository": repository, "path": folder, "commit": commit} -def docs_commit(repo: Path) -> str: - """This repository's HEAD, which must hold everything the build reads.""" - changed = text(repo, "status", "--porcelain", "--", "assemble.yaml", "scripts", SOURCES["build"][1]) +def is_ancestor(repo: Path, commit: str, of: str) -> bool: + result = subprocess.run( + ["git", "-C", str(repo), "merge-base", "--is-ancestor", commit, of], capture_output=True + ) + if result.returncode not in (0, 1): + raise EditionError(f"git merge-base in {repo} failed: {result.stderr.decode(errors='replace').strip()}") + return result.returncode == 0 + + +def docs_commit(repo: Path, ref: str) -> str: + """The commit of this repository's main that the build reads `docs/` and + assemble.yaml from. The assembler is this checkout's own scripts/, so they + must be that commit's too.""" + what = f"{SOURCES['build'][0]} --docs" + commit = commit_of(repo, ref, what) + if not is_ancestor(repo, commit, commit_of(repo, DOCS_MAIN, SOURCES["build"][0])): + raise EditionError( + f"{what}: {ref} is {commit}, which {DOCS_MAIN} does not contain; name a commit " + "of main, because merging gives a branch's commits new hashes" + ) + changed = text(repo, "status", "--porcelain", "--", "scripts") if changed: raise EditionError( - "commit or stash the changes to assemble.yaml, scripts/ and docs/ first; " - "the record names this checkout's HEAD:\n" + changed + "commit or stash the changes to scripts/ first; they are the assembler that " + "builds the edition:\n" + changed + ) + if text(repo, "ls-tree", "-d", "HEAD", "--", "scripts") != text(repo, "ls-tree", "-d", commit, "--", "scripts"): + raise EditionError( + f"this checkout's scripts/ are not those of {commit}, which the record would name; " + "check out a branch whose scripts/ match it, or name the commit of main they come " + "from with --docs" ) - return commit_of(repo, "HEAD", "lancedb/docs") + return commit def resolve_release(args: argparse.Namespace, repo: Path) -> dict: @@ -282,7 +313,7 @@ def resolve_release(args: argparse.Namespace, repo: Path) -> dict: "sources": { "lancedb": source("lancedb", shared), "enterprise": source("enterprise", enterprise), - "build": source("build", docs_commit(repo)), + "build": source("build", docs_commit(repo, args.docs)), }, } @@ -291,7 +322,8 @@ def identity(record: dict) -> dict: """What makes two preparations the same release: commits, not ref names. This repository's own commit is left out: re-running a release from a later - HEAD is still the same release, and its saved edition is kept as it is. + commit of main is still the same release, and its saved edition is kept as + it is. """ release = record["release"] return { @@ -641,7 +673,8 @@ def prepare(args: argparse.Namespace, repo: Path = REPO_ROOT) -> str: source_repo = {"lancedb": args.lancedb, "enterprise": args.sophon}.get(name, repo) export(source_repo.resolve(), commit, folder, roots[name], f"{repository} {commit}") site = staging / "site" - raw = yaml.safe_load(git(repo, "show", "HEAD:assemble.yaml").decode()) + build = requested["sources"]["build"]["commit"] + raw = yaml.safe_load(git(repo, "show", f"{build}:assemble.yaml").decode()) config = assemble.load_config(build_config(raw, roots, site, staging / "assemble.yaml")) _count, warnings = assemble.assemble_site(config) for warning in warnings: @@ -793,6 +826,9 @@ def parse(argv: list[str] | None = None) -> argparse.Namespace: help="a lancedb/lancedb repository (default: ../lancedb)") prep.add_argument("--sophon", type=Path, default=REPO_ROOT.parent / "sophon", help="a lancedb/sophon repository (default: ../sophon)") + prep.add_argument("--docs", metavar="REF", default=DOCS_MAIN, + help="the commit of this repository's main to read docs/ and assemble.yaml " + f"from (default: {DOCS_MAIN})") checking = commands.add_parser("check", help="check the saved editions against their records") checking.add_argument("--base", metavar="COMMIT", help="also require the editions and records on COMMIT, such as a pull " diff --git a/scripts/tests/test_edition.py b/scripts/tests/test_edition.py index c3329a0c..a34fbd23 100644 --- a/scripts/tests/test_edition.py +++ b/scripts/tests/test_edition.py @@ -213,8 +213,9 @@ def docs(tmp_path) -> Path: def docs_checkout(path: Path) -> Path: - """A lancedb/docs checkout: this repository's assemble.yaml and .gitignore, - and docs-owned pages, a navigation fragment and the OpenAPI spec.""" + """A lancedb/docs checkout of main, fetched as origin/main: this + repository's assemble.yaml and .gitignore, and docs-owned pages, a + navigation fragment and the OpenAPI spec.""" repo = new_repo(path) for name in ("assemble.yaml", ".gitignore"): shutil.copy(SCRIPTS.parent / name, repo / name) @@ -228,9 +229,16 @@ def docs_checkout(path: Path) -> Path: (repo / rel).write_text(content) git(repo, "add", "-A") git(repo, "commit", "-q", "-m", "docs") + on_main(repo) return repo +def on_main(docs: Path) -> str: + """HEAD merged into main and fetched: origin/main points at it.""" + git(docs, "update-ref", "refs/remotes/origin/main", "HEAD") + return git(docs, "rev-parse", "HEAD") + + def prepare(docs: Path, sources: dict, *argv: str, lancedb: Path | None = None) -> str: args = edition.parse([ "prepare", *argv, @@ -340,7 +348,7 @@ def test_oss_edition_is_built_from_its_tag_and_named_enterprise_pages(docs, sour assert rec["sources"] == { "lancedb": {"repository": "lancedb/lancedb", "path": "docs/web", "commit": oss["0.98.0"]}, "enterprise": {"repository": "lancedb/sophon", "path": "docs/web", "commit": ent["8.1.0"]}, - "build": {"repository": "lancedb/docs", "path": "docs", "commit": git(docs, "rev-parse", "HEAD")}, + "build": {"repository": "lancedb/docs", "path": "docs", "commit": git(docs, "rev-parse", "origin/main")}, "openapi": { "repository": "lance-format/lance-namespace", "release": yaml.safe_load((docs / "assemble.yaml").read_text())["openapi"]["release"], @@ -410,6 +418,76 @@ def test_preparation_stops_and_saves_nothing(docs, sources, argv, error): assert state(docs) == {} +# --------------------------------------------------------------------------- # +# the docs input commit +# --------------------------------------------------------------------------- # + + +def test_docs_inputs_come_from_main_not_from_the_branch(docs, sources): + main = git(docs, "rev-parse", "origin/main") + spec = yaml.safe_load((docs / "assemble.yaml").read_text())["openapi"]["release"] + # A pull request's branch changes a docs-owned page and assemble.yaml in a + # commit main does not have, and the page again without committing it. + git(docs, "switch", "-q", "-c", "editions") + (docs / "docs/geneva/index.mdx").write_text("# Geneva, on a branch\n") + config = (docs / "assemble.yaml").read_text() + (docs / "assemble.yaml").write_text(config.replace(f"release: {spec}\n", "release: v9.9.9\n")) + commit_all(docs, "unmerged changes") + (docs / "docs/geneva/index.mdx").write_text("# Geneva, uncommitted\n") + + prepare(docs, sources, "oss", "0.98.0", "--enterprise", "v8.1.0") + # The first edition is committed on the branch before a second is prepared. + git(docs, "add", "editions", "releases") + git(docs, "commit", "-q", "-m", "oss 0.98.0") + prepare(docs, sources, "enterprise", "8.1.0") + + for name in ("oss-0.98.0", "enterprise-8.1.0"): + rec = record(docs, name) + assert rec["sources"]["build"]["commit"] == main + assert rec["sources"]["openapi"]["release"] == spec + assert (docs / rec["edition"] / "geneva/index.mdx").read_text() == "# Geneva\n" + + +def test_an_earlier_commit_of_main_can_be_named(docs, sources): + earlier = git(docs, "rev-parse", "HEAD") + (docs / "docs/geneva/index.mdx").write_text("# Geneva, later\n") + commit_all(docs, "a later page") + later = on_main(docs) + + prepare(docs, sources, "oss", "0.98.0", "--enterprise", "v8.1.0", "--docs", earlier) + prepare(docs, sources, "oss", "0.99.0", "--enterprise", "v8.2.0") + + for name, commit, page in (("oss-0.98.0", earlier, "# Geneva\n"), ("oss-0.99.0", later, "# Geneva, later\n")): + rec = record(docs, name) + assert rec["sources"]["build"]["commit"] == commit + assert (docs / rec["edition"] / "geneva/index.mdx").read_text() == page + + +def test_docs_inputs_off_main_or_unlike_the_running_scripts_are_refused(docs, sources): + git(docs, "switch", "-q", "-c", "editions") + (docs / "docs/geneva/index.mdx").write_text("# Geneva, on a branch\n") + head = commit_all(docs, "an unmerged page") + oss = ["oss", "0.98.0", "--enterprise", "v8.1.0"] + + with pytest.raises(edition.EditionError, match=f"--docs: HEAD is {head}, which origin/main does not contain"): + prepare(docs, sources, *oss, "--docs", "HEAD") + + # The branch changes the assembler, committed and then not. + (docs / "scripts").mkdir() + (docs / "scripts/assemble.py").write_text("# a changed assembler\n") + commit_all(docs, "an unmerged assembler") + with pytest.raises(edition.EditionError, match="this checkout's scripts/ are not those of"): + prepare(docs, sources, *oss) + (docs / "scripts/assemble.py").write_text("# changed again\n") + with pytest.raises(edition.EditionError, match="commit or stash the changes to scripts/"): + prepare(docs, sources, *oss) + + git(docs, "update-ref", "-d", "refs/remotes/origin/main") + with pytest.raises(edition.EditionError, match="origin/main is not a commit in .*; fetch it first"): + prepare(docs, sources, *oss) + assert state(docs) == {} + + # --------------------------------------------------------------------------- # # protecting saved editions # --------------------------------------------------------------------------- # @@ -438,10 +516,11 @@ def test_duplicate_preparation_changes_nothing_even_after_a_correction(docs, sou assert "editions/oss/0.98.0 is as prepared; nothing changed" in out correct(docs, "oss-0.98.0", "guide.mdx", "Corrected guide\n") - # This repository moves on, and the same release is prepared again, its - # Enterprise pages named by commit this time. + # This repository's main moves on, and the same release is prepared + # again, its Enterprise pages named by commit this time. (docs / "docs/geneva/index.mdx").write_text("# Geneva, later\n") git(docs, "commit", "-q", "-am", "later") + on_main(docs) before = state(docs) out = prepare(docs, sources, "oss", "0.98.0", "--enterprise", sources["enterprise"]["8.1.0"]) @@ -530,13 +609,14 @@ def test_check_pairs_records_and_requires_corrections_to_be_listed(docs, sources assert problems == ["editions/enterprise/8.1.0 has no release record"] -def test_command_records_only_committed_build_inputs(docs, sources): +def test_command_prepares_from_main_and_checks_against_the_base(docs, sources): """End to end, as a person runs it: the scripts are part of the checkout.""" (docs / "scripts").mkdir() for name in ("assemble.py", "candidate.py", "edition.py"): shutil.copy(SCRIPTS / name, docs / "scripts" / name) git(docs, "add", "-A") git(docs, "commit", "-q", "-m", "scripts") + main = on_main(docs) def run(*argv: str) -> subprocess.CompletedProcess: return subprocess.run( @@ -547,13 +627,13 @@ def run(*argv: str) -> subprocess.CompletedProcess: repos = ["--lancedb", str(sources["lancedb"]), "--sophon", str(sources["sophon"])] result = run("prepare", "oss", "0.98.0", "--enterprise", "v8.1.0", *repos) assert result.returncode == 0, result.stderr - assert record(docs, "oss-0.98.0")["sources"]["build"]["commit"] == git(docs, "rev-parse", "HEAD") + assert record(docs, "oss-0.98.0")["sources"]["build"]["commit"] == main with (docs / "scripts/assemble.py").open("a") as script: script.write("# a local change\n") result = run("prepare", "oss", "0.99.0", "--enterprise", "v8.2.0", *repos) assert result.returncode == 1 - assert "commit or stash the changes" in result.stderr + assert "commit or stash the changes to scripts/" in result.stderr assert not (docs / "editions/oss/0.99.0").exists() git(docs, "checkout", "--", "scripts/assemble.py") @@ -751,12 +831,10 @@ def test_removing_or_replacing_a_saved_release_fails_against_the_base(docs, sour (docs / "releases/oss-0.98.0.yaml").unlink() prepare(docs, sources, "oss", "0.98.0", "--enterprise", "v8.2.0") assert edition.check(docs) == ([KEPT[0], "oss 0.98.0: as prepared"], []) - problems = edition.check(docs, base=base)[1] - assert len(problems) == 1 - assert problems[0].startswith(f"releases/oss-0.98.0.yaml differs from the base {base}: ") - for item in ("prepared.sha256 changed", "release.enterprise_pages.commit changed", - "sources.enterprise.commit changed", "corrections[0] was removed"): - assert item in problems[0] + assert edition.check(docs, base=base)[1] == [kept( + base, "releases/oss-0.98.0.yaml", "prepared.sha256 changed", "release.enterprise_pages.commit changed", + "release.enterprise_pages.ref changed", "sources.enterprise.commit changed", "corrections[0] was removed", + )] def test_an_appended_correction_passes_against_the_base(docs, base): From ef18e9c776e310849e2a02fa31e8ef85a3576d7c Mon Sep 17 00:00:00 2001 From: Brendan Clement Date: Tue, 20 Oct 2026 13:51:25 -0700 Subject: [PATCH 08/19] fix: require a full hash for --bundled-lancedb, and test the guards A short hash of the right commit was reported as a different bundled LanceDB; `--bundled-lancedb` now asks for the full hash. Tests now cover guards that had none: a lightweight qualification tag, a root in assemble.yaml without recorded provenance, Git control files in an edition and a folder in editions/ that is not a product's. Two test docstrings describe their cases rather than where the cases came from. --- scripts/edition.py | 5 ++++ scripts/tests/test_edition.py | 49 +++++++++++++++++++++++++++++------ 2 files changed, 46 insertions(+), 8 deletions(-) diff --git a/scripts/edition.py b/scripts/edition.py index 89a04db9..0a6da08e 100644 --- a/scripts/edition.py +++ b/scripts/edition.py @@ -274,6 +274,11 @@ def resolve_release(args: argparse.Namespace, repo: Path) -> dict: raise EditionError( "--enterprise applies to OSS releases; an Enterprise release reads its own pages" ) + if args.bundled_lancedb and not COMMIT_RE.fullmatch(args.bundled_lancedb): + raise EditionError( + f"--bundled-lancedb takes the full 40-character commit hash the release " + f"bundles, not {args.bundled_lancedb!r}" + ) release_tag = f"v{version}" what = f"lancedb/sophon {release_tag}" enterprise = tag(sophon, release_tag, what)[1] diff --git a/scripts/tests/test_edition.py b/scripts/tests/test_edition.py index a34fbd23..a42ef20c 100644 --- a/scripts/tests/test_edition.py +++ b/scripts/tests/test_edition.py @@ -1,8 +1,8 @@ """Tests for preparing saved editions from release records. Disposable repositories stand in for lancedb/lancedb, lancedb/sophon and this -repository. Their versions are fixtures, not releases: OSS 0.95.0 to 0.99.0 and -Enterprise 8.1.0 to 8.9.0, each built to exercise one case. The docs fixture +repository. Their versions are fixtures, not releases: OSS 0.94.0 to 0.99.0 and +Enterprise 8.1.0 to 8.10.0, each built to exercise one case. The docs fixture uses this repository's own assemble.yaml and .gitignore, and every edition is built by the real assembler. @@ -173,6 +173,9 @@ def sources(tmp_path_factory) -> dict: c["bundled"] = snapshot(lancedb, oss_pages("the bundled revision"), "bundled by 8.1") c["ignored"] = snapshot(lancedb, {**oss_pages("0.95.0"), "docs/web/scratch/notes.mdx": b"x\n"}, "0.95") git(lancedb, "tag", "v0.95.0", c["ignored"]) + control = {"docs/web/.gitattributes": b"* -text\n", "docs/web/static/.gitignore": b"*.svg\n"} + c["control"] = snapshot(lancedb, {**oss_pages("0.94.0"), **control}, "0.94") + git(lancedb, "tag", "v0.94.0", c["control"]) c["0.98.0"] = snapshot(lancedb, oss_pages("0.98.0"), "0.98") annotated(lancedb, "v0.98.0", c["0.98.0"]) c["0.99.0"] = snapshot(lancedb, oss_pages("0.99.0"), "0.99") @@ -188,7 +191,9 @@ def sources(tmp_path_factory) -> dict: def release(version: str, files: dict, qualified: str | None = "self") -> str: commit = snapshot(sophon, files, version) annotated(sophon, f"v{version}", commit) - if qualified: + if qualified == "lightweight": + git(sophon, "tag", f"release-qualified/v{version}", commit) + elif qualified: annotated(sophon, f"release-qualified/v{version}", commit if qualified == "self" else qualified) e[version] = commit return commit @@ -202,7 +207,7 @@ def release(version: str, files: dict, qualified: str | None = "self") -> str: release("8.7.0", enterprise_pages("8.7", c["0.99.0"], **{"docs/web/.notes.mdx": INTERNAL})) release("8.8.0", enterprise_pages("8.8", c["0.99.0"], **{"docs/web/enterprise/ci.md": Symlink("../../ci.md")})) release("8.9.0", enterprise_pages("8.9", "1" * 40)) - git(sophon, "tag", "v8.1.0-lightweight", e["8.1.0"]) + release("8.10.0", enterprise_pages("8.10", c["0.99.0"]), qualified="lightweight") e["main"] = snapshot(sophon, enterprise_pages("unreleased main", c["main"]), "main moves on") return {"lancedb": lancedb, "sophon": sophon, "oss": c, "enterprise": e} @@ -400,9 +405,12 @@ def test_enterprise_edition_reads_the_lancedb_it_bundles(docs, sources): (["oss", "0.98.0", "--enterprise", "main"], "not a branch"), (["oss", "0.98.0", "--enterprise", "v8.1.0", "--bundled-lancedb", "1" * 40], "Enterprise releases only"), (["oss", "0.95.0", "--enterprise", "v8.1.0"], ".gitignore would leave out"), + (["oss", "0.94.0", "--enterprise", "v8.1.0"], + r"would hold \['.gitattributes', 'static/.gitignore'\], which change how Git stores it"), (["enterprise", "8.1.0", "--enterprise", "v8.2.0"], "applies to OSS releases"), (["enterprise", "8.3.0"], "is not qualified"), (["enterprise", "8.4.0"], "but release-qualified/v8.4.0 qualifies"), + (["enterprise", "8.10.0"], "release-qualified/v8.10.0 is not an annotated tag"), (["enterprise", "8.5.0"], "has no docs/web"), (["enterprise", "8.6.0"], "records no bundled LanceDB"), (["enterprise", "8.1.0", "--bundled-lancedb", "1" * 40], "disagrees with lancedb/sophon v8.1.0"), @@ -418,6 +426,26 @@ def test_preparation_stops_and_saves_nothing(docs, sources, argv, error): assert state(docs) == {} +def test_bundled_lancedb_is_named_in_full(docs, sources): + short = sources["oss"]["bundled"][:8] + with pytest.raises(edition.EditionError, match=f"takes the full 40-character commit hash .*, not '{short}'"): + prepare(docs, sources, "enterprise", "8.1.0", "--bundled-lancedb", short) + assert state(docs) == {} + + +def test_a_root_without_recorded_provenance_stops_preparation(docs, sources): + config = yaml.safe_load((docs / "assemble.yaml").read_text()) + config["roots"].append({"name": "notes", "path": "notes", "role": "reference"}) + (docs / "assemble.yaml").write_text(yaml.safe_dump(config, sort_keys=False)) + commit_all(docs, "another root") + on_main(docs) + + with pytest.raises(edition.EditionError, match=r"assemble.yaml has roots \['lancedb', 'enterprise', " + r"'build', 'notes'\], but edition.py records the sources of"): + prepare(docs, sources, "oss", "0.98.0", "--enterprise", "v8.1.0") + assert state(docs) == {} + + # --------------------------------------------------------------------------- # # the docs input commit # --------------------------------------------------------------------------- # @@ -604,9 +632,13 @@ def test_check_pairs_records_and_requires_corrections_to_be_listed(docs, sources correct(docs, "oss-0.98.0", "guide.mdx", "Corrected guide\n") (docs / "editions/enterprise/8.1.0").mkdir(parents=True) + (docs / "editions/cloud").mkdir() lines, problems = edition.check(docs) assert lines == ["oss 0.98.0: corrected (1 correction)"] - assert problems == ["editions/enterprise/8.1.0 has no release record"] + assert problems == [ + "editions/cloud is not a product folder", + "editions/enterprise/8.1.0 has no release record", + ] def test_command_prepares_from_main_and_checks_against_the_base(docs, sources): @@ -670,8 +702,8 @@ def run(*argv: str) -> subprocess.CompletedProcess: def test_a_later_unrecorded_change_fails_after_a_recorded_correction(docs, sources): - """The independent review's case: a recorded correction to guide.mdx, then - an unrelated page deleted without updating the record.""" + """A recorded correction to guide.mdx, then an unrelated page deleted + without updating the record.""" prepare(docs, sources, "oss", "0.98.0", "--enterprise", "v8.1.0") folder = docs / "editions/oss/0.98.0" (folder / "guide.mdx").write_text("Reviewed correction\n") @@ -858,7 +890,8 @@ def rewrite(docs: Path, name: str, change) -> None: def test_records_without_their_pinned_sources_fail(docs, sources): - """The independent review's two cases, through `check`.""" + """An OSS record without its docs source and with a malformed commit, and + an Enterprise record without its qualification and bundled LanceDB.""" prepare(docs, sources, "oss", "0.98.0", "--enterprise", "v8.1.0") prepare(docs, sources, "enterprise", "8.1.0") From cb967eb912aff4ab530b99c9fc3098150cac110a Mon Sep 17 00:00:00 2001 From: Brendan Clement Date: Tue, 20 Oct 2026 13:52:54 -0700 Subject: [PATCH 09/19] docs: explain where edition content comes from and what check enforces Product pages come from the release commits, and the docs-owned pages, dataset cards and REST specification from the commit of main that preparation read; both are fixed when the edition is prepared. The README gives the updated preparation and correction commands, lists the rules `check` enforces, including the base comparison the pull request check runs, and marks the sample record as illustrative. A recreated qualification tag is described as new evidence rather than a duplicate, and naming an OSS edition's Enterprise source as pinning, not approving, it. --- README.md | 129 +++++++++++++++++++++++++++++++++++--------------- VERSIONING.md | 53 ++++++++++++++++----- 2 files changed, 130 insertions(+), 52 deletions(-) diff --git a/README.md b/README.md index 1d37c7b9..2a8775bd 100644 --- a/README.md +++ b/README.md @@ -194,7 +194,8 @@ Publishing relies on settings outside this repository: Check workflows generate a short-lived token scoped to that repository with Contents: read. Checkout does not persist it, and the token is revoked when the job ends. Fork pull requests receive no App secret and cannot run the - combined build; the assembler guard tests still run without credentials. + combined build; the assembler guard tests and the saved-edition check still + run without credentials. Of these, the Publish workflow checks only that production runs from `main`, that the environment supplies the deploy key and that `MINTLIFY_CONTENT_DIR` is @@ -214,52 +215,77 @@ documentation repository. editions into the site, and publishing them, are later steps; for now the site is assembled and published as above. +An edition's content comes from two kinds of commit, and its record names both: + +- **Product pages** come from the release: an OSS edition's from its + `lancedb/lancedb` release tag, and an Enterprise edition's from its + Enterprise release tag and the LanceDB commit that release bundles. +- **Docs-owned content**, the Geneva pages, the dataset cards and the REST API + specification, comes from the docs input commit: the commit of this + repository's `main` that preparation read, recorded as `sources.build.commit`. + It is `main` as it was when the edition was prepared, not when the product + was released. + +Both are fixed when the edition is prepared. Later changes to `main`, or to a +product's sources, never reach a saved edition; only a reviewed correction +changes one. + ### Preparing an edition -Fetch the release tags into the LanceDB and Enterprise checkouts, then run the -command from a checkout of this repository with no uncommitted changes to -`assemble.yaml`, `scripts/` or `docs/`: +Fetch the release tags into the LanceDB and Enterprise checkouts, and this +repository's `main`, then run the command from a checkout of this repository +whose `scripts/` are those of `origin/main`: ```bash +git fetch origin main uv run --no-project --with pyyaml scripts/edition.py prepare enterprise 1.10.0 --lancedb ../lancedb --sophon ../sophon uv run --no-project --with pyyaml scripts/edition.py prepare oss 0.41.0 --enterprise v1.10.0 --lancedb ../lancedb --sophon ../sophon ``` -- **Enterprise** reads the Enterprise release tag `v1.10.0`, which must have a - `release-qualified/v1.10.0` tag on the same commit. The shared pages come - from the LanceDB commit that release bundles at `src/lancedb`, which may be - one no OSS release tags: fetch it with `git -C ../lancedb fetch origin - `. `--bundled-lancedb ` stops preparation if the release - bundles a different one. +- **Enterprise** reads the Enterprise release tag `v1.10.0`, which must have an + annotated `release-qualified/v1.10.0` tag on the same commit. The shared + pages come from the LanceDB commit that release bundles at `src/lancedb`, + which may be one no OSS release tags: fetch it with `git -C ../lancedb fetch + origin `. `--bundled-lancedb `, given the full 40-character + hash, stops preparation if the release bundles a different one. - **OSS** reads lancedb's tag `v0.41.0`. The Enterprise pages come from the - Enterprise tag or full commit hash named with `--enterprise`; a branch is not - accepted. Which Enterprise release to name is not settled yet; see - [Open questions](VERSIONING.md#open-questions). -- The LanceDB and Enterprise roots are read from Git at their commits, never from a - working tree, and only from `docs/web`. This repository's `docs/`, - `assemble.yaml` and assembler come from HEAD. Only stable versions, such as - `1.2.3`, have editions. + Enterprise tag or full commit hash named with `--enterprise`; a branch name + is not accepted, so the record pins an exact commit. Pinning a commit does + not decide that it may be published with that OSS release, which is not + settled yet; see [Open questions](VERSIONING.md#open-questions). +- The LanceDB and Enterprise roots are read from Git at their commits, never + from a working tree, and only from `docs/web`. +- This repository's `docs/` and `assemble.yaml` are read from Git at + `origin/main`, or at an earlier commit of `main` named with `--docs`, never + from your branch or working tree. Squash merges give a branch's commits new + hashes, so a record names only a commit of `main`, and editions prepared one + after another in the same pull request do not depend on its unmerged + commits. The assembler that runs is your checkout's `scripts/`, so they must + be that commit's. +- Only stable versions, such as `1.2.3`, have editions. Preparation saves nothing, and says why, when a tag or commit is missing, a release has no `docs/web`, an Enterprise release is not qualified or bundles another LanceDB, a root holds a symlink, the Enterprise root holds a hidden -file, the assembler stops, or `.gitignore` would leave out one of the -edition's files. +file, the assembler stops, `.gitignore` would leave out one of the edition's +files, the docs input commit is not on `origin/main`, or your `scripts/` are +not that commit's. -Preparing a saved release again changes nothing when its release commits match -the record, even after the edition has been corrected, and stops when they -differ. It never replaces a saved edition. +Preparing a saved release again changes nothing when its release commits and +qualification tag match the record, even after the edition has been corrected, +and stops when they differ. It never replaces a saved edition. -Check the new edition as a site, check every edition against its record, and -commit the edition and its record together: +Check the new edition as a site, check every edition against its record and +against `main`, and commit the edition and its record together: ```bash (cd editions/enterprise/1.10.0 && mint validate && mint broken-links --check-anchors --check-redirects) -uv run --no-project --with pyyaml scripts/edition.py check +uv run --no-project --with pyyaml scripts/edition.py check --base origin/main ``` -A record looks like this, with a placeholder for the Enterprise repository -and shortened commits and checksums: +A record looks like this. It is illustrative: the Enterprise repository is a +placeholder and the commits and checksums are shortened, so `check` would +refuse it as written. ```yaml format: 1 @@ -299,12 +325,34 @@ corrections: [] ``` `release` and `sources` say what the edition was built from, and `prepared` is -the snapshot as first saved; neither changes afterwards. An OSS record has -`enterprise_pages`, the `--enterprise` ref and its commit, in place of -`qualification` and `bundled_lancedb`. `edition.py check` refuses a record -that lacks any of these fields, or whose commits are malformed or disagree, -such as an Enterprise record whose `sources.lancedb.commit` is not its -`release.bundled_lancedb`. +the snapshot as first saved. An OSS record has `enterprise_pages`, the +`--enterprise` ref and its commit, in place of `qualification` and +`bundled_lancedb`. `sources.openapi` names the specification release that +`assemble.yaml` pinned at the docs input commit, and the checksum of the copy +saved in the edition. + +### What `check` enforces + +`edition.py check` works offline, on this checkout: + +- Every record has the fields `prepare` writes, in their formats, and each + source commit agrees with the release field that selected it, such as an + Enterprise record's `sources.lancedb.commit` and `release.bundled_lancedb`. +- Every record has its edition, and every edition its record. +- Each edition matches the checksum its record last accepted: + `prepared.sha256`, or the latest correction's `sha256`. + +With `--base COMMIT`, it also compares the editions and records already on +that commit, by their parsed values rather than their formatting. Each must +still be there. Every field of a record except `corrections` must be +unchanged, and its existing corrections must be unchanged and in order, with +new ones only after them. A new edition is checked by the rules above. + +The test job of the Assemble workflow runs `make check-editions` with a base, +offline and without credentials: for a pull request, the base commit its event +names, and for a push to `main`, the commit `main` pointed at before. It +fetches that commit by hash. A manual run of the workflow has no base, and +checks each edition against its record only. ### Correcting a saved edition @@ -320,7 +368,8 @@ A saved edition can be corrected after review, at the same URL: 3. In each edition's record, add an entry at the end of `corrections` with a one-line summary, the files it changes, and that checksum; for the first - correction, replace `corrections: []`. Leave the rest of the record as it is. + correction, replace `corrections: []`. Leave the rest of the record, and + any earlier corrections, as they are. ```yaml corrections: @@ -329,11 +378,13 @@ A saved edition can be corrected after review, at the same URL: sha256: 3f9c2a7e… ``` -4. Run the checks above. `edition.py check` now passes: each edition must match - the checksum its record last accepted, the prepared one or the latest - correction's. Any later change to the edition, including a deleted file, - fails again until it is recorded the same way, as a further entry. Check a - corrected example the way that example needs; a typo needs no test run. +4. Run the checks above, with `--base origin/main` after fetching `main`. + `edition.py check` now passes: each edition matches the checksum of its + latest correction. Any later change to the edition, including a deleted + file, fails again until it is recorded the same way, as a further entry. + `--base` refuses a change to an earlier entry or to the rest of the + record, as the pull request's check does. Check a corrected example the way + that example needs; a typo needs no test run. 5. If the error is still in the current source, in `lancedb/lancedb`, the Enterprise documentation repository or `docs/` here, fix it there as well. 6. Open one pull request that names every edition it corrects. Reviewers decide diff --git a/VERSIONING.md b/VERSIONING.md index f4df7d99..41840872 100644 --- a/VERSIONING.md +++ b/VERSIONING.md @@ -90,11 +90,21 @@ editions/ ``` A record names the product and version, the release tag and commit, the -Enterprise qualification tag, the LanceDB an Enterprise release bundles, and -the commit of every root the assembler read, including this repository's own -pages (Geneva, the dataset cards and the REST API specification) and the -specification's upstream release. Older editions never inherit today's shared -pages. +Enterprise qualification tag, the LanceDB an Enterprise release bundles, the +commit of every root the assembler read, and the specification's upstream +release. + +An edition's sources are fixed when it is prepared: + +- **Product pages** come from the release commits: the OSS release, or the + Enterprise release and the LanceDB it bundles. +- **Docs-owned content**, the Geneva pages, the dataset cards and the REST API + specification, comes from the docs input commit: the commit of this + repository's `main` that preparation read, which the record names with the + others. It is `main` as it was when the edition was prepared, not when the + product was released. +- Later changes to `main`, or to a product's sources, never reach a saved + edition, so older editions never inherit today's pages. Saved editions are kept by default but are not immutable. A reviewed correction changes an edition's files at the same URL. The record keeps the @@ -103,13 +113,22 @@ correction with the checksum of the edition it accepted; the saved edition must match the latest one. Git history holds the correction pull requests. Publication checksums describe the corrected candidate. +A pull request may add editions, and add corrections at the end of a record. +It may not remove an edition or a record, change a record's release, sources +or prepared snapshot, or change an earlier correction. The Assemble workflow's +test job checks this offline, on every pull request against its base commit +and on every push to `main`, by comparing the records' parsed values. + Preparation never overwrites a saved edition: -- The same release prepared again from the same release commits changes - nothing, even after the edition has been corrected. A later commit of this - repository does not make it a different release. -- Different commits for a saved release, such as a moved tag or another - Enterprise source, stop preparation for review. +- The same release prepared again from the same release commits and + qualification tag changes nothing, even after the edition has been + corrected. A later commit of this repository's `main` does not make it a + different release. +- Different inputs for a saved release, such as a moved tag, another + Enterprise source, or a qualification tag deleted and created again, stop + preparation for review. A recreated qualification tag is new qualification + evidence, not a duplicate. - A failed preparation leaves no edition and no record. ## Preparing a release @@ -123,8 +142,15 @@ shared pages from that commit. A standalone OSS edition is not needed. **OSS.** Preparation takes the `lancedb/lancedb` release tag. The full navigation includes the Enterprise pages, so the Enterprise tag or commit -they come from is named explicitly. Nothing reads the Enterprise repository's -`main` or infers a compatibility relationship. +they come from is named explicitly. Naming it pins the source; it does not +decide that the source may be published with that OSS release, which is still +open. Nothing reads the Enterprise repository's `main` or infers a +compatibility relationship. + +**This repository.** The docs-owned pages and `assemble.yaml` are read from a +commit of `main`, never from a branch: squash merges give a branch's commits +new hashes, so only a commit of `main` stays recoverable. The assembler that +runs must be that commit's. A missing tag, absent documentation, an unqualified or inconsistently tagged Enterprise release, or private-source files that may not be published stop @@ -183,7 +209,8 @@ approved publication or change the preferred edition. Promotion reuses the staged artifact. Missing qualification, absent sources, ambiguous inputs or failed checks stop -the workflow with a report. Duplicate events are harmless. +the workflow with a report. A duplicate event, for a release already saved +from the same commits and qualification tag, is harmless. Rolling back changes the preferred edition and keeps published folders. Restoring an earlier whole-site build would remove newer editions, so it needs From 8e5a04145d9956a4bd62adfeb262db664c768783 Mon Sep 17 00:00:00 2001 From: Brendan Clement Date: Tue, 20 Oct 2026 15:09:24 -0700 Subject: [PATCH 10/19] fix: fetch docs main from lancedb/docs itself when preparing Preparation trusted origin/main without knowing where it came from. In the usual fork layout, origin is the contributor's fork, so a docs-owned change only on the fork's main reached the edition and its record named the commit as lancedb/docs. Preparation now requires origin to be lancedb/docs over HTTPS or SSH, with URL rewriting applied as fetching applies it, and stops with how to fix it before fetching or saving anything. It then fetches main from origin itself and checks the docs input commit against what that fetch brought, not against a remote-tracking branch fetched earlier. The ancestry and matching-scripts checks are unchanged, and `check` stays offline. Tests stand in for lancedb/docs with a local bare repository that they accept as its URL; one test checks the real URL forms alone. --- README.md | 21 +++--- VERSIONING.md | 6 +- scripts/edition.py | 62 +++++++++++++---- scripts/tests/test_edition.py | 123 +++++++++++++++++++++++++++++++--- 4 files changed, 176 insertions(+), 36 deletions(-) diff --git a/README.md b/README.md index 2a8775bd..dd414cb3 100644 --- a/README.md +++ b/README.md @@ -232,12 +232,12 @@ changes one. ### Preparing an edition -Fetch the release tags into the LanceDB and Enterprise checkouts, and this -repository's `main`, then run the command from a checkout of this repository -whose `scripts/` are those of `origin/main`: +Fetch the release tags into the LanceDB and Enterprise checkouts. Then run the +command in a clone of this repository whose `origin` is +`https://github.com/lancedb/docs.git`, or its SSH form, rather than a fork, on +a branch whose `scripts/` are those of `main`: ```bash -git fetch origin main uv run --no-project --with pyyaml scripts/edition.py prepare enterprise 1.10.0 --lancedb ../lancedb --sophon ../sophon uv run --no-project --with pyyaml scripts/edition.py prepare oss 0.41.0 --enterprise v1.10.0 --lancedb ../lancedb --sophon ../sophon ``` @@ -255,9 +255,12 @@ uv run --no-project --with pyyaml scripts/edition.py prepare oss 0.41.0 --enterp settled yet; see [Open questions](VERSIONING.md#open-questions). - The LanceDB and Enterprise roots are read from Git at their commits, never from a working tree, and only from `docs/web`. -- This repository's `docs/` and `assemble.yaml` are read from Git at - `origin/main`, or at an earlier commit of `main` named with `--docs`, never - from your branch or working tree. Squash merges give a branch's commits new +- Preparation fetches `main` from `origin` itself, and reads this + repository's `docs/` and `assemble.yaml` from Git at its head, or at an + earlier commit of `main` named with `--docs`. It never reads them from your + branch, your working tree or an `origin/main` fetched before. A fork's `main` + can hold commits `lancedb/docs` never has, so an `origin` that is not + `lancedb/docs` stops preparation. Squash merges give a branch's commits new hashes, so a record names only a commit of `main`, and editions prepared one after another in the same pull request do not depend on its unmerged commits. The assembler that runs is your checkout's `scripts/`, so they must @@ -268,8 +271,8 @@ Preparation saves nothing, and says why, when a tag or commit is missing, a release has no `docs/web`, an Enterprise release is not qualified or bundles another LanceDB, a root holds a symlink, the Enterprise root holds a hidden file, the assembler stops, `.gitignore` would leave out one of the edition's -files, the docs input commit is not on `origin/main`, or your `scripts/` are -not that commit's. +files, `origin` is not `lancedb/docs`, the docs input commit is not on its +`main`, or your `scripts/` are not that commit's. Preparing a saved release again changes nothing when its release commits and qualification tag match the record, even after the edition has been corrected, diff --git a/VERSIONING.md b/VERSIONING.md index 41840872..08564691 100644 --- a/VERSIONING.md +++ b/VERSIONING.md @@ -149,8 +149,10 @@ compatibility relationship. **This repository.** The docs-owned pages and `assemble.yaml` are read from a commit of `main`, never from a branch: squash merges give a branch's commits -new hashes, so only a commit of `main` stays recoverable. The assembler that -runs must be that commit's. +new hashes, so only a commit of `main` stays recoverable. Preparation fetches +`main` itself, from a clone whose origin is `lancedb/docs`, because a fork's +`main` can hold commits `lancedb/docs` never has. The assembler that runs must +be that commit's. A missing tag, absent documentation, an unqualified or inconsistently tagged Enterprise release, or private-source files that may not be published stop diff --git a/scripts/edition.py b/scripts/edition.py index 0a6da08e..cf6f8e6e 100644 --- a/scripts/edition.py +++ b/scripts/edition.py @@ -24,11 +24,12 @@ Each root is read from Git objects at its commit, never from a working tree, and only from its documentation folder: `docs/web` of lancedb/lancedb and -lancedb/sophon, and `docs` of this repository at a commit of its main branch, -origin/main or one named with --docs, which also supplies assemble.yaml. A -branch's commits get new hashes when it is merged, so a record names only a -commit of main. The assembler that runs is this checkout's scripts/, which -must be that commit's. +lancedb/sophon, and `docs` of this repository at a commit of lancedb/docs +main, which also supplies assemble.yaml. Preparation fetches that main from +origin, which must be lancedb/docs rather than a fork, and reads its head or +an earlier commit named with --docs. A branch's commits get new hashes when it +is merged, so a record names only a commit of main. The assembler that runs is +this checkout's scripts/, which must be that commit's. A saved edition is never replaced. Preparing a release that already has one changes nothing when the release inputs match its record, even after the @@ -78,9 +79,15 @@ "enterprise": ("lancedb/sophon", "docs/web"), "build": ("lancedb/docs", "docs"), } -# This repository's main as fetched. Squash and rebase merges give a branch's -# commits new hashes, so only a commit of main stays recoverable. -DOCS_MAIN = "origin/main" +# This repository's main, which preparation fetches from origin. Squash and +# rebase merges give a branch's commits new hashes, so only a commit of main +# stays recoverable, and a fork's main can hold commits lancedb/docs never +# has, so origin must be lancedb/docs itself, over HTTPS or SSH. +DOCS_REMOTE = "origin" +DOCS_MAIN = f"{DOCS_REMOTE}/main" +DOCS_URLS = re.compile( + r"(?:https://github\.com/|ssh://git@github\.com/|git@github\.com:)lancedb/docs(?:\.git)?", re.IGNORECASE +) # An Enterprise release bundles LanceDB as this gitlink, and a release that # passed qualification has an annotated tag of this name on its commit. BUNDLED_LANCEDB = "src/lancedb" @@ -219,16 +226,43 @@ def is_ancestor(repo: Path, commit: str, of: str) -> bool: return result.returncode == 0 +def docs_main(repo: Path) -> str: + """lancedb/docs main, fetched now from origin, which must be lancedb/docs. + + An existing remote-tracking branch is not trusted: it may have been fetched + from a fork, or before origin was changed. Only what this fetch brings is. + """ + result = subprocess.run( + ["git", "-C", str(repo), "remote", "get-url", "--all", DOCS_REMOTE], capture_output=True + ) + # As fetching would use them, with any `insteadOf` rewriting applied. + urls = result.stdout.decode().splitlines() if result.returncode == 0 else [] + others = [url for url in urls if not DOCS_URLS.fullmatch(url)] + if not urls or others: + raise EditionError( + f"this checkout's {DOCS_REMOTE} is {', '.join(others) if urls else 'not set'}, not " + f"{SOURCES['build'][0]}; a fork's main can hold commits {SOURCES['build'][0]} never has, " + "so prepare an edition in a clone of https://github.com/lancedb/docs.git, over HTTPS " + "or SSH" + ) + try: + git(repo, "fetch", "--quiet", "--no-tags", DOCS_REMOTE, f"+refs/heads/main:refs/remotes/{DOCS_MAIN}") + except EditionError as exc: + raise EditionError(f"could not fetch {SOURCES['build'][0]} main: {exc}") from None + return commit_of(repo, DOCS_MAIN, SOURCES["build"][0]) + + def docs_commit(repo: Path, ref: str) -> str: - """The commit of this repository's main that the build reads `docs/` and + """The commit of lancedb/docs main that the build reads `docs/` and assemble.yaml from. The assembler is this checkout's own scripts/, so they must be that commit's too.""" what = f"{SOURCES['build'][0]} --docs" + main = docs_main(repo) commit = commit_of(repo, ref, what) - if not is_ancestor(repo, commit, commit_of(repo, DOCS_MAIN, SOURCES["build"][0])): + if not is_ancestor(repo, commit, main): raise EditionError( - f"{what}: {ref} is {commit}, which {DOCS_MAIN} does not contain; name a commit " - "of main, because merging gives a branch's commits new hashes" + f"{what}: {ref} is {commit}, which {SOURCES['build'][0]} main ({main}) does not " + "contain; name a commit of main, because merging gives a branch's commits new hashes" ) changed = text(repo, "status", "--porcelain", "--", "scripts") if changed: @@ -832,8 +866,8 @@ def parse(argv: list[str] | None = None) -> argparse.Namespace: prep.add_argument("--sophon", type=Path, default=REPO_ROOT.parent / "sophon", help="a lancedb/sophon repository (default: ../sophon)") prep.add_argument("--docs", metavar="REF", default=DOCS_MAIN, - help="the commit of this repository's main to read docs/ and assemble.yaml " - f"from (default: {DOCS_MAIN})") + help="the commit of lancedb/docs main to read docs/ and assemble.yaml from " + f"(default: {DOCS_MAIN}, as fetched from origin when preparing)") checking = commands.add_parser("check", help="check the saved editions against their records") checking.add_argument("--base", metavar="COMMIT", help="also require the editions and records on COMMIT, such as a pull " diff --git a/scripts/tests/test_edition.py b/scripts/tests/test_edition.py index a42ef20c..36433280 100644 --- a/scripts/tests/test_edition.py +++ b/scripts/tests/test_edition.py @@ -28,6 +28,12 @@ import edition # noqa: E402 +# Bare repositories named lancedb-docs.git stand in for lancedb/docs, whose +# real URLs preparation requires of origin. Tests accept both; one test checks +# the real ones alone. +LANCEDB_DOCS_URLS = edition.DOCS_URLS +TEST_DOCS_URLS = re.compile(rf"{LANCEDB_DOCS_URLS.pattern}|/.+/lancedb-docs\.git", re.IGNORECASE) + NAVIGATION = { "navigation": { "tabs": [ @@ -80,6 +86,11 @@ def isolated_git(monkeypatch): monkeypatch.setenv(key, value) +@pytest.fixture(autouse=True) +def stand_in_for_lancedb_docs(monkeypatch): + monkeypatch.setattr(edition, "DOCS_URLS", TEST_DOCS_URLS) + + def git(cwd: Path, *args: str, data: bytes | None = None) -> str: return subprocess.run( ["git", *args], cwd=cwd, env=environment(), input=data, check=True, capture_output=True @@ -218,9 +229,10 @@ def docs(tmp_path) -> Path: def docs_checkout(path: Path) -> Path: - """A lancedb/docs checkout of main, fetched as origin/main: this - repository's assemble.yaml and .gitignore, and docs-owned pages, a - navigation fragment and the OpenAPI spec.""" + """A checkout of lancedb/docs main, whose origin is a bare repository that + stands in for lancedb/docs: this repository's assemble.yaml and + .gitignore, and docs-owned pages, a navigation fragment and the OpenAPI + spec.""" repo = new_repo(path) for name in ("assemble.yaml", ".gitignore"): shutil.copy(SCRIPTS.parent / name, repo / name) @@ -234,13 +246,17 @@ def docs_checkout(path: Path) -> Path: (repo / rel).write_text(content) git(repo, "add", "-A") git(repo, "commit", "-q", "-m", "docs") + origin = path.parent / f"{path.name}-upstream" / "lancedb-docs.git" + git(path.parent, "init", "-q", "--bare", "-b", "main", str(origin)) + git(repo, "remote", "add", "origin", str(origin)) on_main(repo) return repo def on_main(docs: Path) -> str: - """HEAD merged into main and fetched: origin/main points at it.""" - git(docs, "update-ref", "refs/remotes/origin/main", "HEAD") + """HEAD merged into lancedb/docs main: pushed to the repository standing in + for it, where preparation fetches main from.""" + git(docs, "push", "-q", "--force", "origin", "HEAD:refs/heads/main") return git(docs, "rev-parse", "HEAD") @@ -497,7 +513,9 @@ def test_docs_inputs_off_main_or_unlike_the_running_scripts_are_refused(docs, so head = commit_all(docs, "an unmerged page") oss = ["oss", "0.98.0", "--enterprise", "v8.1.0"] - with pytest.raises(edition.EditionError, match=f"--docs: HEAD is {head}, which origin/main does not contain"): + main = git(docs, "rev-parse", "origin/main") + with pytest.raises(edition.EditionError, + match=rf"--docs: HEAD is {head}, which lancedb/docs main \({main}\) does not contain"): prepare(docs, sources, *oss, "--docs", "HEAD") # The branch changes the assembler, committed and then not. @@ -510,12 +528,86 @@ def test_docs_inputs_off_main_or_unlike_the_running_scripts_are_refused(docs, so with pytest.raises(edition.EditionError, match="commit or stash the changes to scripts/"): prepare(docs, sources, *oss) - git(docs, "update-ref", "-d", "refs/remotes/origin/main") - with pytest.raises(edition.EditionError, match="origin/main is not a commit in .*; fetch it first"): + git(docs, "remote", "set-url", "origin", str(docs.parent / "missing/lancedb-docs.git")) + with pytest.raises(edition.EditionError, match="could not fetch lancedb/docs main: git fetch .* failed"): + prepare(docs, sources, *oss) + git(docs, "remote", "remove", "origin") + with pytest.raises(edition.EditionError, match="this checkout's origin is not set, not lancedb/docs"): prepare(docs, sources, *oss) assert state(docs) == {} +def test_only_the_lancedb_docs_urls_are_accepted_as_origin(docs, monkeypatch): + monkeypatch.setattr(edition, "DOCS_URLS", LANCEDB_DOCS_URLS) + # Nothing here reaches the network: an accepted URL's fetch stops at once. + monkeypatch.setenv("GIT_ALLOW_PROTOCOL", "file") + for url in ("https://github.com/lancedb/docs", "https://github.com/lancedb/docs.git", + "https://github.com/LanceDB/docs.git", "git@github.com:lancedb/docs.git", + "ssh://git@github.com/lancedb/docs.git"): + git(docs, "remote", "set-url", "origin", url) + with pytest.raises(edition.EditionError, match="could not fetch lancedb/docs main: .*not allowed"): + edition.docs_main(docs) + for url in ("https://github.com/someone/docs.git", "git@github.com:someone/docs.git", + "https://github.com/lancedb/docs-fork.git", "https://github.com/lancedb/docs.git.old", + "http://github.com/lancedb/docs.git", "https://github.com.example.com/lancedb/docs.git", + "https://example.com/lancedb/docs.git", "file:///srv/lancedb/docs.git", + "/srv/git/lancedb-docs.git"): + git(docs, "remote", "set-url", "origin", url) + with pytest.raises(edition.EditionError, match=f"this checkout's origin is {re.escape(url)}, not"): + edition.docs_main(docs) + + +def test_a_fork_origin_is_refused_before_anything_is_fetched_or_saved(docs, sources): + """The usual fork layout: origin is a fork, upstream is lancedb/docs, and + the fork's main has a docs-owned change lancedb/docs does not have.""" + fork = docs.parent / "fork.git" + git(docs.parent, "clone", "-q", "--bare", git(docs, "remote", "get-url", "origin"), str(fork)) + git(docs, "remote", "rename", "origin", "upstream") + git(docs, "remote", "add", "origin", str(fork)) + (docs / "docs/geneva/index.mdx").write_text("# Only in the fork\n") + fork_main = commit_all(docs, "a change only the fork has") + git(docs, "push", "-q", "origin", "HEAD:refs/heads/main") + assert not edition.is_ancestor(docs, fork_main, "upstream/main") + refs = git(docs, "for-each-ref") + + refused = rf"this checkout's origin is {re.escape(str(fork))}, not lancedb/docs; a fork's main" + with pytest.raises(edition.EditionError, match=refused): + prepare(docs, sources, "oss", "0.98.0", "--enterprise", "v8.1.0") + with pytest.raises(edition.EditionError, match=refused): + prepare(docs, sources, "oss", "0.98.0", "--enterprise", "v8.1.0", "--docs", "upstream/main") + # Every URL of origin must be lancedb/docs's. + git(docs, "remote", "set-url", "origin", git(docs, "remote", "get-url", "upstream")) + git(docs, "remote", "set-url", "--add", "origin", str(fork)) + with pytest.raises(edition.EditionError, match=refused): + prepare(docs, sources, "oss", "0.98.0", "--enterprise", "v8.1.0") + # The URL is the one fetching would use, after any rewriting. + git(docs, "config", "--unset-all", "remote.origin.url") + git(docs, "remote", "set-url", "origin", "https://github.com/lancedb/docs.git") + git(docs, "config", f"url.{fork}.insteadOf", "https://github.com/lancedb/docs.git") + with pytest.raises(edition.EditionError, match=refused): + prepare(docs, sources, "oss", "0.98.0", "--enterprise", "v8.1.0") + assert state(docs) == {} + assert git(docs, "for-each-ref") == refs + + +def test_a_main_not_fetched_from_lancedb_docs_is_not_trusted(docs, sources): + main = git(docs, "rev-parse", "origin/main") + git(docs, "switch", "-q", "-c", "editions") + (docs / "docs/geneva/index.mdx").write_text("# Geneva, on a branch\n") + branch = commit_all(docs, "an unmerged page") + # As if origin/main had been fetched from elsewhere, or set by hand. + git(docs, "update-ref", "refs/remotes/origin/main", branch) + + with pytest.raises(edition.EditionError, match=rf"which lancedb/docs main \({main}\) does not contain"): + prepare(docs, sources, "oss", "0.98.0", "--enterprise", "v8.1.0", "--docs", branch) + assert state(docs) == {} + git(docs, "update-ref", "refs/remotes/origin/main", branch) + prepare(docs, sources, "oss", "0.98.0", "--enterprise", "v8.1.0") + assert record(docs, "oss-0.98.0")["sources"]["build"]["commit"] == main + assert (docs / "editions/oss/0.98.0/geneva/index.mdx").read_text() == "# Geneva\n" + assert git(docs, "rev-parse", "origin/main") == main + + # --------------------------------------------------------------------------- # # protecting saved editions # --------------------------------------------------------------------------- # @@ -650,9 +742,16 @@ def test_command_prepares_from_main_and_checks_against_the_base(docs, sources): git(docs, "commit", "-q", "-m", "scripts") main = on_main(docs) + # The checkout's own edition.py, given the test's lancedb/docs URLs: the + # real ones refuse the local repository standing in for it. + runner = ( + "import re, sys; sys.path.insert(0, 'scripts'); import edition; " + "edition.DOCS_URLS = re.compile(sys.argv[1], re.IGNORECASE); raise SystemExit(edition.main(sys.argv[2:]))" + ) + def run(*argv: str) -> subprocess.CompletedProcess: return subprocess.run( - [sys.executable, "-B", "scripts/edition.py", *argv], + [sys.executable, "-B", "-c", runner, TEST_DOCS_URLS.pattern, *argv], cwd=docs, env=environment(), capture_output=True, text=True, ) @@ -918,8 +1017,10 @@ def enterprise(rec): def records(tmp_path_factory, sources) -> dict[str, dict]: """The records the builder writes, which are the contract.""" docs = docs_checkout(tmp_path_factory.mktemp("records").resolve() / "docs") - prepare(docs, sources, "oss", "0.98.0", "--enterprise", "v8.1.0") - prepare(docs, sources, "enterprise", "8.1.0") + with pytest.MonkeyPatch.context() as patch: + patch.setattr(edition, "DOCS_URLS", TEST_DOCS_URLS) + prepare(docs, sources, "oss", "0.98.0", "--enterprise", "v8.1.0") + prepare(docs, sources, "enterprise", "8.1.0") return {name: record(docs, name) for name in ("oss-0.98.0", "enterprise-8.1.0")} From 1331aeba9ee1c4ab85f3417f137b6575aa5e8649 Mon Sep 17 00:00:00 2001 From: Brendan Clement Date: Tue, 20 Oct 2026 15:09:41 -0700 Subject: [PATCH 11/19] docs: say saved editions are kept only while the check is required The check protects saved editions only if it must pass before a pull request merges, which is a repository rule set up separately. The check on a push to main reports a violation in that push but cannot undo it, and a later push does not find it, because its base already includes the change. The README, VERSIONING.md and the module docstring now say so. --- README.md | 5 +++++ VERSIONING.md | 5 ++++- scripts/edition.py | 6 +++--- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index dd414cb3..78eae375 100644 --- a/README.md +++ b/README.md @@ -357,6 +357,11 @@ names, and for a push to `main`, the commit `main` pointed at before. It fetches that commit by hash. A manual run of the workflow has no base, and checks each edition against its record only. +This keeps saved editions only while the check must pass before a pull request +merges, which is a repository rule set up separately. The check on a push to +`main` reports a violation in that push but cannot undo it, and a later push +does not find it, because its base already includes the change. + ### Correcting a saved edition A saved edition can be corrected after review, at the same URL: diff --git a/VERSIONING.md b/VERSIONING.md index 08564691..16130197 100644 --- a/VERSIONING.md +++ b/VERSIONING.md @@ -117,7 +117,10 @@ A pull request may add editions, and add corrections at the end of a record. It may not remove an edition or a record, change a record's release, sources or prepared snapshot, or change an earlier correction. The Assemble workflow's test job checks this offline, on every pull request against its base commit -and on every push to `main`, by comparing the records' parsed values. +and on every push to `main`, by comparing the records' parsed values. That +protects saved editions only while the check is required before merging, a +repository rule set up separately. On a push to `main` it can report a +violation but not undo it, and later pushes no longer see it. Preparation never overwrites a saved edition: diff --git a/scripts/edition.py b/scripts/edition.py index cf6f8e6e..45bb1fd5 100644 --- a/scripts/edition.py +++ b/scripts/edition.py @@ -34,9 +34,9 @@ A saved edition is never replaced. Preparing a release that already has one changes nothing when the release inputs match its record, even after the edition has been corrected, and stops when they differ. A preparation that -fails leaves neither an edition nor a record behind. Pull requests run -`check --base` against their base commit, so a merged edition and its record -stay as they are, apart from recorded corrections. +fails leaves neither an edition nor a record behind. `check --base`, run on a +pull request against its base commit and required before it merges, keeps a +merged edition and its record as they are, apart from recorded corrections. Usage: From 04de9ea8271c71f694e6868a2740d6cb31b167ce Mon Sep 17 00:00:00 2001 From: Brendan Clement Date: Tue, 20 Oct 2026 16:29:25 -0700 Subject: [PATCH 12/19] docs: trim redundant edition comments --- .github/workflows/assemble.yml | 9 +-- Makefile | 4 +- scripts/assemble.py | 7 +-- scripts/edition.py | 101 ++++++--------------------------- scripts/tests/test_edition.py | 55 +++--------------- 5 files changed, 28 insertions(+), 148 deletions(-) diff --git a/.github/workflows/assemble.yml b/.github/workflows/assemble.yml index 0024e48b..b6aff1e1 100644 --- a/.github/workflows/assemble.yml +++ b/.github/workflows/assemble.yml @@ -29,8 +29,7 @@ env: SOPHON_DOCS_ROOT: sophon/docs/web jobs: - # Forks run the guard tests and the saved-edition check without - # private-source credentials. + # Fork-safe checks need no private-source credentials. test: name: Test the assembler runs-on: ubuntu-latest @@ -43,11 +42,7 @@ jobs: - name: Test the overlay, navigation and private-root guards run: make test-assemble - # A pull request is compared with the base commit its event names, and a - # push with the commit the branch pointed at before it. That commit is - # fetched by hash, so the comparison never depends on a remote-tracking - # branch or on what the shallow checkout happens to hold. A manual run - # has no base and checks each edition against its record only. + # Fetch the event's base SHA so shallow clones and moving branches do not affect preservation checks. - name: Check that saved editions and their records are kept env: EVENT: ${{ github.event_name }} diff --git a/Makefile b/Makefile index bfcae4b2..efa2221a 100644 --- a/Makefile +++ b/Makefile @@ -32,9 +32,7 @@ assemble: test-assemble: @$(ASSEMBLE_RUN) --with pytest pytest scripts/tests -q -# Check every saved edition against its release record. With BASE set to a -# commit, also check that the editions and records on it are kept. See "Saved -# editions" in README.md. +# Set BASE to enforce preservation of editions already merged. check-editions: @$(ASSEMBLE_RUN) scripts/edition.py check $(if $(BASE),--base $(BASE)) diff --git a/scripts/assemble.py b/scripts/assemble.py index 2a141e2b..a49da234 100644 --- a/scripts/assemble.py +++ b/scripts/assemble.py @@ -1017,12 +1017,7 @@ def emit( def assemble_site(config: Config) -> tuple[int, list[str]]: - """Run every stage on `config` and write its output. - - Returns the number of files written and the warnings. `main` and the - edition builder (`edition.py`) both assemble through here, so a saved - edition is built exactly the way the site is. - """ + """Build live sites and saved editions identically; return file count and warnings.""" resolved = resolve(config) docs_json, nav_raw = assemble_nav(resolved) warnings = validate(config, resolved, docs_json) diff --git a/scripts/edition.py b/scripts/edition.py index 45bb1fd5..8ce25915 100644 --- a/scripts/edition.py +++ b/scripts/edition.py @@ -2,41 +2,10 @@ Prepare a saved documentation edition from one product release, and check the saved editions. -An edition is the site the assembler builds from one release's own sources, -saved under `editions///`. Its release record, -`releases/-.yaml`, names the release and the commit every -root was read from. Later builds compose the saved editions; nothing fetches -fresh content into one. See VERSIONING.md. - - prepare oss VERSION the lancedb/lancedb tag vVERSION, with the - Enterprise pages from a lancedb/sophon tag or - commit named explicitly with --enterprise - prepare enterprise VERSION the lancedb/sophon tag vVERSION, which must be - qualified, with the shared pages from the - LanceDB commit that release bundles - check check every record, pair it with its edition, - and require the edition to match the checksum - the record last accepted: the prepared one, or - its latest correction's - check --base COMMIT also require every edition and record on - COMMIT to remain, each record as it was there - but for corrections added at the end - -Each root is read from Git objects at its commit, never from a working tree, -and only from its documentation folder: `docs/web` of lancedb/lancedb and -lancedb/sophon, and `docs` of this repository at a commit of lancedb/docs -main, which also supplies assemble.yaml. Preparation fetches that main from -origin, which must be lancedb/docs rather than a fork, and reads its head or -an earlier commit named with --docs. A branch's commits get new hashes when it -is merged, so a record names only a commit of main. The assembler that runs is -this checkout's scripts/, which must be that commit's. - -A saved edition is never replaced. Preparing a release that already has one -changes nothing when the release inputs match its record, even after the -edition has been corrected, and stops when they differ. A preparation that -fails leaves neither an edition nor a record behind. `check --base`, run on a -pull request against its base commit and required before it merges, keeps a -merged edition and its record as they are, apart from recorded corrections. +Product sources come from their release commits. Docs-owned content and +assemble.yaml come from canonical lancedb/docs main, with matching scripts/. +Preparation never replaces an existing edition. Preservation across merges +requires `check --base` to pass before merging. See VERSIONING.md. Usage: @@ -71,25 +40,19 @@ # Editions are of stable releases. Prereleases need a policy first. VERSION_RE = re.compile(r"(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)") COMMIT_RE = candidate.COMMIT_RE -# Where each root in assemble.yaml is read from: repository and folder. A root -# added there without a source here stops preparation, so nothing reaches an -# edition without its provenance in the record. +# Every assemble.yaml root needs provenance here. SOURCES = { "lancedb": ("lancedb/lancedb", "docs/web"), "enterprise": ("lancedb/sophon", "docs/web"), "build": ("lancedb/docs", "docs"), } -# This repository's main, which preparation fetches from origin. Squash and -# rebase merges give a branch's commits new hashes, so only a commit of main -# stays recoverable, and a fork's main can hold commits lancedb/docs never -# has, so origin must be lancedb/docs itself, over HTTPS or SSH. +# Pin docs inputs to canonical main; squash merges can discard branch commits. DOCS_REMOTE = "origin" DOCS_MAIN = f"{DOCS_REMOTE}/main" DOCS_URLS = re.compile( r"(?:https://github\.com/|ssh://git@github\.com/|git@github\.com:)lancedb/docs(?:\.git)?", re.IGNORECASE ) -# An Enterprise release bundles LanceDB as this gitlink, and a release that -# passed qualification has an annotated tag of this name on its commit. +# Qualification requires an annotated tag on the release commit. BUNDLED_LANCEDB = "src/lancedb" QUALIFIED_TAG = "release-qualified/v{version}" # Files that would change how Git stores the edition's own files. @@ -108,9 +71,7 @@ class EditionError(Exception): """Inputs or state that must stop preparation before anything is saved.""" -# --------------------------------------------------------------------------- # # git -# --------------------------------------------------------------------------- # def git(repo: Path, *args: str, data: bytes | None = None) -> bytes: @@ -207,9 +168,7 @@ def export(repo: Path, commit: str, folder: str, dest: Path, what: str) -> None: offset = header_end + 1 + size + 1 -# --------------------------------------------------------------------------- # # release inputs -# --------------------------------------------------------------------------- # def source(name: str, commit: str) -> dict: @@ -227,11 +186,7 @@ def is_ancestor(repo: Path, commit: str, of: str) -> bool: def docs_main(repo: Path) -> str: - """lancedb/docs main, fetched now from origin, which must be lancedb/docs. - - An existing remote-tracking branch is not trusted: it may have been fetched - from a fork, or before origin was changed. Only what this fetch brings is. - """ + """Fetch canonical main; an existing tracking ref may come from a fork.""" result = subprocess.run( ["git", "-C", str(repo), "remote", "get-url", "--all", DOCS_REMOTE], capture_output=True ) @@ -253,9 +208,7 @@ def docs_main(repo: Path) -> str: def docs_commit(repo: Path, ref: str) -> str: - """The commit of lancedb/docs main that the build reads `docs/` and - assemble.yaml from. The assembler is this checkout's own scripts/, so they - must be that commit's too.""" + """Select docs inputs from main and require this checkout's scripts to match.""" what = f"{SOURCES['build'][0]} --docs" main = docs_main(repo) commit = commit_of(repo, ref, what) @@ -358,12 +311,7 @@ def resolve_release(args: argparse.Namespace, repo: Path) -> dict: def identity(record: dict) -> dict: - """What makes two preparations the same release: commits, not ref names. - - This repository's own commit is left out: re-running a release from a later - commit of main is still the same release, and its saved edition is kept as - it is. - """ + """Exclude docs main so preparing again after it advances keeps the saved edition.""" release = record["release"] return { "release": release.get("commit"), @@ -375,9 +323,7 @@ def identity(record: dict) -> dict: } -# --------------------------------------------------------------------------- # # records -# --------------------------------------------------------------------------- # def record_path(repo: Path, product: str, version: str) -> Path: @@ -401,12 +347,9 @@ def load_record(path: Path) -> dict: def record_problems(record: object, filename: str) -> list[str]: - """Each field of `record` that differs from what `prepare` writes. + """Validate fields offline; commit existence is not checked. - Formats are checked first; a field that disagrees with another is - reported only when both are well formed, so each problem names the field - that is wrong. Nothing is fetched: the commits are checked for form and - agreement, not against the repositories. + Compare fields only after validating their formats to avoid duplicate errors. """ problems: list[str] = [] @@ -465,7 +408,6 @@ def line(value: object) -> bool: if filename != f"{product}-{version}.yaml": problems.append(f"product and version name {product}-{version}.yaml, not {filename}") - # The release, and the commits only it can name. own = "lancedb" if product == "oss" else "enterprise" extra = ("enterprise_pages",) if product == "oss" else ("qualification", "bundled_lancedb") if product and fields("release", ("repository", "tag", "commit", *extra)): @@ -485,7 +427,6 @@ def line(value: object) -> bool: check("release.qualification.object", commit, "a tag object hash") check("release.bundled_lancedb", commit, "a commit hash") - # Every root the assembler read, and the spec the docs root ships. if fields("sources", tuple(SOURCES), ("openapi",)): for name, (repository, folder) in SOURCES.items(): if fields(f"sources.{name}", ("repository", "path", "commit")): @@ -499,7 +440,6 @@ def line(value: object) -> bool: check(f"sources.openapi.{key}", line, "a name") check("sources.openapi.sha256", checksum, "a SHA-256 checksum") - # A source and the release field that selected it name the same commit. if product: pairs = ( [("sources.lancedb.commit", "release.commit"), @@ -573,8 +513,7 @@ def ignored_paths(repo: Path, listing: str, index: bool = False) -> set[str]: def snapshot_state(repo: Path, record: dict) -> str: - """Whether the saved edition is the snapshot its record last accepted: - the one prepared, or the one its latest correction recorded.""" + """Require the latest recorded checksum, including corrections.""" current = candidate.site_checksum(saved_hashes(repo, record["edition"])) corrections = record["corrections"] accepted = corrections[-1]["sha256"] if corrections else record["prepared"]["sha256"] @@ -591,9 +530,7 @@ def snapshot_state(repo: Path, record: dict) -> str: return f"corrected ({count} correction{'s' if count > 1 else ''})" -# --------------------------------------------------------------------------- # # prepare -# --------------------------------------------------------------------------- # def existing(repo: Path, requested: dict) -> str: @@ -752,13 +689,10 @@ def prepare(args: argparse.Namespace, repo: Path = REPO_ROOT) -> str: ) -# --------------------------------------------------------------------------- # # check -# --------------------------------------------------------------------------- # def differences(was: object, now: object, path: str) -> list[str]: - """How two parsed values differ, field by field.""" if isinstance(was, dict) and isinstance(now, dict): return [ change @@ -773,11 +707,9 @@ def differences(was: object, now: object, path: str) -> list[str]: def preserved(repo: Path, base: str) -> list[str]: - """What has happened since `base` to the editions and records it has. + """Require base editions and records to remain; only append corrections. - Each of them remains. A record keeps every field it had there, except that - its corrections may gain entries at the end. Records are compared as - parsed values, so a change of formatting is not a change. + Compare parsed records so formatting changes are harmless. """ problems = [] listing = git(repo, "ls-tree", "-z", "--name-only", base, "--", "releases/").decode() @@ -820,8 +752,7 @@ def preserved(repo: Path, base: str) -> list[str]: def check(repo: Path = REPO_ROOT, base: str | None = None) -> tuple[list[str], list[str]]: - """Each edition's state, and every problem with records and editions; with - `base`, also every change since that commit to the ones it has.""" + """Return edition statuses and errors, optionally enforcing preservation from base.""" lines, problems, paired = [], [], set() for path in sorted((repo / "releases").glob("*.yaml")): # The file name says which edition is this record's, even when the diff --git a/scripts/tests/test_edition.py b/scripts/tests/test_edition.py index 36433280..4a58150a 100644 --- a/scripts/tests/test_edition.py +++ b/scripts/tests/test_edition.py @@ -28,9 +28,7 @@ import edition # noqa: E402 -# Bare repositories named lancedb-docs.git stand in for lancedb/docs, whose -# real URLs preparation requires of origin. Tests accept both; one test checks -# the real ones alone. +# Allow local stand-ins without adding a production URL-check bypass. LANCEDB_DOCS_URLS = edition.DOCS_URLS TEST_DOCS_URLS = re.compile(rf"{LANCEDB_DOCS_URLS.pattern}|/.+/lancedb-docs\.git", re.IGNORECASE) @@ -134,9 +132,7 @@ def new_repo(path: Path) -> Path: def oss_pages(label: str) -> dict: - """lancedb's docs/web as of `label`: navigation, a page with an example and a - draft section, a draft page, an asset, and the Enterprise placeholder the - current assembler needs.""" + """Minimal docs/web fixture, including the required Enterprise placeholder.""" web = "docs/web/" guide = ( "---\ntitle: Guide\n---\n\n" @@ -229,10 +225,7 @@ def docs(tmp_path) -> Path: def docs_checkout(path: Path) -> Path: - """A checkout of lancedb/docs main, whose origin is a bare repository that - stands in for lancedb/docs: this repository's assemble.yaml and - .gitignore, and docs-owned pages, a navigation fragment and the OpenAPI - spec.""" + """Docs checkout with a local bare origin standing in for canonical main.""" repo = new_repo(path) for name in ("assemble.yaml", ".gitignore"): shutil.copy(SCRIPTS.parent / name, repo / name) @@ -254,8 +247,7 @@ def docs_checkout(path: Path) -> Path: def on_main(docs: Path) -> str: - """HEAD merged into lancedb/docs main: pushed to the repository standing in - for it, where preparation fetches main from.""" + """Make HEAD available through the canonical-main stand-in.""" git(docs, "push", "-q", "--force", "origin", "HEAD:refs/heads/main") return git(docs, "rev-parse", "HEAD") @@ -302,8 +294,7 @@ def state(docs: Path) -> dict[str, bytes | None]: def correct(docs: Path, name: str, page: str, text: str) -> str: - """A reviewed correction, as the README describes it: edit the saved page, - then record it.""" + """Apply the README's edit-and-record correction workflow.""" product, version = name.split("-") (docs / "editions" / product / version / page).write_text(text) return record_correction(docs, name, page) @@ -331,9 +322,7 @@ def commit_all(docs: Path, message: str) -> str: return git(docs, "rev-parse", "HEAD") -# --------------------------------------------------------------------------- # # preparing editions -# --------------------------------------------------------------------------- # def test_oss_edition_is_built_from_its_tag_and_named_enterprise_pages(docs, sources): @@ -346,8 +335,6 @@ def test_oss_edition_is_built_from_its_tag_and_named_enterprise_pages(docs, sour "static/assets/marker.svg", ] guide = files["guide.mdx"].decode() - # The tag's own page, with its example written in, its draft section and - # draft page kept for Mint, and its asset byte for byte. assert "Shared guide as of 0.98.0." in guide assert "unreleased main" not in guide and "Uncommitted" not in guide assert "```python Python icon=\"python\"\nprint('0.98.0')\n```" in guide @@ -388,8 +375,7 @@ def test_enterprise_edition_reads_the_lancedb_it_bundles(docs, sources): prepare(docs, sources, "enterprise", "8.1.0", "--bundled-lancedb", bundled) files = saved(docs / "editions/enterprise/8.1.0") - # The shared pages are the bundled revision's, which no OSS release tags, - # and no OSS edition is needed for them. + # The bundled revision has no OSS release tag or saved edition. assert "Shared guide as of the bundled revision." in files["guide.mdx"].decode() assert files["enterprise/security.mdx"] == b"Enterprise security for 8.1\n" assert not (docs / "editions/oss").exists() @@ -462,9 +448,7 @@ def test_a_root_without_recorded_provenance_stops_preparation(docs, sources): assert state(docs) == {} -# --------------------------------------------------------------------------- # # the docs input commit -# --------------------------------------------------------------------------- # def test_docs_inputs_come_from_main_not_from_the_branch(docs, sources): @@ -480,7 +464,6 @@ def test_docs_inputs_come_from_main_not_from_the_branch(docs, sources): (docs / "docs/geneva/index.mdx").write_text("# Geneva, uncommitted\n") prepare(docs, sources, "oss", "0.98.0", "--enterprise", "v8.1.0") - # The first edition is committed on the branch before a second is prepared. git(docs, "add", "editions", "releases") git(docs, "commit", "-q", "-m", "oss 0.98.0") prepare(docs, sources, "enterprise", "8.1.0") @@ -518,7 +501,6 @@ def test_docs_inputs_off_main_or_unlike_the_running_scripts_are_refused(docs, so match=rf"--docs: HEAD is {head}, which lancedb/docs main \({main}\) does not contain"): prepare(docs, sources, *oss, "--docs", "HEAD") - # The branch changes the assembler, committed and then not. (docs / "scripts").mkdir() (docs / "scripts/assemble.py").write_text("# a changed assembler\n") commit_all(docs, "an unmerged assembler") @@ -575,7 +557,6 @@ def test_a_fork_origin_is_refused_before_anything_is_fetched_or_saved(docs, sour prepare(docs, sources, "oss", "0.98.0", "--enterprise", "v8.1.0") with pytest.raises(edition.EditionError, match=refused): prepare(docs, sources, "oss", "0.98.0", "--enterprise", "v8.1.0", "--docs", "upstream/main") - # Every URL of origin must be lancedb/docs's. git(docs, "remote", "set-url", "origin", git(docs, "remote", "get-url", "upstream")) git(docs, "remote", "set-url", "--add", "origin", str(fork)) with pytest.raises(edition.EditionError, match=refused): @@ -608,9 +589,7 @@ def test_a_main_not_fetched_from_lancedb_docs_is_not_trusted(docs, sources): assert git(docs, "rev-parse", "origin/main") == main -# --------------------------------------------------------------------------- # # protecting saved editions -# --------------------------------------------------------------------------- # def test_conflicting_inputs_for_a_saved_release_stop(docs, sources, tmp_path): @@ -620,7 +599,6 @@ def test_conflicting_inputs_for_a_saved_release_stop(docs, sources, tmp_path): with pytest.raises(edition.EditionError, match="enterprise_pages recorded .*, requested"): prepare(docs, sources, "oss", "0.98.0", "--enterprise", "v8.2.0") - # The release tag moved after the edition was saved. retagged = tmp_path / "retagged" git(tmp_path, "clone", "-q", str(sources["lancedb"]), str(retagged)) git(retagged, "tag", "-f", "-a", "-m", "moved", "v0.98.0", sources["oss"]["0.99.0"]) @@ -636,8 +614,7 @@ def test_duplicate_preparation_changes_nothing_even_after_a_correction(docs, sou assert "editions/oss/0.98.0 is as prepared; nothing changed" in out correct(docs, "oss-0.98.0", "guide.mdx", "Corrected guide\n") - # This repository's main moves on, and the same release is prepared - # again, its Enterprise pages named by commit this time. + # A commit reference and a tag naming it are the same release input. (docs / "docs/geneva/index.mdx").write_text("# Geneva, later\n") git(docs, "commit", "-q", "-am", "later") on_main(docs) @@ -704,9 +681,7 @@ def test_an_edition_or_a_record_without_its_pair_is_refused(docs, sources): assert not (docs / "editions/oss/0.98.0").exists() -# --------------------------------------------------------------------------- # # checking saved editions -# --------------------------------------------------------------------------- # def test_check_pairs_records_and_requires_corrections_to_be_listed(docs, sources): @@ -742,8 +717,7 @@ def test_command_prepares_from_main_and_checks_against_the_base(docs, sources): git(docs, "commit", "-q", "-m", "scripts") main = on_main(docs) - # The checkout's own edition.py, given the test's lancedb/docs URLs: the - # real ones refuse the local repository standing in for it. + # Run the checkout's CLI with the local canonical-main stand-in. runner = ( "import re, sys; sys.path.insert(0, 'scripts'); import edition; " "edition.DOCS_URLS = re.compile(sys.argv[1], re.IGNORECASE); raise SystemExit(edition.main(sys.argv[2:]))" @@ -795,9 +769,7 @@ def run(*argv: str) -> subprocess.CompletedProcess: assert result.stderr.startswith(f"edition: --base: {'1' * 40} is not a commit in ") -# --------------------------------------------------------------------------- # # corrections are bound to the edition's checksum -# --------------------------------------------------------------------------- # def test_a_later_unrecorded_change_fails_after_a_recorded_correction(docs, sources): @@ -826,14 +798,12 @@ def test_each_correction_records_the_snapshot_it_accepts(docs, sources): first = correct(docs, "oss-0.98.0", "guide.mdx", "Corrected guide\n") assert edition.check(docs) == (["oss 0.98.0: corrected (1 correction)"], []) - # Later changes nobody recorded: an edit, then a deletion. page = docs / "editions/oss/0.98.0/index.mdx" page.write_text("Unreviewed edit\n") assert "has changed since its latest recorded correction" in edition.check(docs)[1][0] page.unlink() assert "has changed since its latest recorded correction" in edition.check(docs)[1][0] - # Recording the deletion, once reviewed, as a second correction accepts it. second = record_correction(docs, "oss-0.98.0", "index.mdx") assert edition.check(docs) == (["oss 0.98.0: corrected (2 corrections)"], []) rec = record(docs, "oss-0.98.0") @@ -841,13 +811,11 @@ def test_each_correction_records_the_snapshot_it_accepts(docs, sources): assert [c["sha256"] for c in rec["corrections"]] == [first, second] assert second == checksum(saved(docs / "editions/oss/0.98.0")) - # Preparing the release again keeps the edition and both corrections. before = state(docs) out = prepare(docs, sources, "oss", "0.98.0", "--enterprise", "v8.1.0") assert out.endswith("editions/oss/0.98.0 is corrected (2 corrections); nothing changed") assert state(docs) == before - # It stays a no-op when a change has not been recorded, and says so. (docs / "editions/oss/0.98.0/guide.mdx").write_text("Another unreviewed edit\n") before = state(docs) with pytest.raises(edition.EditionError, match="already saved; nothing changed, but .* has changed"): @@ -855,9 +823,7 @@ def test_each_correction_records_the_snapshot_it_accepts(docs, sources): assert state(docs) == before -# --------------------------------------------------------------------------- # # merged editions and records are kept -# --------------------------------------------------------------------------- # @pytest.fixture @@ -891,7 +857,6 @@ def test_unchanged_editions_and_a_new_one_pass_against_the_base(docs, sources, b rewrite(docs, "enterprise-8.1.0", lambda rec: None) assert edition.check(docs, base=base) == ([*KEPT, "oss 0.99.0: as prepared"], []) - # A new edition is checked against its own record, as always. (docs / "editions/oss/0.99.0/index.mdx").write_text("Edited without a record\n") lines, problems = edition.check(docs, base=base) assert lines == KEPT @@ -949,7 +914,6 @@ def test_removing_or_replacing_a_saved_release_fails_against_the_base(docs, sour "is never removed", ]) - # Only the edition removed: reported once, with its record. git(docs, "reset", "-q", "--hard", base) shutil.rmtree(docs / "editions/enterprise/8.1.0") assert edition.check(docs, base=base) == ([KEPT[1]], [ @@ -976,9 +940,7 @@ def test_an_appended_correction_passes_against_the_base(docs, base): assert record(docs, "oss-0.98.0")["corrections"][0] == rec["corrections"][0] -# --------------------------------------------------------------------------- # # release records keep their provenance -# --------------------------------------------------------------------------- # def rewrite(docs: Path, name: str, change) -> None: @@ -1025,7 +987,6 @@ def records(tmp_path_factory, sources) -> dict[str, dict]: def change(path: str, value=KeyError): - """Set the field at a dotted path, or delete it.""" def apply(rec): *parents, key = path.split(".") for parent in parents: From 028e5010571148271ffcd002a3f559a07d7997b3 Mon Sep 17 00:00:00 2001 From: Brendan Clement Date: Tue, 20 Oct 2026 16:50:24 -0700 Subject: [PATCH 13/19] fix: resolve fetched docs main by its full ref Preparation fetched lancedb/docs main into refs/remotes/origin/main but then resolved the shorthand origin/main, which was also the default for --docs. A local branch or tag named origin/main wins that lookup, so its unmerged commit passed the ancestry check and went into the edition. The fetched main is now resolved as refs/remotes/origin/main, and without --docs preparation uses the commit that fetch returned. A commit named with --docs is still checked against it. --- scripts/edition.py | 13 ++++----- scripts/tests/test_edition.py | 50 +++++++++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+), 6 deletions(-) diff --git a/scripts/edition.py b/scripts/edition.py index 8ce25915..7b1df1dc 100644 --- a/scripts/edition.py +++ b/scripts/edition.py @@ -48,7 +48,8 @@ } # Pin docs inputs to canonical main; squash merges can discard branch commits. DOCS_REMOTE = "origin" -DOCS_MAIN = f"{DOCS_REMOTE}/main" +# Fully qualified: a local branch or tag named origin/main would win the shorthand. +DOCS_MAIN = f"refs/remotes/{DOCS_REMOTE}/main" DOCS_URLS = re.compile( r"(?:https://github\.com/|ssh://git@github\.com/|git@github\.com:)lancedb/docs(?:\.git)?", re.IGNORECASE ) @@ -201,17 +202,17 @@ def docs_main(repo: Path) -> str: "or SSH" ) try: - git(repo, "fetch", "--quiet", "--no-tags", DOCS_REMOTE, f"+refs/heads/main:refs/remotes/{DOCS_MAIN}") + git(repo, "fetch", "--quiet", "--no-tags", DOCS_REMOTE, f"+refs/heads/main:{DOCS_MAIN}") except EditionError as exc: raise EditionError(f"could not fetch {SOURCES['build'][0]} main: {exc}") from None return commit_of(repo, DOCS_MAIN, SOURCES["build"][0]) -def docs_commit(repo: Path, ref: str) -> str: +def docs_commit(repo: Path, ref: str | None) -> str: """Select docs inputs from main and require this checkout's scripts to match.""" what = f"{SOURCES['build'][0]} --docs" main = docs_main(repo) - commit = commit_of(repo, ref, what) + commit = main if ref is None else commit_of(repo, ref, what) if not is_ancestor(repo, commit, main): raise EditionError( f"{what}: {ref} is {commit}, which {SOURCES['build'][0]} main ({main}) does not " @@ -796,9 +797,9 @@ def parse(argv: list[str] | None = None) -> argparse.Namespace: help="a lancedb/lancedb repository (default: ../lancedb)") prep.add_argument("--sophon", type=Path, default=REPO_ROOT.parent / "sophon", help="a lancedb/sophon repository (default: ../sophon)") - prep.add_argument("--docs", metavar="REF", default=DOCS_MAIN, + prep.add_argument("--docs", metavar="REF", help="the commit of lancedb/docs main to read docs/ and assemble.yaml from " - f"(default: {DOCS_MAIN}, as fetched from origin when preparing)") + "(default: main as fetched from origin when preparing)") checking = commands.add_parser("check", help="check the saved editions against their records") checking.add_argument("--base", metavar="COMMIT", help="also require the editions and records on COMMIT, such as a pull " diff --git a/scripts/tests/test_edition.py b/scripts/tests/test_edition.py index 4a58150a..5f794ed1 100644 --- a/scripts/tests/test_edition.py +++ b/scripts/tests/test_edition.py @@ -589,6 +589,56 @@ def test_a_main_not_fetched_from_lancedb_docs_is_not_trusted(docs, sources): assert git(docs, "rev-parse", "origin/main") == main +def shadowed_main(docs: Path, kind: str) -> tuple[str, str]: + """A local branch or tag named origin/main on an unmerged commit, which the + shorthand origin/main resolves to before the remote-tracking ref.""" + main = git(docs, "rev-parse", "refs/remotes/origin/main") + git(docs, "switch", "-q", "-c", "editions") + (docs / "docs/geneva/index.mdx").write_text("# Geneva, unmerged\n") + unmerged = commit_all(docs, "an unmerged page") + git(docs, kind, "origin/main", unmerged) + assert git(docs, "rev-parse", "origin/main") == unmerged + return main, unmerged + + +@pytest.mark.parametrize("kind", ["branch", "tag"]) +def test_a_local_ref_named_origin_main_is_not_canonical_main(docs, sources, kind): + main, _unmerged = shadowed_main(docs, kind) + + prepare(docs, sources, "oss", "0.98.0", "--enterprise", "v8.1.0") + + rec = record(docs, "oss-0.98.0") + assert rec["sources"]["build"]["commit"] == main + assert (docs / rec["edition"] / "geneva/index.mdx").read_text() == "# Geneva\n" + + +def test_main_is_fetched_where_it_is_read_when_origin_tracks_another_branch(docs, sources): + """Git updates no tracking ref from the configured refspec here, so the + fetch itself must write refs/remotes/origin/main.""" + main = git(docs, "rev-parse", "refs/remotes/origin/main") + git(docs, "config", "remote.origin.fetch", "+refs/heads/other:refs/remotes/origin/other") + git(docs, "switch", "-q", "-c", "editions") + (docs / "docs/geneva/index.mdx").write_text("# Geneva, unmerged\n") + git(docs, "update-ref", "refs/remotes/origin/main", commit_all(docs, "an unmerged page")) + + prepare(docs, sources, "oss", "0.98.0", "--enterprise", "v8.1.0") + + rec = record(docs, "oss-0.98.0") + assert rec["sources"]["build"]["commit"] == main + assert (docs / rec["edition"] / "geneva/index.mdx").read_text() == "# Geneva\n" + + +@pytest.mark.parametrize("kind", ["branch", "tag"]) +def test_naming_an_unmerged_commit_is_refused_despite_a_local_origin_main(docs, sources, kind): + main, unmerged = shadowed_main(docs, kind) + + for ref in (unmerged, "origin/main"): + with pytest.raises(edition.EditionError, + match=rf"--docs: {ref} is {unmerged}, which lancedb/docs main \({main}\) does not contain"): + prepare(docs, sources, "oss", "0.98.0", "--enterprise", "v8.1.0", "--docs", ref) + assert state(docs) == {} + + # protecting saved editions From 7452caf4a50079302abf875aa8db54ac9e79807f Mon Sep 17 00:00:00 2001 From: Brendan Clement Date: Tue, 20 Oct 2026 16:50:24 -0700 Subject: [PATCH 14/19] docs: clarify the edition check's credentials and review prerequisite The CI step fetches its comparison commit with the job's GitHub token; `edition.py check` itself is offline and needs no Enterprise repository credentials. Alongside requiring the check before merging, changes to the checker and its workflow need enforced review, such as required code-owner review of scripts/, the Makefile and .github/workflows/. --- README.md | 22 +++++++++++++--------- VERSIONING.md | 14 +++++++++----- 2 files changed, 22 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 78eae375..6d10932a 100644 --- a/README.md +++ b/README.md @@ -195,7 +195,7 @@ Publishing relies on settings outside this repository: Contents: read. Checkout does not persist it, and the token is revoked when the job ends. Fork pull requests receive no App secret and cannot run the combined build; the assembler guard tests and the saved-edition check still - run without credentials. + run without Enterprise repository credentials. Of these, the Publish workflow checks only that production runs from `main`, that the environment supplies the deploy key and that `MINTLIFY_CONTENT_DIR` is @@ -351,16 +351,20 @@ still be there. Every field of a record except `corrections` must be unchanged, and its existing corrections must be unchanged and in order, with new ones only after them. A new edition is checked by the rules above. -The test job of the Assemble workflow runs `make check-editions` with a base, -offline and without credentials: for a pull request, the base commit its event -names, and for a push to `main`, the commit `main` pointed at before. It -fetches that commit by hash. A manual run of the workflow has no base, and -checks each edition against its record only. +The test job of the Assemble workflow runs `make check-editions` with a base: +for a pull request, the base commit its event names, and for a push to `main`, +the commit `main` pointed at before. The job first fetches that commit by hash +with its own GitHub token; `edition.py check` then runs offline and needs no +Enterprise repository credentials. A manual run of the workflow has no base, +and checks each edition against its record only. This keeps saved editions only while the check must pass before a pull request -merges, which is a repository rule set up separately. The check on a push to -`main` reports a violation in that push but cannot undo it, and a later push -does not find it, because its base already includes the change. +merges, and while changes to the checker and its workflow need enforced review, +such as required code-owner review of `scripts/`, the `Makefile` and +`.github/workflows/`, since a pull request runs its own copy of the check. Both +are repository rules set up separately. The check on a push to `main` reports +a violation in that push but cannot undo it, and a later push does not find it, +because its base already includes the change. ### Correcting a saved edition diff --git a/VERSIONING.md b/VERSIONING.md index 16130197..cda43b6e 100644 --- a/VERSIONING.md +++ b/VERSIONING.md @@ -116,11 +116,15 @@ Publication checksums describe the corrected candidate. A pull request may add editions, and add corrections at the end of a record. It may not remove an edition or a record, change a record's release, sources or prepared snapshot, or change an earlier correction. The Assemble workflow's -test job checks this offline, on every pull request against its base commit -and on every push to `main`, by comparing the records' parsed values. That -protects saved editions only while the check is required before merging, a -repository rule set up separately. On a push to `main` it can report a -violation but not undo it, and later pushes no longer see it. +test job checks this on every pull request against its base commit and on +every push to `main`, by comparing the records' parsed values. The job fetches +that commit with its own GitHub token; the check itself is offline and needs +no Enterprise repository credentials. That protects saved editions only while +the check is required before merging and changes to the checker and its +workflow need enforced review, such as required code-owner review of +`scripts/`, the `Makefile` and `.github/workflows/`. Both are repository rules +set up separately. On a push to `main` the check can report a violation but +not undo it, and later pushes no longer see it. Preparation never overwrites a saved edition: From 4f2d15e95eb7168109f9e4617d49fd446d4b8fec Mon Sep 17 00:00:00 2001 From: Brendan Clement Date: Tue, 20 Oct 2026 19:44:07 -0700 Subject: [PATCH 15/19] docs: use the full main ref in local edition checks --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 6d10932a..983dc8aa 100644 --- a/README.md +++ b/README.md @@ -283,7 +283,7 @@ against `main`, and commit the edition and its record together: ```bash (cd editions/enterprise/1.10.0 && mint validate && mint broken-links --check-anchors --check-redirects) -uv run --no-project --with pyyaml scripts/edition.py check --base origin/main +uv run --no-project --with pyyaml scripts/edition.py check --base refs/remotes/origin/main ``` A record looks like this. It is illustrative: the Enterprise repository is a @@ -390,7 +390,7 @@ A saved edition can be corrected after review, at the same URL: sha256: 3f9c2a7e… ``` -4. Run the checks above, with `--base origin/main` after fetching `main`. +4. Run the checks above, with `--base refs/remotes/origin/main` after fetching `main`. `edition.py check` now passes: each edition matches the checksum of its latest correction. Any later change to the edition, including a deleted file, fails again until it is recorded the same way, as a further entry. From 1be275c78e0a774d285d3321f0e7c02c80cb1109 Mon Sep 17 00:00:00 2001 From: Brendan Clement Date: Thu, 22 Oct 2026 12:59:41 -0700 Subject: [PATCH 16/19] docs: simplify the versioning design --- VERSIONING.md | 335 +++++++++++++------------------------------------- 1 file changed, 88 insertions(+), 247 deletions(-) diff --git a/VERSIONING.md b/VERSIONING.md index cda43b6e..b27151c5 100644 --- a/VERSIONING.md +++ b/VERSIONING.md @@ -1,252 +1,93 @@ # Versioned documentation -Readers should find the documentation for the product version they use. -Engineers keep writing documentation beside the product code, in -`lancedb/lancedb` and the Enterprise documentation repository. Release -automation prepares each version's documentation, and people review it before -it is merged and before it reaches production. - -This is the design that work follows. It is built in four steps: - -1. **Release records and saved editions.** A record names one release's - sources, and a manual command saves the edition built from them. This step - is in place: see [Saved editions](README.md#saved-editions). -2. **Composing editions into the site:** version paths, links, navigation, - selectors, redirects and an "About this version" page for each edition. -3. **Supervised publication:** merged editions and corrections go to staging - automatically, and a person approves promoting that same candidate to - production. -4. **Release discovery:** each product's own qualification signal starts the - same preparation, and a bot opens the documentation pull request. - -Until steps 2 and 3 are in place and a cutover is approved, the site is built -and published as it is today, and production stays where it is. - -## Decisions - -| Topic | Decision | -| --- | --- | -| Hosting | Mintlify, with its native product and version selectors. | -| Default | The latest approved, published Enterprise edition. | -| Unversioned links | Resolve to the same page in that edition. Versioned links stay in their edition. | -| Navigation | Every edition keeps the full documentation set, with OSS and Enterprise badges. | -| Release cadence | OSS and Enterprise editions publish independently. An Enterprise release needs no new OSS release. | -| Enterprise sources | An Enterprise edition uses its own pages and the shared pages of the exact LanceDB it bundles, which may be an older release or a commit no release tags. | -| History | No backfill. Editions start with releases that have their documentation sources. | -| Storage | One folder per edition, all on one branch of this repository. | -| Corrections | A reviewed edit to a saved edition is allowed for a historical correction. Fix the current source too when the error is still there. | -| Automation | Preparation, checks, pull requests and staging are automated. People review before merging and approve production promotion. | -| Missing page on version switch | Mintlify's native fallback, with no custom switching logic. | - -"Latest" means the approved preferred edition of a product, not whichever job -finished last. A later maintenance release must not replace a newer preferred -release line. Selecting the latest Enterprise edition does not select the -latest OSS edition: the LanceDB it bundles can be older. - -## Components - -```mermaid -flowchart TD - L["lancedb/lancedb
Shared pages and examples"] --> B - S["Enterprise source
Enterprise pages and bundled LanceDB"] --> B - D["This repository
Docs-owned pages"] --> B - R["Release record
Version and exact source commits"] --> B - B["Edition builder
Saves a new edition"] --> E - E["Saved editions"] - C["Reviewed correction"] --> E - E --> A["Site composition
Version paths, links and navigation"] - N["Shared site configuration
Branding and preferred editions"] --> A - A --> V["Checked site candidate"] - V --> T["Staging"] - T --> H["Production approval"] - H --> P["Promote the same candidate"] -``` - -These are parts of the existing tooling, not new services. - -- **Edition builder** (`scripts/edition.py`) reads the recorded commits, - builds the edition with the existing assembler, and saves it. The assembler - stays where pages are composed: overlays, examples written into pages, - availability badges and native drafts all come from it. -- **Saved edition**: the pages, assets, navigation and draft controls of one - release. Later builds never fetch fresh content into it. -- **Site composition** gives each saved edition its version path and builds - the navigation and redirects. It does not rebuild editions. -- **Publisher** puts the checked candidate on staging and, after approval, the - same candidate on production. - -Global branding and Mintlify's renderer change independently of editions, so -a saved edition keeps its content, not its exact pixels. - -## Editions and their records - -```text -releases/ - oss-.yaml - enterprise-.yaml -editions/ - oss// - enterprise// -``` - -A record names the product and version, the release tag and commit, the -Enterprise qualification tag, the LanceDB an Enterprise release bundles, the -commit of every root the assembler read, and the specification's upstream -release. - -An edition's sources are fixed when it is prepared: - -- **Product pages** come from the release commits: the OSS release, or the - Enterprise release and the LanceDB it bundles. -- **Docs-owned content**, the Geneva pages, the dataset cards and the REST API - specification, comes from the docs input commit: the commit of this - repository's `main` that preparation read, which the record names with the - others. It is `main` as it was when the edition was prepared, not when the - product was released. -- Later changes to `main`, or to a product's sources, never reach a saved - edition, so older editions never inherit today's pages. - -Saved editions are kept by default but are not immutable. A reviewed -correction changes an edition's files at the same URL. The record keeps the -original sources and the checksum of the snapshot as prepared, and lists each -correction with the checksum of the edition it accepted; the saved edition -must match the latest one. Git history holds the correction pull requests. -Publication checksums describe the corrected candidate. - -A pull request may add editions, and add corrections at the end of a record. -It may not remove an edition or a record, change a record's release, sources -or prepared snapshot, or change an earlier correction. The Assemble workflow's -test job checks this on every pull request against its base commit and on -every push to `main`, by comparing the records' parsed values. The job fetches -that commit with its own GitHub token; the check itself is offline and needs -no Enterprise repository credentials. That protects saved editions only while -the check is required before merging and changes to the checker and its -workflow need enforced review, such as required code-owner review of -`scripts/`, the `Makefile` and `.github/workflows/`. Both are repository rules -set up separately. On a push to `main` the check can report a violation but -not undo it, and later pushes no longer see it. - -Preparation never overwrites a saved edition: - -- The same release prepared again from the same release commits and - qualification tag changes nothing, even after the edition has been - corrected. A later commit of this repository's `main` does not make it a - different release. -- Different inputs for a saved release, such as a moved tag, another - Enterprise source, or a qualification tag deleted and created again, stop - preparation for review. A recreated qualification tag is new qualification - evidence, not a duplicate. -- A failed preparation leaves no edition and no record. - -## Preparing a release - -The manual command and later automatic discovery run the same preparation. -Nobody assembles files by hand. - -**Enterprise.** Preparation takes the Enterprise release tag, which must -be qualified, reads the LanceDB commit that release bundles, and takes the -shared pages from that commit. A standalone OSS edition is not needed. - -**OSS.** Preparation takes the `lancedb/lancedb` release tag. The full -navigation includes the Enterprise pages, so the Enterprise tag or commit -they come from is named explicitly. Naming it pins the source; it does not -decide that the source may be published with that OSS release, which is still -open. Nothing reads the Enterprise repository's `main` or infers a -compatibility relationship. - -**This repository.** The docs-owned pages and `assemble.yaml` are read from a -commit of `main`, never from a branch: squash merges give a branch's commits -new hashes, so only a commit of `main` stays recoverable. Preparation fetches -`main` itself, from a clone whose origin is `lancedb/docs`, because a fork's -`main` can hold commits `lancedb/docs` never has. The assembler that runs must -be that commit's. - -A missing tag, absent documentation, an unqualified or inconsistently tagged -Enterprise release, or private-source files that may not be published stop -preparation with the reason. Nothing is made up from today's sources. - -The bundled LanceDB is not a client SDK compatibility matrix. Client minimum -versions belong beside the examples they apply to, when evidence supports -them, and the tested client version is not a claimed minimum. +Readers should be able to find the docs for the version of LanceDB they run. +The docs are still written next to the code, in `lancedb/lancedb` and the +Enterprise documentation repository. What's new is that each release's docs are +saved when it ships, and the site lets readers switch between those saved +versions. + +Release records and saved editions are in place; see +[Saved editions](README.md#saved-editions). Composing the site, publishing it +and finding new releases come next. Until a cutover is approved, the site is +built and published as it is today. + +## How it works + +Each release gets a saved edition: the site the assembler built from that +release's sources, stored in `editions/oss//` or +`editions/enterprise//`. A record in `releases/` lists the commits it +came from. An edition is built once. Later changes to `main` or to a product's +docs don't reach it, so older versions keep the pages they shipped with. + +The published site is put together from the saved editions. Each edition gets +its own path, such as `/enterprise/v1-10-0/`, and readers switch products and +versions with Mintlify's built-in selectors. Each version also has an About +this version page listing its release and sources. Branding, the banner, the +footer and integrations are shared across versions. + +A composed site goes to staging first, and production gets the same build after +someone approves it. + +New editions arrive as pull requests. They can be prepared by hand with +`scripts/edition.py prepare`, and a discovery job prepares one for each new +qualified release. People review and merge every one. + +## What goes into an edition + +An Enterprise edition has the Enterprise release's own pages, plus the shared +pages from the exact LanceDB commit that release bundles. That commit can be +older than the latest OSS release, or one that no OSS release tags. + +An OSS edition has the pages from its `lancedb/lancedb` release tag. The +navigation also covers the Enterprise pages, so an OSS edition takes them from +an Enterprise release that's named when the edition is prepared. Discovery uses +the preferred Enterprise edition. + +Pages owned by this repository, like the Geneva pages, dataset cards and REST +API reference, come from `main` as it was when the edition was prepared. + +Every edition keeps the full navigation, with badges showing what's available +in OSS and in Enterprise. OSS and Enterprise editions ship on their own +schedules, and an Enterprise release doesn't need a matching OSS release. + +Only stable releases get editions. An Enterprise release counts once its +release tag and its `release-qualified/` tag point at the same commit. An OSS +release counts once its GitHub release is published and isn't marked as a +prerelease. Betas and release candidates are skipped, and older releases aren't +backfilled. + +## Which version readers see + +Each product has a preferred edition, which readers get by default. New +visitors land on the preferred Enterprise edition, and links without a version +go to the same page there. Links that include a version stay in that version. + +The preferred edition only changes when someone updates it in a reviewed pull +request. A new release, or a patch to an older release line, doesn't change it. + +If an old link has no matching page in the default edition, it gets no +redirect. Composition lists those links so someone can add the page or a +specific redirect. + +When a reader switches to a version that doesn't have the page they're on, +Mintlify decides where they land. ## Corrections -```mermaid -flowchart LR - E["Error in a published edition"] --> F["Edit the saved files"] - F --> PR["Correction PR names the editions"] - PR --> C["Checks and review"] - C --> S["Corrected candidate on staging"] - S --> A["Approve production promotion"] - F --> U["Fix the current source too
if the error is still there"] -``` - -A correction to a saved Enterprise edition edits its folder and keeps its -version and URL. It needs no product release and no historical source branch. - -Corrections are explicit. If three editions are wrong, the pull request names -and edits all three. Checking a corrected example fits the example; a typo -needs no SDK test suite. Preparing a release again never rebuilds a corrected -edition from its original sources. - -## Reader behavior - -Site composition will provide this: - -- First-time readers land on the latest published Enterprise edition. -- Unversioned links go to the same page in that edition. Redirects come from - the actual page map and keep valid anchors; paths are not blindly prefixed. -- Versioned links stay in their edition. -- Readers can choose the newest OSS edition independently. -- Selectors use short labels. An "About this version" page shows the full - release identity and its sources. -- Version switching is Mintlify's own. In a hosted trial, switching from a - page that only a newer version has landed on another page of the older - version, without explanation. That is Mintlify's fallback, not a promise. -- Test versions and probe pages never ship as editions. - -An old unversioned URL with no matching page still needs a policy. A -catch-all redirect that hides missing content is not one. - -## Publication and discovery - -At first the preparation bot opens pull requests and does not merge them. A -merged edition or correction goes to staging. Production needs a person's -approval of the candidate that was reviewed. A stale approval cannot promote a -different candidate, and an older queued run cannot overwrite a newer -approved publication or change the preferred edition. Promotion reuses the -staged artifact. - -Missing qualification, absent sources, ambiguous inputs or failed checks stop -the workflow with a report. A duplicate event, for a release already saved -from the same commits and qualification tag, is harmless. - -Rolling back changes the preferred edition and keeps published folders. -Restoring an earlier whole-site build would remove newer editions, so it needs -a deliberate decision. - -Merging and production promotion can later become automatic, separately. -That changes approval policy, not the builder or the publisher. - -## Open questions - -These are settled where they first matter, before production: - -1. Each product's qualification signal for discovery. Enterprise releases - carry a qualification tag, which preparation already requires. For OSS, - the stable-release completion signal, beta and release-candidate handling, - and the release owner's approval are still to be confirmed. -2. Which Enterprise source supplies the Enterprise pages of an OSS edition. - Preparation takes it explicitly until this is decided. -3. Missing targets for legacy URLs. -4. Approval enforcement in the repository, and the supported Mintlify staging - setup. -5. Preferred-release order across stable and maintenance lines. - -**Enterprise pages without placeholders.** The assembler currently needs an -OSS page at the same path for every Enterprise page. Removing that -requirement, with its navigation and draft handling, is a separate change that -must land before the first real editions. Editions save whatever the -assembler builds, so they need no change for it. Once it lands, prepare test -editions again and check Enterprise-only pages, intentional overlays, -navigation and hidden drafts together. +A saved edition can be corrected through a reviewed pull request that edits its +files and records the correction. The version and its URLs stay the same. If +the mistake is still in the current docs, fix it there too. Preparing the same +release again keeps the correction. + +## Publishing + +Merged editions and corrections go to staging on their own. Production gets the +build staging shows, after approval in the `production` GitHub environment, and +promotion publishes that build without rebuilding it. An older run can't +replace a newer approved one. + +To roll back, change the preferred edition. Published versions stay where they +are. + +Merging and production promotion could become automatic later, each as its own +decision. From b72df4056c40a6ffbda798e939a8099af0bc105e Mon Sep 17 00:00:00 2001 From: Brendan Clement Date: Thu, 22 Oct 2026 13:55:25 -0700 Subject: [PATCH 17/19] refactor: name the Enterprise repository option explicitly --- README.md | 4 ++-- scripts/edition.py | 14 +++++++------- scripts/tests/test_edition.py | 4 ++-- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 983dc8aa..52f75d90 100644 --- a/README.md +++ b/README.md @@ -238,8 +238,8 @@ command in a clone of this repository whose `origin` is a branch whose `scripts/` are those of `main`: ```bash -uv run --no-project --with pyyaml scripts/edition.py prepare enterprise 1.10.0 --lancedb ../lancedb --sophon ../sophon -uv run --no-project --with pyyaml scripts/edition.py prepare oss 0.41.0 --enterprise v1.10.0 --lancedb ../lancedb --sophon ../sophon +uv run --no-project --with pyyaml scripts/edition.py prepare enterprise 1.10.0 --lancedb ../lancedb --enterprise-repo ../enterprise +uv run --no-project --with pyyaml scripts/edition.py prepare oss 0.41.0 --enterprise v1.10.0 --lancedb ../lancedb --enterprise-repo ../enterprise ``` - **Enterprise** reads the Enterprise release tag `v1.10.0`, which must have an diff --git a/scripts/edition.py b/scripts/edition.py index 7b1df1dc..a4f8aff6 100644 --- a/scripts/edition.py +++ b/scripts/edition.py @@ -9,10 +9,10 @@ Usage: - python scripts/edition.py prepare oss VERSION --enterprise REF [--lancedb REPO] [--sophon REPO] + python scripts/edition.py prepare oss VERSION --enterprise REF [--lancedb REPO] [--enterprise-repo REPO] [--docs REF] python scripts/edition.py prepare enterprise VERSION [--bundled-lancedb COMMIT] - [--lancedb REPO] [--sophon REPO] [--docs REF] + [--lancedb REPO] [--enterprise-repo REPO] [--docs REF] python scripts/edition.py check [--base COMMIT] """ @@ -238,7 +238,7 @@ def resolve_release(args: argparse.Namespace, repo: Path) -> dict: product, version = args.product, args.version if not VERSION_RE.fullmatch(version): raise EditionError(f"{version!r} is not a stable release version such as 1.2.3") - lancedb, sophon = args.lancedb.resolve(), args.sophon.resolve() + lancedb, sophon = args.lancedb.resolve(), args.enterprise_repo.resolve() if product == "oss": if args.bundled_lancedb: raise EditionError("--bundled-lancedb applies to Enterprise releases only") @@ -647,7 +647,7 @@ def prepare(args: argparse.Namespace, repo: Path = REPO_ROOT) -> str: for name, (repository, folder) in SOURCES.items(): roots[name] = staging / "roots" / name commit = requested["sources"][name]["commit"] - source_repo = {"lancedb": args.lancedb, "enterprise": args.sophon}.get(name, repo) + source_repo = {"lancedb": args.lancedb, "enterprise": args.enterprise_repo}.get(name, repo) export(source_repo.resolve(), commit, folder, roots[name], f"{repository} {commit}") site = staging / "site" build = requested["sources"]["build"]["commit"] @@ -790,13 +790,13 @@ def parse(argv: list[str] | None = None) -> argparse.Namespace: prep.add_argument("product", choices=PRODUCTS) prep.add_argument("version", help="the release version, such as 1.2.3") prep.add_argument("--enterprise", metavar="REF", - help="OSS only: the lancedb/sophon tag or commit for the Enterprise pages") + help="OSS only: the Enterprise tag or commit for the Enterprise pages") prep.add_argument("--bundled-lancedb", metavar="COMMIT", help="Enterprise only: the LanceDB commit the release is expected to bundle") prep.add_argument("--lancedb", type=Path, default=REPO_ROOT.parent / "lancedb", help="a lancedb/lancedb repository (default: ../lancedb)") - prep.add_argument("--sophon", type=Path, default=REPO_ROOT.parent / "sophon", - help="a lancedb/sophon repository (default: ../sophon)") + prep.add_argument("--enterprise-repo", type=Path, default=REPO_ROOT.parent / "enterprise", + help="an Enterprise source repository (default: ../enterprise)") prep.add_argument("--docs", metavar="REF", help="the commit of lancedb/docs main to read docs/ and assemble.yaml from " "(default: main as fetched from origin when preparing)") diff --git a/scripts/tests/test_edition.py b/scripts/tests/test_edition.py index 5f794ed1..3fc25bcf 100644 --- a/scripts/tests/test_edition.py +++ b/scripts/tests/test_edition.py @@ -255,7 +255,7 @@ def on_main(docs: Path) -> str: def prepare(docs: Path, sources: dict, *argv: str, lancedb: Path | None = None) -> str: args = edition.parse([ "prepare", *argv, - "--lancedb", str(lancedb or sources["lancedb"]), "--sophon", str(sources["sophon"]), + "--lancedb", str(lancedb or sources["lancedb"]), "--enterprise-repo", str(sources["sophon"]), ]) return edition.prepare(args, repo=docs) @@ -779,7 +779,7 @@ def run(*argv: str) -> subprocess.CompletedProcess: cwd=docs, env=environment(), capture_output=True, text=True, ) - repos = ["--lancedb", str(sources["lancedb"]), "--sophon", str(sources["sophon"])] + repos = ["--lancedb", str(sources["lancedb"]), "--enterprise-repo", str(sources["sophon"])] result = run("prepare", "oss", "0.98.0", "--enterprise", "v8.1.0", *repos) assert result.returncode == 0, result.stderr assert record(docs, "oss-0.98.0")["sources"]["build"]["commit"] == main From db144b3325850e79f0bc92eb276e1eed8736fef9 Mon Sep 17 00:00:00 2001 From: Brendan Clement Date: Thu, 22 Oct 2026 15:42:59 -0700 Subject: [PATCH 18/19] fix: keep repeat edition preparation independent of the assembler --- README.md | 10 +++-- VERSIONING.md | 11 +++--- scripts/edition.py | 21 +++++++---- scripts/tests/test_edition.py | 70 +++++++++++++++++++++++++++++++++-- 4 files changed, 92 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 52f75d90..d562efb3 100644 --- a/README.md +++ b/README.md @@ -250,9 +250,9 @@ uv run --no-project --with pyyaml scripts/edition.py prepare oss 0.41.0 --enterp hash, stops preparation if the release bundles a different one. - **OSS** reads lancedb's tag `v0.41.0`. The Enterprise pages come from the Enterprise tag or full commit hash named with `--enterprise`; a branch name - is not accepted, so the record pins an exact commit. Pinning a commit does - not decide that it may be published with that OSS release, which is not - settled yet; see [Open questions](VERSIONING.md#open-questions). + is not accepted, so the record pins an exact commit. The named Enterprise + release supplies the Enterprise pages in that OSS edition; see + [What goes into an edition](VERSIONING.md#what-goes-into-an-edition). - The LanceDB and Enterprise roots are read from Git at their commits, never from a working tree, and only from `docs/web`. - Preparation fetches `main` from `origin` itself, and reads this @@ -403,7 +403,9 @@ A saved edition can be corrected after review, at the same URL: whether each correction is right; the checksum shows only that the record describes the files. -Preparing that release again later keeps the edition and its corrections. +Preparing that release again later checks its release inputs and saved files, +then keeps the edition and its corrections. It does not fetch docs main or +require matching assembler scripts unless it is building a new edition. ## Code snippets diff --git a/VERSIONING.md b/VERSIONING.md index b27151c5..dffabc97 100644 --- a/VERSIONING.md +++ b/VERSIONING.md @@ -2,9 +2,10 @@ Readers should be able to find the docs for the version of LanceDB they run. The docs are still written next to the code, in `lancedb/lancedb` and the -Enterprise documentation repository. What's new is that each release's docs are -saved when it ships, and the site lets readers switch between those saved -versions. +Enterprise documentation repository. Each release gets a saved edition that +combines its product pages, the selected shared or Enterprise pages, and pages +owned by this repository. These come from separately recorded commits. Readers +can switch between editions on the site. Release records and saved editions are in place; see [Saved editions](README.md#saved-editions). Composing the site, publishing it @@ -13,8 +14,8 @@ built and published as it is today. ## How it works -Each release gets a saved edition: the site the assembler built from that -release's sources, stored in `editions/oss//` or +Each edition is the site the assembler built from those sources, stored in +`editions/oss//` or `editions/enterprise//`. A record in `releases/` lists the commits it came from. An edition is built once. Later changes to `main` or to a product's docs don't reach it, so older versions keep the pages they shipped with. diff --git a/scripts/edition.py b/scripts/edition.py index a4f8aff6..00cbefd9 100644 --- a/scripts/edition.py +++ b/scripts/edition.py @@ -196,15 +196,17 @@ def docs_main(repo: Path) -> str: others = [url for url in urls if not DOCS_URLS.fullmatch(url)] if not urls or others: raise EditionError( - f"this checkout's {DOCS_REMOTE} is {', '.join(others) if urls else 'not set'}, not " + f"this checkout's {DOCS_REMOTE} is {'not supported' if urls else 'not set'}, not " f"{SOURCES['build'][0]}; a fork's main can hold commits {SOURCES['build'][0]} never has, " "so prepare an edition in a clone of https://github.com/lancedb/docs.git, over HTTPS " "or SSH" ) try: git(repo, "fetch", "--quiet", "--no-tags", DOCS_REMOTE, f"+refs/heads/main:{DOCS_MAIN}") - except EditionError as exc: - raise EditionError(f"could not fetch {SOURCES['build'][0]} main: {exc}") from None + except EditionError: + raise EditionError( + f"could not fetch {SOURCES['build'][0]} main; check network access and Git authentication" + ) from None return commit_of(repo, DOCS_MAIN, SOURCES["build"][0]) @@ -233,8 +235,7 @@ def docs_commit(repo: Path, ref: str | None) -> str: return commit -def resolve_release(args: argparse.Namespace, repo: Path) -> dict: - """The record a preparation would write, before anything is built.""" +def release_inputs(args: argparse.Namespace) -> dict: product, version = args.product, args.version if not VERSION_RE.fullmatch(version): raise EditionError(f"{version!r} is not a stable release version such as 1.2.3") @@ -306,11 +307,16 @@ def resolve_release(args: argparse.Namespace, repo: Path) -> dict: "sources": { "lancedb": source("lancedb", shared), "enterprise": source("enterprise", enterprise), - "build": source("build", docs_commit(repo, args.docs)), }, } +def resolve_release(args: argparse.Namespace, repo: Path) -> dict: + requested = release_inputs(args) + requested["sources"]["build"] = source("build", docs_commit(repo, args.docs)) + return requested + + def identity(record: dict) -> dict: """Exclude docs main so preparing again after it advances keeps the saved edition.""" release = record["release"] @@ -634,12 +640,13 @@ def place(staging: Path, site: Path, record: str, repo: Path, edition: Path, pat def prepare(args: argparse.Namespace, repo: Path = REPO_ROOT) -> str: - requested = resolve_release(args, repo) + requested = release_inputs(args) edition = repo / requested["edition"] path = record_path(repo, requested["product"], requested["version"]) if os.path.lexists(edition) or os.path.lexists(path): return existing(repo, requested) + requested["sources"]["build"] = source("build", docs_commit(repo, args.docs)) (repo / "build").mkdir(exist_ok=True) staging = Path(tempfile.mkdtemp(prefix="edition-", dir=repo / "build")).resolve() try: diff --git a/scripts/tests/test_edition.py b/scripts/tests/test_edition.py index 3fc25bcf..3eaf3b77 100644 --- a/scripts/tests/test_edition.py +++ b/scripts/tests/test_edition.py @@ -511,7 +511,7 @@ def test_docs_inputs_off_main_or_unlike_the_running_scripts_are_refused(docs, so prepare(docs, sources, *oss) git(docs, "remote", "set-url", "origin", str(docs.parent / "missing/lancedb-docs.git")) - with pytest.raises(edition.EditionError, match="could not fetch lancedb/docs main: git fetch .* failed"): + with pytest.raises(edition.EditionError, match="could not fetch lancedb/docs main; check network access and Git authentication"): prepare(docs, sources, *oss) git(docs, "remote", "remove", "origin") with pytest.raises(edition.EditionError, match="this checkout's origin is not set, not lancedb/docs"): @@ -527,7 +527,7 @@ def test_only_the_lancedb_docs_urls_are_accepted_as_origin(docs, monkeypatch): "https://github.com/LanceDB/docs.git", "git@github.com:lancedb/docs.git", "ssh://git@github.com/lancedb/docs.git"): git(docs, "remote", "set-url", "origin", url) - with pytest.raises(edition.EditionError, match="could not fetch lancedb/docs main: .*not allowed"): + with pytest.raises(edition.EditionError, match="could not fetch lancedb/docs main; check network access and Git authentication"): edition.docs_main(docs) for url in ("https://github.com/someone/docs.git", "git@github.com:someone/docs.git", "https://github.com/lancedb/docs-fork.git", "https://github.com/lancedb/docs.git.old", @@ -535,10 +535,45 @@ def test_only_the_lancedb_docs_urls_are_accepted_as_origin(docs, monkeypatch): "https://example.com/lancedb/docs.git", "file:///srv/lancedb/docs.git", "/srv/git/lancedb-docs.git"): git(docs, "remote", "set-url", "origin", url) - with pytest.raises(edition.EditionError, match=f"this checkout's origin is {re.escape(url)}, not"): + with pytest.raises(edition.EditionError, match="this checkout's origin is not supported, not"): edition.docs_main(docs) + +@pytest.mark.parametrize("rewritten", [False, True]) +def test_rejected_remote_does_not_print_credentials(docs, monkeypatch, rewritten): + secret = "fixture-only-remote-password" + remote = f"https://user:{secret}@github.com/lancedb/docs.git" + monkeypatch.setattr(edition, "DOCS_URLS", LANCEDB_DOCS_URLS) + if rewritten: + git(docs, "remote", "set-url", "origin", "https://github.com/lancedb/docs.git") + git(docs, "config", f"url.{remote}.insteadOf", "https://github.com/lancedb/docs.git") + else: + git(docs, "remote", "set-url", "origin", remote) + with pytest.raises(edition.EditionError, match="origin is not supported") as caught: + edition.docs_main(docs) + assert secret not in str(caught.value) + assert remote not in str(caught.value) + + +def test_fetch_failure_does_not_print_remote_diagnostics(docs, monkeypatch): + secret = "fixture-only-fetch-password" + run = subprocess.run + + def fail_fetch(argv, **kwargs): + if argv[:4] == ["git", "-C", str(docs), "fetch"]: + return subprocess.CompletedProcess(argv, 128, b"", ( + f"fatal: unable to access 'https://user:{secret}@example.com/docs.git': denied" + ).encode()) + return run(argv, **kwargs) + + monkeypatch.setattr(subprocess, "run", fail_fetch) + with pytest.raises(edition.EditionError, match="check network access and Git authentication") as caught: + edition.docs_main(docs) + assert secret not in str(caught.value) + assert "example.com" not in str(caught.value) + + def test_a_fork_origin_is_refused_before_anything_is_fetched_or_saved(docs, sources): """The usual fork layout: origin is a fork, upstream is lancedb/docs, and the fork's main has a docs-owned change lancedb/docs does not have.""" @@ -552,7 +587,7 @@ def test_a_fork_origin_is_refused_before_anything_is_fetched_or_saved(docs, sour assert not edition.is_ancestor(docs, fork_main, "upstream/main") refs = git(docs, "for-each-ref") - refused = rf"this checkout's origin is {re.escape(str(fork))}, not lancedb/docs; a fork's main" + refused = r"this checkout's origin is not supported, not lancedb/docs; a fork's main" with pytest.raises(edition.EditionError, match=refused): prepare(docs, sources, "oss", "0.98.0", "--enterprise", "v8.1.0") with pytest.raises(edition.EditionError, match=refused): @@ -658,6 +693,33 @@ def test_conflicting_inputs_for_a_saved_release_stop(docs, sources, tmp_path): assert state(docs) == before + +@pytest.mark.parametrize("scripts_state", ["uncommitted", "branch", "main"]) +def test_saved_edition_does_not_need_the_current_assembler(docs, sources, monkeypatch, scripts_state): + prepare(docs, sources, "oss", "0.98.0", "--enterprise", "v8.1.0") + correct(docs, "oss-0.98.0", "guide.mdx", "Corrected guide\n") + before = state(docs) + (docs / "scripts").mkdir() + (docs / "scripts/assemble.py").write_text("# a later assembler\n") + if scripts_state != "uncommitted": + commit_all(docs, "change assembler") + if scripts_state == "main": + on_main(docs) + + def must_not_fetch(_repo): + pytest.fail("an existing edition must not fetch docs main") + + monkeypatch.setattr(edition, "docs_main", must_not_fetch) + out = prepare(docs, sources, "oss", "0.98.0", "--enterprise", "v8.1.0") + assert "is corrected (1 correction); nothing changed" in out + assert state(docs) == before + (docs / "editions/oss/0.98.0/guide.mdx").write_text("Unrecorded edit\n") + with pytest.raises(edition.EditionError, match="has changed since its latest recorded correction"): + prepare(docs, sources, "oss", "0.98.0", "--enterprise", "v8.1.0") + with pytest.raises(edition.EditionError, match="enterprise_pages recorded"): + prepare(docs, sources, "oss", "0.98.0", "--enterprise", "v8.2.0") + + def test_duplicate_preparation_changes_nothing_even_after_a_correction(docs, sources): prepare(docs, sources, "oss", "0.98.0", "--enterprise", "v8.1.0") out = prepare(docs, sources, "oss", "0.98.0", "--enterprise", "v8.1.0") From 37794992208d2abf07cbd2b13f872279e607ea23 Mon Sep 17 00:00:00 2001 From: Brendan Clement Date: Thu, 22 Oct 2026 15:49:04 -0700 Subject: [PATCH 19/19] refactor: use Enterprise naming outside integration identifiers --- .github/workflows/assemble.yml | 12 ++++---- .github/workflows/docs-ci.yml | 8 ++--- .github/workflows/prose-style.yml | 2 +- Makefile | 2 +- assemble.yaml | 2 +- scripts/assemble.py | 4 +-- scripts/edition.py | 22 +++++++------- scripts/tests/test_assemble.py | 30 +++++++++---------- scripts/tests/test_candidate.py | 6 ++-- scripts/tests/test_edition.py | 30 +++++++++---------- skills/docs-writer/SKILL.md | 4 +-- workflows/docs-audit/manifests/indexing.toml | 2 +- .../docs-audit/manifests/namespaces.toml | 2 +- workflows/docs-audit/manifests/search.toml | 2 +- workflows/docs-audit/manifests/storage.toml | 2 +- .../manifests/table-operations.toml | 2 +- 16 files changed, 66 insertions(+), 66 deletions(-) diff --git a/.github/workflows/assemble.yml b/.github/workflows/assemble.yml index b6aff1e1..098689d9 100644 --- a/.github/workflows/assemble.yml +++ b/.github/workflows/assemble.yml @@ -14,7 +14,7 @@ on: description: lancedb/lancedb ref to read docs/web from default: main sophon_ref: - description: lancedb/sophon ref to read docs/web from + description: Enterprise ref to read docs/web from default: main permissions: @@ -72,8 +72,8 @@ jobs: sparse-checkout: docs/web persist-credentials: false - - name: Create a read-only Sophon token - id: sophon-token + - name: Create a read-only Enterprise token + id: enterprise-token uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 with: client-id: ${{ vars.SOPHON_DOCS_APP_CLIENT_ID }} @@ -88,7 +88,7 @@ jobs: repository: lancedb/sophon ref: ${{ inputs.sophon_ref || 'main' }} path: sophon - token: ${{ steps.sophon-token.outputs.token }} + token: ${{ steps.enterprise-token.outputs.token }} sparse-checkout: docs/web submodules: false persist-credentials: false @@ -147,10 +147,10 @@ jobs: env: SOURCES: ${{ steps.assemble.outputs.sources }} LANCEDB_REF: ${{ inputs.lancedb_ref || 'main' }} - SOPHON_REF: ${{ inputs.sophon_ref || 'main' }} + ENTERPRISE_REF: ${{ inputs.sophon_ref || 'main' }} run: | python3 scripts/candidate.py record build/site --sources "$SOURCES" \ - --ref "lancedb=$LANCEDB_REF" --ref "enterprise=$SOPHON_REF" \ + --ref "lancedb=$LANCEDB_REF" --ref "enterprise=$ENTERPRISE_REF" \ --output build/candidate.json --summary "$GITHUB_STEP_SUMMARY" - name: Upload the candidate diff --git a/.github/workflows/docs-ci.yml b/.github/workflows/docs-ci.yml index 55f965d2..874d1861 100644 --- a/.github/workflows/docs-ci.yml +++ b/.github/workflows/docs-ci.yml @@ -1,6 +1,6 @@ name: Docs Check -# The site is assembled from lancedb/lancedb, lancedb/sophon and this +# The site is assembled from LanceDB, Enterprise documentation and this # repository, so links are checked on the assembled tree. `docs/` alone is no # longer a site: checking it reported pages that live in the other roots as # missing. The Assemble workflow runs the same checks before it records a @@ -39,8 +39,8 @@ jobs: sparse-checkout: docs/web persist-credentials: false - - name: Create a read-only Sophon token - id: sophon-token + - name: Create a read-only Enterprise token + id: enterprise-token uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 with: client-id: ${{ vars.SOPHON_DOCS_APP_CLIENT_ID }} @@ -55,7 +55,7 @@ jobs: repository: lancedb/sophon ref: main path: sophon - token: ${{ steps.sophon-token.outputs.token }} + token: ${{ steps.enterprise-token.outputs.token }} sparse-checkout: docs/web submodules: false persist-credentials: false diff --git a/.github/workflows/prose-style.yml b/.github/workflows/prose-style.yml index f8f5735d..eabe31e0 100644 --- a/.github/workflows/prose-style.yml +++ b/.github/workflows/prose-style.yml @@ -4,7 +4,7 @@ name: Prose style # annotations on the lines a pull request adds or changes and never fail the # job; an install, sync or configuration error still does. Reviewers decide # everything else. The rules job tests the shared package that lancedb/lancedb -# and lancedb/sophon pin by release. +# and the Enterprise documentation pin by release. on: pull_request: diff --git a/Makefile b/Makefile index efa2221a..fb6b9f63 100644 --- a/Makefile +++ b/Makefile @@ -24,7 +24,7 @@ hf-sync: # Assemble the published tree from the roots declared in assemble.yaml into # build/site. Set LANCEDB_DOCS_ROOT and SOPHON_DOCS_ROOT when the lancedb and -# sophon checkouts are not beside this one. +# Enterprise checkouts are not beside this one. assemble: @$(ASSEMBLE_RUN) $(ASSEMBLE_SCRIPT) diff --git a/assemble.yaml b/assemble.yaml index 5af5978f..9316d83f 100644 --- a/assemble.yaml +++ b/assemble.yaml @@ -20,7 +20,7 @@ roots: role: reference # Enterprise pages, from the private repository's public `docs/web` folder; - # the rest of `sophon/docs` is internal and must never be read. Every path + # the rest of its `docs/` directory must never be read. Every path # here also exists in `lancedb`, which owns the navigation; an overlay page # replaces the reference page at the same path rather than adding one. # `private` refuses symlinks and hidden files under the root. Override the diff --git a/scripts/assemble.py b/scripts/assemble.py index a49da234..85dc886d 100644 --- a/scripts/assemble.py +++ b/scripts/assemble.py @@ -3,7 +3,7 @@ The site is not published straight from this repository. It is assembled here from the roots in `assemble.yaml` -- the open-source pages from -`lancedb/lancedb`, the Enterprise pages from `sophon` as whole-page overlays, and +`lancedb/lancedb`, the Enterprise pages as whole-page overlays, and this repository's `docs/`. CI checks each assembly and keeps it as a candidate; only the Publish workflow puts a chosen candidate on the `assembled` branch (see `candidate.py`). @@ -416,7 +416,7 @@ def merge_nav(base: dict, fragment: dict) -> dict: sidebar order is what a reader navigates by, and dropping Governance below Support or Datasets past Use Cases silently reorders the whole site. - This is the same shape sophon's Enterprise fragment uses. + This is the same shape the Enterprise fragment uses. """ def descend(node: dict | list, path: list[str]) -> list: diff --git a/scripts/edition.py b/scripts/edition.py index 00cbefd9..54c644eb 100644 --- a/scripts/edition.py +++ b/scripts/edition.py @@ -118,9 +118,9 @@ def pinned(repo: Path, ref: str, what: str) -> str: return tag(repo, ref, what)[1] -def bundled_lancedb(sophon: Path, commit: str, what: str) -> str: +def bundled_lancedb(enterprise_repo: Path, commit: str, what: str) -> str: """The LanceDB commit an Enterprise release bundles, from its gitlink.""" - listing = git(sophon, "ls-tree", "-z", commit, "--", BUNDLED_LANCEDB).decode() + listing = git(enterprise_repo, "ls-tree", "-z", commit, "--", BUNDLED_LANCEDB).decode() meta = listing.split("\t", 1)[0].split() if len(meta) != 3 or meta[0] != "160000": raise EditionError( @@ -239,18 +239,18 @@ def release_inputs(args: argparse.Namespace) -> dict: product, version = args.product, args.version if not VERSION_RE.fullmatch(version): raise EditionError(f"{version!r} is not a stable release version such as 1.2.3") - lancedb, sophon = args.lancedb.resolve(), args.enterprise_repo.resolve() + lancedb, enterprise_repo = args.lancedb.resolve(), args.enterprise_repo.resolve() if product == "oss": if args.bundled_lancedb: raise EditionError("--bundled-lancedb applies to Enterprise releases only") if not args.enterprise: raise EditionError( - "an OSS edition includes the Enterprise pages, so name the lancedb/sophon " + "an OSS edition includes the Enterprise pages, so name the Enterprise " "tag or commit they come from with --enterprise" ) release_tag = f"v{version}" commit = tag(lancedb, release_tag, f"lancedb/lancedb {release_tag}")[1] - enterprise = pinned(sophon, args.enterprise, "lancedb/sophon --enterprise") + enterprise = pinned(enterprise_repo, args.enterprise, "Enterprise --enterprise") release = { "repository": SOURCES["lancedb"][0], "tag": release_tag, @@ -269,22 +269,22 @@ def release_inputs(args: argparse.Namespace) -> dict: f"bundles, not {args.bundled_lancedb!r}" ) release_tag = f"v{version}" - what = f"lancedb/sophon {release_tag}" - enterprise = tag(sophon, release_tag, what)[1] + what = f"Enterprise {release_tag}" + enterprise = tag(enterprise_repo, release_tag, what)[1] qualified = QUALIFIED_TAG.format(version=version) try: - qualification, qualified_commit = tag(sophon, qualified, what) + qualification, qualified_commit = tag(enterprise_repo, qualified, what) except EditionError: raise EditionError( - f"{what} is not qualified: {sophon} has no tag {qualified}" + f"{what} is not qualified: {enterprise_repo} has no tag {qualified}" ) from None - if text(sophon, "cat-file", "-t", qualification) != "tag": + if text(enterprise_repo, "cat-file", "-t", qualification) != "tag": raise EditionError(f"{what}: {qualified} is not an annotated tag") if qualified_commit != enterprise: raise EditionError( f"{what} is {enterprise}, but {qualified} qualifies {qualified_commit}" ) - shared = bundled_lancedb(sophon, enterprise, what) + shared = bundled_lancedb(enterprise_repo, enterprise, what) if args.bundled_lancedb and args.bundled_lancedb != shared: raise EditionError( f"--bundled-lancedb {args.bundled_lancedb} disagrees with {what}, which " diff --git a/scripts/tests/test_assemble.py b/scripts/tests/test_assemble.py index 9feef853..a92d0a8e 100644 --- a/scripts/tests/test_assemble.py +++ b/scripts/tests/test_assemble.py @@ -3,7 +3,7 @@ Each test builds a small site in a temporary directory: a reference root that owns `enterprise/security` and the navigation, and a private overlay root at -`sophon/docs/web` beside an internal folder that must never be published. +`enterprise_repo/docs/web` beside an internal folder that must never be published. Run with `make test-assemble`. """ @@ -52,23 +52,23 @@ def write(path: Path, text: str) -> Path: def site(base: Path, *, checkout: bool = True) -> dict[str, Path]: - """A reference root, a Sophon-like checkout, and an internal folder.""" + """A reference root, an Enterprise checkout, and an internal folder.""" reference = base / "lancedb/docs/web" write(reference / "docs.json", json.dumps(NAVIGATION)) write(reference / "enterprise/security.mdx", "Open-source security page\n") write(reference / "static/styles/style.css", "body {}\n") - sophon = base / "sophon" + enterprise_repo = base / "enterprise_repo" if checkout: - (sophon / ".git").mkdir(parents=True) - write(sophon / "docs/ci.md", INTERNAL) + (enterprise_repo / ".git").mkdir(parents=True) + write(enterprise_repo / "docs/ci.md", INTERNAL) internal = base / "internal" write(internal / "enterprise/security.mdx", INTERNAL) write(internal / "web/enterprise/security.mdx", INTERNAL) return { "reference": reference, - "sophon": sophon, - "public": sophon / "docs/web", + "enterprise_repo": enterprise_repo, + "public": enterprise_repo / "docs/web", "internal": internal, } @@ -137,11 +137,11 @@ def test_symlinked_private_root_is_refused(base, checkout): @pytest.mark.parametrize("checkout", [True, False]) def test_symlinked_parent_of_private_root_is_refused(base, checkout): paths = site(base, checkout=checkout) - (paths["sophon"] / "docs/ci.md").unlink() - (paths["sophon"] / "docs").rmdir() - (paths["sophon"] / "docs").symlink_to(paths["internal"], target_is_directory=True) + (paths["enterprise_repo"] / "docs/ci.md").unlink() + (paths["enterprise_repo"] / "docs").rmdir() + (paths["enterprise_repo"] / "docs").symlink_to(paths["internal"], target_is_directory=True) - with pytest.raises(assemble.AssembleError, match=r"sophon/docs is a symlink"): + with pytest.raises(assemble.AssembleError, match=r"enterprise_repo/docs is a symlink"): build(config(base, paths["reference"], paths["public"])) @@ -149,10 +149,10 @@ def test_symlinked_checkout_is_refused(base): paths = site(base) public_root(paths) elsewhere = base / "elsewhere" - paths["sophon"].rename(elsewhere) - paths["sophon"].symlink_to(elsewhere, target_is_directory=True) + paths["enterprise_repo"].rename(elsewhere) + paths["enterprise_repo"].symlink_to(elsewhere, target_is_directory=True) - with pytest.raises(assemble.AssembleError, match=r"sophon is a symlink"): + with pytest.raises(assemble.AssembleError, match=r"enterprise_repo is a symlink"): build(config(base, paths["reference"], paths["public"])) @@ -161,7 +161,7 @@ def test_symlink_above_the_checkout_is_allowed(base): paths = site(real) public_root(paths) (base / "linked").symlink_to(real, target_is_directory=True) - overlay = base / "linked/sophon/docs/web" + overlay = base / "linked/enterprise_repo/docs/web" output = build(config(base, paths["reference"], overlay)) diff --git a/scripts/tests/test_candidate.py b/scripts/tests/test_candidate.py index 05e31c93..1db38007 100644 --- a/scripts/tests/test_candidate.py +++ b/scripts/tests/test_candidate.py @@ -719,7 +719,7 @@ def test_independent_review_probes(tmp_path, remote, case): def assemble_sources(base: Path, index: bytes) -> dict: - """lancedb, sophon and docs checkouts with the shape assemble.yaml reads.""" + """LanceDB, Enterprise and docs checkouts with the shape assemble.yaml reads.""" navigation = { "navigation": { "tabs": [ @@ -739,7 +739,7 @@ def assemble_sources(base: Path, index: bytes) -> dict: "docs/web/index.mdx": index, "docs/web/enterprise/security.mdx": b"Open-source security\n", }, - "sophon": {"docs/web/enterprise/security.mdx": b"Enterprise security\n"}, + "enterprise_repo": {"docs/web/enterprise/security.mdx": b"Enterprise security\n"}, "docs": {"docs/geneva/udfs.mdx": b"Geneva UDFs\n"}, }.items(): repo = base / name @@ -760,7 +760,7 @@ def assemble(base: Path, repos: dict, output: Path) -> str: path: {repos["lancedb"] / "docs/web"} role: reference - name: enterprise - path: {repos["sophon"] / "docs/web"} + path: {repos["enterprise_repo"] / "docs/web"} role: overlay private: true - name: build diff --git a/scripts/tests/test_edition.py b/scripts/tests/test_edition.py index 3eaf3b77..12553d91 100644 --- a/scripts/tests/test_edition.py +++ b/scripts/tests/test_edition.py @@ -1,6 +1,6 @@ """Tests for preparing saved editions from release records. -Disposable repositories stand in for lancedb/lancedb, lancedb/sophon and this +Disposable repositories stand in for LanceDB, Enterprise and this repository. Their versions are fixtures, not releases: OSS 0.94.0 to 0.99.0 and Enterprise 8.1.0 to 8.10.0, each built to exercise one case. The docs fixture uses this repository's own assemble.yaml and .gitignore, and every edition is @@ -170,7 +170,7 @@ def enterprise_pages(label: str, bundled: str | None, **extra) -> dict: @pytest.fixture(scope="module") def sources(tmp_path_factory) -> dict: - """lancedb and sophon repositories with tagged fixture releases. Shared, so + """LanceDB and Enterprise repositories with tagged fixture releases. Shared, so tests only read them.""" base = tmp_path_factory.mktemp("sources").resolve() lancedb = new_repo(base / "lancedb") @@ -192,16 +192,16 @@ def sources(tmp_path_factory) -> dict: (lancedb / "docs/web").mkdir(parents=True) (lancedb / "docs/web/guide.mdx").write_text("Uncommitted text\n") - sophon = new_repo(base / "sophon") + enterprise_repo = new_repo(base / "enterprise_repo") e = {} def release(version: str, files: dict, qualified: str | None = "self") -> str: - commit = snapshot(sophon, files, version) - annotated(sophon, f"v{version}", commit) + commit = snapshot(enterprise_repo, files, version) + annotated(enterprise_repo, f"v{version}", commit) if qualified == "lightweight": - git(sophon, "tag", f"release-qualified/v{version}", commit) + git(enterprise_repo, "tag", f"release-qualified/v{version}", commit) elif qualified: - annotated(sophon, f"release-qualified/v{version}", commit if qualified == "self" else qualified) + annotated(enterprise_repo, f"release-qualified/v{version}", commit if qualified == "self" else qualified) e[version] = commit return commit @@ -215,8 +215,8 @@ def release(version: str, files: dict, qualified: str | None = "self") -> str: release("8.8.0", enterprise_pages("8.8", c["0.99.0"], **{"docs/web/enterprise/ci.md": Symlink("../../ci.md")})) release("8.9.0", enterprise_pages("8.9", "1" * 40)) release("8.10.0", enterprise_pages("8.10", c["0.99.0"]), qualified="lightweight") - e["main"] = snapshot(sophon, enterprise_pages("unreleased main", c["main"]), "main moves on") - return {"lancedb": lancedb, "sophon": sophon, "oss": c, "enterprise": e} + e["main"] = snapshot(enterprise_repo, enterprise_pages("unreleased main", c["main"]), "main moves on") + return {"lancedb": lancedb, "enterprise_repo": enterprise_repo, "oss": c, "enterprise": e} @pytest.fixture @@ -255,7 +255,7 @@ def on_main(docs: Path) -> str: def prepare(docs: Path, sources: dict, *argv: str, lancedb: Path | None = None) -> str: args = edition.parse([ "prepare", *argv, - "--lancedb", str(lancedb or sources["lancedb"]), "--enterprise-repo", str(sources["sophon"]), + "--lancedb", str(lancedb or sources["lancedb"]), "--enterprise-repo", str(sources["enterprise_repo"]), ]) return edition.prepare(args, repo=docs) @@ -382,14 +382,14 @@ def test_enterprise_edition_reads_the_lancedb_it_bundles(docs, sources): assert not any(INTERNAL in data for data in files.values()) rec = record(docs, "enterprise-8.1.0") - sophon = sources["sophon"] + enterprise_repo = sources["enterprise_repo"] assert rec["release"] == { "repository": "lancedb/sophon", "tag": "v8.1.0", "commit": sources["enterprise"]["8.1.0"], "qualification": { "tag": "release-qualified/v8.1.0", - "object": git(sophon, "rev-parse", "refs/tags/release-qualified/v8.1.0"), + "object": git(enterprise_repo, "rev-parse", "refs/tags/release-qualified/v8.1.0"), }, "bundled_lancedb": bundled, } @@ -403,7 +403,7 @@ def test_enterprise_edition_reads_the_lancedb_it_bundles(docs, sources): (["oss", "0.97.0", "--enterprise", "v8.1.0"], "has no docs/web"), (["oss", "0.96.0", "--enterprise", "v8.1.0"], "has no tag v0.96.0"), (["oss", "1.0.0-beta.1", "--enterprise", "v8.1.0"], "not a stable release version"), - (["oss", "0.98.0"], "name the lancedb/sophon tag or commit"), + (["oss", "0.98.0"], "name the Enterprise tag or commit"), (["oss", "0.98.0", "--enterprise", "main"], "not a branch"), (["oss", "0.98.0", "--enterprise", "v8.1.0", "--bundled-lancedb", "1" * 40], "Enterprise releases only"), (["oss", "0.95.0", "--enterprise", "v8.1.0"], ".gitignore would leave out"), @@ -415,7 +415,7 @@ def test_enterprise_edition_reads_the_lancedb_it_bundles(docs, sources): (["enterprise", "8.10.0"], "release-qualified/v8.10.0 is not an annotated tag"), (["enterprise", "8.5.0"], "has no docs/web"), (["enterprise", "8.6.0"], "records no bundled LanceDB"), - (["enterprise", "8.1.0", "--bundled-lancedb", "1" * 40], "disagrees with lancedb/sophon v8.1.0"), + (["enterprise", "8.1.0", "--bundled-lancedb", "1" * 40], "disagrees with Enterprise v8.1.0"), (["enterprise", "8.9.0"], "fetch it first"), (["enterprise", "8.7.0"], r"\.notes\.mdx is hidden"), (["enterprise", "8.8.0"], "ci.md is a symlink"), @@ -841,7 +841,7 @@ def run(*argv: str) -> subprocess.CompletedProcess: cwd=docs, env=environment(), capture_output=True, text=True, ) - repos = ["--lancedb", str(sources["lancedb"]), "--enterprise-repo", str(sources["sophon"])] + repos = ["--lancedb", str(sources["lancedb"]), "--enterprise-repo", str(sources["enterprise_repo"])] result = run("prepare", "oss", "0.98.0", "--enterprise", "v8.1.0", *repos) assert result.returncode == 0, result.stderr assert record(docs, "oss-0.98.0")["sources"]["build"]["commit"] == main diff --git a/skills/docs-writer/SKILL.md b/skills/docs-writer/SKILL.md index c8c1909a..32d96050 100644 --- a/skills/docs-writer/SKILL.md +++ b/skills/docs-writer/SKILL.md @@ -146,7 +146,7 @@ Docs must reflect what the code actually does. Before writing or editing any cla Resolution order for a repo name the user mentions: -1. Sibling checkout of `/Users/prrao/code/docs`: e.g., `../lancedb`, `../sophon`, `../geneva`. Check with `ls ../` first. +1. Sibling checkout of `/Users/prrao/code/docs`: e.g., `../lancedb`, `../enterprise`, `../geneva`. Check with `ls ../` first. 2. Anywhere under `/Users/prrao/code/`: try `ls /Users/prrao/code/ | grep -i `. 3. If neither exists locally, ask the user for the path (or a clone URL) before proceeding — do **not** guess or substitute a different repo. @@ -180,6 +180,6 @@ Prefer these over ad-hoc emphasis: - Inventing export names that don't match the `{Py|Ts|Rs}{TitleCase}` formula. - Using a relative import path like `'../snippets/indexing.mdx'` (must be `'/snippets/indexing.mdx'`). - Writing Python-only examples for a feature that exists in all three SDKs. -- Documenting parameters that don't exist in the source repo (e.g., `../lancedb`, `../sophon`, `../geneva`). +- Documenting parameters that don't exist in the source repo (e.g., `../lancedb`, `../enterprise`, `../geneva`). - Ignoring a user's explicit instruction to cross-check against a named repo — that instruction is mandatory, not advisory. - Running `python scripts/mdx_snippets_gen.py` without the `-s tests/` flag (the script needs a source dir). diff --git a/workflows/docs-audit/manifests/indexing.toml b/workflows/docs-audit/manifests/indexing.toml index fde90864..b2c14dec 100644 --- a/workflows/docs-audit/manifests/indexing.toml +++ b/workflows/docs-audit/manifests/indexing.toml @@ -1,5 +1,5 @@ name = "indexing" -description = "Audit the indexing docs against user-facing evidence from lancedb, docs snippets/tests, and sophon." +description = "Audit the indexing docs against user-facing evidence from lancedb, docs snippets/tests, and Enterprise." docs_repo = "docs" rotation_unit = "page" keywords = [ diff --git a/workflows/docs-audit/manifests/namespaces.toml b/workflows/docs-audit/manifests/namespaces.toml index 4ee819c7..a0b55dc2 100644 --- a/workflows/docs-audit/manifests/namespaces.toml +++ b/workflows/docs-audit/manifests/namespaces.toml @@ -1,5 +1,5 @@ name = "namespaces" -description = "Audit namespace and catalog-model docs against user-facing evidence from lancedb SDKs, docs snippets/tests, and sophon enterprise namespace surfaces." +description = "Audit namespace and catalog-model docs against user-facing evidence from lancedb SDKs, docs snippets/tests, and Enterprise namespace surfaces." docs_repo = "docs" rotation_unit = "page" keywords = [ diff --git a/workflows/docs-audit/manifests/search.toml b/workflows/docs-audit/manifests/search.toml index 9469a394..6f160372 100644 --- a/workflows/docs-audit/manifests/search.toml +++ b/workflows/docs-audit/manifests/search.toml @@ -1,5 +1,5 @@ name = "search" -description = "Audit the search docs against user-facing evidence from lancedb, docs snippets/tests, and sophon." +description = "Audit the search docs against user-facing evidence from lancedb, docs snippets/tests, and Enterprise." docs_repo = "docs" rotation_unit = "page" keywords = [ diff --git a/workflows/docs-audit/manifests/storage.toml b/workflows/docs-audit/manifests/storage.toml index 95764110..0524b0b0 100644 --- a/workflows/docs-audit/manifests/storage.toml +++ b/workflows/docs-audit/manifests/storage.toml @@ -1,5 +1,5 @@ name = "storage" -description = "Audit the storage docs against user-facing evidence from lancedb, docs snippets/tests, and sophon." +description = "Audit the storage docs against user-facing evidence from lancedb, docs snippets/tests, and Enterprise." docs_repo = "docs" rotation_unit = "page" keywords = [ diff --git a/workflows/docs-audit/manifests/table-operations.toml b/workflows/docs-audit/manifests/table-operations.toml index 9b9e9ac8..fe511a75 100644 --- a/workflows/docs-audit/manifests/table-operations.toml +++ b/workflows/docs-audit/manifests/table-operations.toml @@ -1,5 +1,5 @@ name = "table-operations" -description = "Audit the table operations docs against user-facing evidence from lancedb, docs snippets/tests, and sophon." +description = "Audit the table operations docs against user-facing evidence from lancedb, docs snippets/tests, and Enterprise." docs_repo = "docs" rotation_unit = "page" keywords = [