diff --git a/.github/workflows/assemble.yml b/.github/workflows/assemble.yml index ec664eeb..098689d9 100644 --- a/.github/workflows/assemble.yml +++ b/.github/workflows/assemble.yml @@ -14,7 +14,7 @@ on: description: lancedb/lancedb ref to read docs/web from default: main sophon_ref: - description: lancedb/sophon ref to read docs/web from + description: Enterprise ref to read docs/web from default: main permissions: @@ -29,7 +29,7 @@ env: SOPHON_DOCS_ROOT: sophon/docs/web jobs: - # Forks run the guard tests without private-source credentials. + # Fork-safe checks need no private-source credentials. test: name: Test the assembler runs-on: ubuntu-latest @@ -42,6 +42,22 @@ jobs: - name: Test the overlay, navigation and private-root guards run: make test-assemble + # Fetch the event's base SHA so shallow clones and moving branches do not affect preservation checks. + - name: Check that saved editions and their records are kept + env: + EVENT: ${{ github.event_name }} + # Not BASE itself: make would read it from the environment. + BASE_SHA: ${{ github.event_name == 'pull_request' && github.event.pull_request.base.sha || github.event_name == 'push' && github.event.before || '' }} + run: | + set -euo pipefail + if [ -z "$BASE_SHA" ] || [ "$BASE_SHA" = 0000000000000000000000000000000000000000 ]; then + echo "No base commit for this $EVENT event; checking each edition against its record only." + make check-editions + else + git fetch --no-tags --depth=1 origin "$BASE_SHA" + make check-editions BASE="$BASE_SHA" + fi + assemble: runs-on: ubuntu-latest steps: @@ -56,8 +72,8 @@ jobs: sparse-checkout: docs/web persist-credentials: false - - name: Create a read-only Sophon token - id: sophon-token + - name: Create a read-only Enterprise token + id: enterprise-token uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 with: client-id: ${{ vars.SOPHON_DOCS_APP_CLIENT_ID }} @@ -72,7 +88,7 @@ jobs: repository: lancedb/sophon ref: ${{ inputs.sophon_ref || 'main' }} path: sophon - token: ${{ steps.sophon-token.outputs.token }} + token: ${{ steps.enterprise-token.outputs.token }} sparse-checkout: docs/web submodules: false persist-credentials: false @@ -131,10 +147,10 @@ jobs: env: SOURCES: ${{ steps.assemble.outputs.sources }} LANCEDB_REF: ${{ inputs.lancedb_ref || 'main' }} - SOPHON_REF: ${{ inputs.sophon_ref || 'main' }} + ENTERPRISE_REF: ${{ inputs.sophon_ref || 'main' }} run: | python3 scripts/candidate.py record build/site --sources "$SOURCES" \ - --ref "lancedb=$LANCEDB_REF" --ref "enterprise=$SOPHON_REF" \ + --ref "lancedb=$LANCEDB_REF" --ref "enterprise=$ENTERPRISE_REF" \ --output build/candidate.json --summary "$GITHUB_STEP_SUMMARY" - name: Upload the candidate diff --git a/.github/workflows/docs-ci.yml b/.github/workflows/docs-ci.yml index 55f965d2..874d1861 100644 --- a/.github/workflows/docs-ci.yml +++ b/.github/workflows/docs-ci.yml @@ -1,6 +1,6 @@ name: Docs Check -# The site is assembled from lancedb/lancedb, lancedb/sophon and this +# The site is assembled from LanceDB, Enterprise documentation and this # repository, so links are checked on the assembled tree. `docs/` alone is no # longer a site: checking it reported pages that live in the other roots as # missing. The Assemble workflow runs the same checks before it records a @@ -39,8 +39,8 @@ jobs: sparse-checkout: docs/web persist-credentials: false - - name: Create a read-only Sophon token - id: sophon-token + - name: Create a read-only Enterprise token + id: enterprise-token uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 with: client-id: ${{ vars.SOPHON_DOCS_APP_CLIENT_ID }} @@ -55,7 +55,7 @@ jobs: repository: lancedb/sophon ref: main path: sophon - token: ${{ steps.sophon-token.outputs.token }} + token: ${{ steps.enterprise-token.outputs.token }} sparse-checkout: docs/web submodules: false persist-credentials: false diff --git a/.github/workflows/prose-style.yml b/.github/workflows/prose-style.yml index f8f5735d..eabe31e0 100644 --- a/.github/workflows/prose-style.yml +++ b/.github/workflows/prose-style.yml @@ -4,7 +4,7 @@ name: Prose style # annotations on the lines a pull request adds or changes and never fail the # job; an install, sync or configuration error still does. Reviewers decide # everything else. The rules job tests the shared package that lancedb/lancedb -# and lancedb/sophon pin by release. +# and the Enterprise documentation pin by release. on: pull_request: diff --git a/AGENTS.md b/AGENTS.md index 4c5d5e9c..5989cd69 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -11,4 +11,9 @@ This is a documentation site for [LanceDB](https://docs.lancedb.com). ## Snippets - Code snippets are generated in `lancedb/lancedb` from the tests under `docs/web-tests/`, not in this repository. See "Code snippets" in README.md for the command. -- The four Geneva snippets in `docs/snippets/` are frozen copies; do not regenerate or edit them. \ No newline at end of file +- The four Geneva snippets in `docs/snippets/` are frozen copies; do not regenerate or edit them. + +## Saved editions + +- `editions/` holds saved documentation editions and `releases/` their records. Create an edition only with `scripts/edition.py prepare`, and change one only as a reviewed correction listed in its record. See "Saved editions" in README.md. +- Write new documentation in `lancedb/lancedb` or the Enterprise documentation repository, not in a saved edition. diff --git a/CLAUDE.md b/CLAUDE.md index 4c5d5e9c..5989cd69 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -11,4 +11,9 @@ This is a documentation site for [LanceDB](https://docs.lancedb.com). ## Snippets - Code snippets are generated in `lancedb/lancedb` from the tests under `docs/web-tests/`, not in this repository. See "Code snippets" in README.md for the command. -- The four Geneva snippets in `docs/snippets/` are frozen copies; do not regenerate or edit them. \ No newline at end of file +- The four Geneva snippets in `docs/snippets/` are frozen copies; do not regenerate or edit them. + +## Saved editions + +- `editions/` holds saved documentation editions and `releases/` their records. Create an edition only with `scripts/edition.py prepare`, and change one only as a reviewed correction listed in its record. See "Saved editions" in README.md. +- Write new documentation in `lancedb/lancedb` or the Enterprise documentation repository, not in a saved edition. diff --git a/Makefile b/Makefile index 667dc691..fb6b9f63 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,7 @@ ASSEMBLE_SCRIPT := scripts/assemble.py ASSEMBLE_RUN := uv run --no-project --with pyyaml # uv run automatically handles virtualenv, so no activation needed -.PHONY: py ts rs snippets hf-sync assemble test-assemble check-spec sync-spec +.PHONY: py ts rs snippets hf-sync assemble test-assemble check-editions check-spec sync-spec # Snippets are generated in lancedb/lancedb, beside the tests they come from; # these targets only say so. See "Code snippets" in README.md. @@ -24,7 +24,7 @@ hf-sync: # Assemble the published tree from the roots declared in assemble.yaml into # build/site. Set LANCEDB_DOCS_ROOT and SOPHON_DOCS_ROOT when the lancedb and -# sophon checkouts are not beside this one. +# Enterprise checkouts are not beside this one. assemble: @$(ASSEMBLE_RUN) $(ASSEMBLE_SCRIPT) @@ -32,6 +32,10 @@ assemble: test-assemble: @$(ASSEMBLE_RUN) --with pytest pytest scripts/tests -q +# Set BASE to enforce preservation of editions already merged. +check-editions: + @$(ASSEMBLE_RUN) scripts/edition.py check $(if $(BASE),--base $(BASE)) + # Fail if the tracked OpenAPI spec has drifted from the release pinned in # assemble.yaml. Run in CI so the pin cannot rot unnoticed. check-spec: diff --git a/README.md b/README.md index 7de2d1a3..d562efb3 100644 --- a/README.md +++ b/README.md @@ -5,8 +5,8 @@ Home of the [LanceDB](https://lancedb.com/) documentation. Built using [Mintlify ## Development The published site is assembled from three roots, listed in `assemble.yaml`: -the open-source pages in `lancedb/lancedb` and the Enterprise pages in -`lancedb/sophon`, both under `docs/web`, and this repository's `docs/`. +the open-source pages in `lancedb/lancedb`, the Enterprise documentation, +both under `docs/web`, and this repository's `docs/`. This repository's `docs/` holds the pages it still owns: the Geneva pages, the dataset cards and the REST API reference. The four Geneva snippets @@ -187,14 +187,15 @@ Publishing relies on settings outside this repository: - Mintlify's Git settings: repository `lancedb/docs`, branch `assembled`, and the content directory above. Mintlify serves the branch its settings name, so publishing to `assembled` reaches production only once that is the branch. -- The LanceDB Docs Reader GitHub App installed on `lancedb/sophon` only, with +- The LanceDB Docs Reader GitHub App installed on the Enterprise documentation repository only, with Contents: read and the required Metadata: read permission. Set its Client ID in the `SOPHON_DOCS_APP_CLIENT_ID` repository variable and its PEM private key in the `SOPHON_DOCS_APP_PRIVATE_KEY` repository secret. The Assemble and Docs - Check workflows generate a short-lived token scoped to Sophon with + Check workflows generate a short-lived token scoped to that repository with Contents: read. Checkout does not persist it, and the token is revoked when the job ends. Fork pull requests receive no App secret and cannot run the - combined build; the assembler guard tests still run without credentials. + combined build; the assembler guard tests and the saved-edition check still + run without Enterprise repository credentials. Of these, the Publish workflow checks only that production runs from `main`, that the environment supplies the deploy key and that `MINTLIFY_CONTENT_DIR` is @@ -202,6 +203,210 @@ set. It cannot see whether the environment requires reviewers and admits only `main`, or whether the ruleset refuses every other writer: verify those separately before the first production publication. +## Saved editions + +Each published documentation edition is saved in this repository: the site the +assembler built from one product release's own sources, in +`editions///`, with a release record in +`releases/-.yaml` that names every source it was read from. +New documentation is still written in `lancedb/lancedb` and the Enterprise +documentation repository. +[VERSIONING.md](VERSIONING.md) describes the design. Composing the saved +editions into the site, and publishing them, are later steps; for now the site +is assembled and published as above. + +An edition's content comes from two kinds of commit, and its record names both: + +- **Product pages** come from the release: an OSS edition's from its + `lancedb/lancedb` release tag, and an Enterprise edition's from its + Enterprise release tag and the LanceDB commit that release bundles. +- **Docs-owned content**, the Geneva pages, the dataset cards and the REST API + specification, comes from the docs input commit: the commit of this + repository's `main` that preparation read, recorded as `sources.build.commit`. + It is `main` as it was when the edition was prepared, not when the product + was released. + +Both are fixed when the edition is prepared. Later changes to `main`, or to a +product's sources, never reach a saved edition; only a reviewed correction +changes one. + +### Preparing an edition + +Fetch the release tags into the LanceDB and Enterprise checkouts. Then run the +command in a clone of this repository whose `origin` is +`https://github.com/lancedb/docs.git`, or its SSH form, rather than a fork, on +a branch whose `scripts/` are those of `main`: + +```bash +uv run --no-project --with pyyaml scripts/edition.py prepare enterprise 1.10.0 --lancedb ../lancedb --enterprise-repo ../enterprise +uv run --no-project --with pyyaml scripts/edition.py prepare oss 0.41.0 --enterprise v1.10.0 --lancedb ../lancedb --enterprise-repo ../enterprise +``` + +- **Enterprise** reads the Enterprise release tag `v1.10.0`, which must have an + annotated `release-qualified/v1.10.0` tag on the same commit. The shared + pages come from the LanceDB commit that release bundles at `src/lancedb`, + which may be one no OSS release tags: fetch it with `git -C ../lancedb fetch + origin `. `--bundled-lancedb `, given the full 40-character + hash, stops preparation if the release bundles a different one. +- **OSS** reads lancedb's tag `v0.41.0`. The Enterprise pages come from the + Enterprise tag or full commit hash named with `--enterprise`; a branch name + is not accepted, so the record pins an exact commit. The named Enterprise + release supplies the Enterprise pages in that OSS edition; see + [What goes into an edition](VERSIONING.md#what-goes-into-an-edition). +- The LanceDB and Enterprise roots are read from Git at their commits, never + from a working tree, and only from `docs/web`. +- Preparation fetches `main` from `origin` itself, and reads this + repository's `docs/` and `assemble.yaml` from Git at its head, or at an + earlier commit of `main` named with `--docs`. It never reads them from your + branch, your working tree or an `origin/main` fetched before. A fork's `main` + can hold commits `lancedb/docs` never has, so an `origin` that is not + `lancedb/docs` stops preparation. Squash merges give a branch's commits new + hashes, so a record names only a commit of `main`, and editions prepared one + after another in the same pull request do not depend on its unmerged + commits. The assembler that runs is your checkout's `scripts/`, so they must + be that commit's. +- Only stable versions, such as `1.2.3`, have editions. + +Preparation saves nothing, and says why, when a tag or commit is missing, a +release has no `docs/web`, an Enterprise release is not qualified or bundles +another LanceDB, a root holds a symlink, the Enterprise root holds a hidden +file, the assembler stops, `.gitignore` would leave out one of the edition's +files, `origin` is not `lancedb/docs`, the docs input commit is not on its +`main`, or your `scripts/` are not that commit's. + +Preparing a saved release again changes nothing when its release commits and +qualification tag match the record, even after the edition has been corrected, +and stops when they differ. It never replaces a saved edition. + +Check the new edition as a site, check every edition against its record and +against `main`, and commit the edition and its record together: + +```bash +(cd editions/enterprise/1.10.0 && mint validate && mint broken-links --check-anchors --check-redirects) +uv run --no-project --with pyyaml scripts/edition.py check --base refs/remotes/origin/main +``` + +A record looks like this. It is illustrative: the Enterprise repository is a +placeholder and the commits and checksums are shortened, so `check` would +refuse it as written. + +```yaml +format: 1 +product: enterprise +version: 1.10.0 +edition: editions/enterprise/1.10.0 +release: + repository: + tag: v1.10.0 + commit: 5a1f0c9e… + qualification: + tag: release-qualified/v1.10.0 + object: 9c3e27d4… + bundled_lancedb: 7d0b4a12… +sources: + lancedb: + repository: lancedb/lancedb + path: docs/web + commit: 7d0b4a12… + enterprise: + repository: + path: docs/web + commit: 5a1f0c9e… + build: + repository: lancedb/docs + path: docs + commit: e2c4b8f0… + openapi: + repository: lance-format/lance-namespace + release: v0.12.0 + path: docs/src/spec.yaml + sha256: 41ab93c7… +prepared: + files: 312 + sha256: b07d5e61… +corrections: [] +``` + +`release` and `sources` say what the edition was built from, and `prepared` is +the snapshot as first saved. An OSS record has `enterprise_pages`, the +`--enterprise` ref and its commit, in place of `qualification` and +`bundled_lancedb`. `sources.openapi` names the specification release that +`assemble.yaml` pinned at the docs input commit, and the checksum of the copy +saved in the edition. + +### What `check` enforces + +`edition.py check` works offline, on this checkout: + +- Every record has the fields `prepare` writes, in their formats, and each + source commit agrees with the release field that selected it, such as an + Enterprise record's `sources.lancedb.commit` and `release.bundled_lancedb`. +- Every record has its edition, and every edition its record. +- Each edition matches the checksum its record last accepted: + `prepared.sha256`, or the latest correction's `sha256`. + +With `--base COMMIT`, it also compares the editions and records already on +that commit, by their parsed values rather than their formatting. Each must +still be there. Every field of a record except `corrections` must be +unchanged, and its existing corrections must be unchanged and in order, with +new ones only after them. A new edition is checked by the rules above. + +The test job of the Assemble workflow runs `make check-editions` with a base: +for a pull request, the base commit its event names, and for a push to `main`, +the commit `main` pointed at before. The job first fetches that commit by hash +with its own GitHub token; `edition.py check` then runs offline and needs no +Enterprise repository credentials. A manual run of the workflow has no base, +and checks each edition against its record only. + +This keeps saved editions only while the check must pass before a pull request +merges, and while changes to the checker and its workflow need enforced review, +such as required code-owner review of `scripts/`, the `Makefile` and +`.github/workflows/`, since a pull request runs its own copy of the check. Both +are repository rules set up separately. The check on a push to `main` reports +a violation in that push but cannot undo it, and a later push does not find it, +because its base already includes the change. + +### Correcting a saved edition + +A saved edition can be corrected after review, at the same URL: + +1. Edit the affected files in each edition that has the error, and in no other. +2. Run `edition.py check`. It fails, because the edition no longer matches its + record, and prints the corrected edition's checksum: + + ```text + edition: editions/enterprise/1.10.0 has changed since it was prepared; its files now have checksum 3f9c2a7e…. For a reviewed correction, add an entry under `corrections` in releases/enterprise-1.10.0.yaml with `sha256: 3f9c2a7e…` + ``` + +3. In each edition's record, add an entry at the end of `corrections` with a + one-line summary, the files it changes, and that checksum; for the first + correction, replace `corrections: []`. Leave the rest of the record, and + any earlier corrections, as they are. + + ```yaml + corrections: + - summary: Fix the default retry count in the merge-insert example. + files: [tables/merge-insert.mdx] + sha256: 3f9c2a7e… + ``` + +4. Run the checks above, with `--base refs/remotes/origin/main` after fetching `main`. + `edition.py check` now passes: each edition matches the checksum of its + latest correction. Any later change to the edition, including a deleted + file, fails again until it is recorded the same way, as a further entry. + `--base` refuses a change to an earlier entry or to the rest of the + record, as the pull request's check does. Check a corrected example the way + that example needs; a typo needs no test run. +5. If the error is still in the current source, in `lancedb/lancedb`, + the Enterprise documentation repository or `docs/` here, fix it there as well. +6. Open one pull request that names every edition it corrects. Reviewers decide + whether each correction is right; the checksum shows only that the record + describes the files. + +Preparing that release again later checks its release inputs and saved files, +then keeps the edition and its corrections. It does not fetch docs main or +require matching assembler scripts unless it is building a new edition. + ## Code snippets The code examples on the open-source pages are tested programs in diff --git a/STYLE.md b/STYLE.md index e2a2599e..144f0f22 100644 --- a/STYLE.md +++ b/STYLE.md @@ -5,7 +5,7 @@ Be direct, approachable and technically precise. Give readers enough context to make a decision, then show them how to act on it. This guide covers prose on docs.lancedb.com: the pages in `lancedb/lancedb` -`docs/web`, the public Enterprise pages in `lancedb/sophon` `docs/web`, and the +`docs/web`, the public Enterprise pages under `docs/web`, and the Geneva, dataset and REST pages in this repository. [WRITING.md](WRITING.md) covers Mintlify mechanics. diff --git a/VERSIONING.md b/VERSIONING.md new file mode 100644 index 00000000..dffabc97 --- /dev/null +++ b/VERSIONING.md @@ -0,0 +1,94 @@ +# Versioned documentation + +Readers should be able to find the docs for the version of LanceDB they run. +The docs are still written next to the code, in `lancedb/lancedb` and the +Enterprise documentation repository. Each release gets a saved edition that +combines its product pages, the selected shared or Enterprise pages, and pages +owned by this repository. These come from separately recorded commits. Readers +can switch between editions on the site. + +Release records and saved editions are in place; see +[Saved editions](README.md#saved-editions). Composing the site, publishing it +and finding new releases come next. Until a cutover is approved, the site is +built and published as it is today. + +## How it works + +Each edition is the site the assembler built from those sources, stored in +`editions/oss//` or +`editions/enterprise//`. A record in `releases/` lists the commits it +came from. An edition is built once. Later changes to `main` or to a product's +docs don't reach it, so older versions keep the pages they shipped with. + +The published site is put together from the saved editions. Each edition gets +its own path, such as `/enterprise/v1-10-0/`, and readers switch products and +versions with Mintlify's built-in selectors. Each version also has an About +this version page listing its release and sources. Branding, the banner, the +footer and integrations are shared across versions. + +A composed site goes to staging first, and production gets the same build after +someone approves it. + +New editions arrive as pull requests. They can be prepared by hand with +`scripts/edition.py prepare`, and a discovery job prepares one for each new +qualified release. People review and merge every one. + +## What goes into an edition + +An Enterprise edition has the Enterprise release's own pages, plus the shared +pages from the exact LanceDB commit that release bundles. That commit can be +older than the latest OSS release, or one that no OSS release tags. + +An OSS edition has the pages from its `lancedb/lancedb` release tag. The +navigation also covers the Enterprise pages, so an OSS edition takes them from +an Enterprise release that's named when the edition is prepared. Discovery uses +the preferred Enterprise edition. + +Pages owned by this repository, like the Geneva pages, dataset cards and REST +API reference, come from `main` as it was when the edition was prepared. + +Every edition keeps the full navigation, with badges showing what's available +in OSS and in Enterprise. OSS and Enterprise editions ship on their own +schedules, and an Enterprise release doesn't need a matching OSS release. + +Only stable releases get editions. An Enterprise release counts once its +release tag and its `release-qualified/` tag point at the same commit. An OSS +release counts once its GitHub release is published and isn't marked as a +prerelease. Betas and release candidates are skipped, and older releases aren't +backfilled. + +## Which version readers see + +Each product has a preferred edition, which readers get by default. New +visitors land on the preferred Enterprise edition, and links without a version +go to the same page there. Links that include a version stay in that version. + +The preferred edition only changes when someone updates it in a reviewed pull +request. A new release, or a patch to an older release line, doesn't change it. + +If an old link has no matching page in the default edition, it gets no +redirect. Composition lists those links so someone can add the page or a +specific redirect. + +When a reader switches to a version that doesn't have the page they're on, +Mintlify decides where they land. + +## Corrections + +A saved edition can be corrected through a reviewed pull request that edits its +files and records the correction. The version and its URLs stay the same. If +the mistake is still in the current docs, fix it there too. Preparing the same +release again keeps the correction. + +## Publishing + +Merged editions and corrections go to staging on their own. Production gets the +build staging shows, after approval in the `production` GitHub environment, and +promotion publishes that build without rebuilding it. An older run can't +replace a newer approved one. + +To roll back, change the preferred edition. Published versions stay where they +are. + +Merging and production promotion could become automatic later, each as its own +decision. diff --git a/assemble.yaml b/assemble.yaml index 5af5978f..9316d83f 100644 --- a/assemble.yaml +++ b/assemble.yaml @@ -20,7 +20,7 @@ roots: role: reference # Enterprise pages, from the private repository's public `docs/web` folder; - # the rest of `sophon/docs` is internal and must never be read. Every path + # the rest of its `docs/` directory must never be read. Every path # here also exists in `lancedb`, which owns the navigation; an overlay page # replaces the reference page at the same path rather than adding one. # `private` refuses symlinks and hidden files under the root. Override the diff --git a/scripts/assemble.py b/scripts/assemble.py index 43562171..85dc886d 100644 --- a/scripts/assemble.py +++ b/scripts/assemble.py @@ -3,7 +3,7 @@ The site is not published straight from this repository. It is assembled here from the roots in `assemble.yaml` -- the open-source pages from -`lancedb/lancedb`, the Enterprise pages from `sophon` as whole-page overlays, and +`lancedb/lancedb`, the Enterprise pages as whole-page overlays, and this repository's `docs/`. CI checks each assembly and keeps it as a candidate; only the Publish workflow puts a chosen candidate on the `assembled` branch (see `candidate.py`). @@ -416,7 +416,7 @@ def merge_nav(base: dict, fragment: dict) -> dict: sidebar order is what a reader navigates by, and dropping Governance below Support or Datasets past Use Cases silently reorders the whole site. - This is the same shape sophon's Enterprise fragment uses. + This is the same shape the Enterprise fragment uses. """ def descend(node: dict | list, path: list[str]) -> list: @@ -1016,6 +1016,17 @@ def emit( return written + 1 +def assemble_site(config: Config) -> tuple[int, list[str]]: + """Build live sites and saved editions identically; return file count and warnings.""" + resolved = resolve(config) + docs_json, nav_raw = assemble_nav(resolved) + warnings = validate(config, resolved, docs_json) + merge(resolved) + inline_examples(resolved) + annotate(resolved, docs_json) + return emit(config, resolved, docs_json, nav_raw), warnings + + def revision(path: Path) -> str: """The commit a root was read from, so every build names its inputs. @@ -1133,13 +1144,7 @@ def main() -> int: print(f"openapi spec {'updated from' if changed else 'already matches'} {tag}") return 0 - resolved = resolve(config) - docs_json, nav_raw = assemble_nav(resolved) - warnings = validate(config, resolved, docs_json) - merge(resolved) - inline_examples(resolved) - annotate(resolved, docs_json) - count = emit(config, resolved, docs_json, nav_raw) + count, warnings = assemble_site(config) except AssembleError as exc: print(f"assemble: {exc}", file=sys.stderr) return 1 diff --git a/scripts/edition.py b/scripts/edition.py new file mode 100644 index 00000000..54c644eb --- /dev/null +++ b/scripts/edition.py @@ -0,0 +1,835 @@ +""" +Prepare a saved documentation edition from one product release, and check the +saved editions. + +Product sources come from their release commits. Docs-owned content and +assemble.yaml come from canonical lancedb/docs main, with matching scripts/. +Preparation never replaces an existing edition. Preservation across merges +requires `check --base` to pass before merging. See VERSIONING.md. + +Usage: + + python scripts/edition.py prepare oss VERSION --enterprise REF [--lancedb REPO] [--enterprise-repo REPO] + [--docs REF] + python scripts/edition.py prepare enterprise VERSION [--bundled-lancedb COMMIT] + [--lancedb REPO] [--enterprise-repo REPO] [--docs REF] + python scripts/edition.py check [--base COMMIT] +""" + +from __future__ import annotations + +import argparse +import os +import re +import shutil +import subprocess +import sys +import tempfile +from pathlib import Path, PurePosixPath + +import yaml + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +import assemble # noqa: E402 +import candidate # noqa: E402 + +REPO_ROOT = Path(__file__).resolve().parent.parent +FORMAT = 1 +PRODUCTS = ("oss", "enterprise") +# Editions are of stable releases. Prereleases need a policy first. +VERSION_RE = re.compile(r"(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)") +COMMIT_RE = candidate.COMMIT_RE +# Every assemble.yaml root needs provenance here. +SOURCES = { + "lancedb": ("lancedb/lancedb", "docs/web"), + "enterprise": ("lancedb/sophon", "docs/web"), + "build": ("lancedb/docs", "docs"), +} +# Pin docs inputs to canonical main; squash merges can discard branch commits. +DOCS_REMOTE = "origin" +# Fully qualified: a local branch or tag named origin/main would win the shorthand. +DOCS_MAIN = f"refs/remotes/{DOCS_REMOTE}/main" +DOCS_URLS = re.compile( + r"(?:https://github\.com/|ssh://git@github\.com/|git@github\.com:)lancedb/docs(?:\.git)?", re.IGNORECASE +) +# Qualification requires an annotated tag on the release commit. +BUNDLED_LANCEDB = "src/lancedb" +QUALIFIED_TAG = "release-qualified/v{version}" +# Files that would change how Git stores the edition's own files. +GIT_CONTROL = frozenset({".gitignore", ".gitattributes"}) +HEADER = """\ +# Release record, written by scripts/edition.py prepare. `release` and `sources` +# say what the edition was built from, and `prepared` describes the snapshot as +# first saved. Once merged, neither changes; `edition.py check --base` refuses +# a change to them. A reviewed correction to the saved edition adds an entry at +# the end of `corrections` with the checksum of the corrected edition; see +# "Saved editions" in README.md. +""" + + +class EditionError(Exception): + """Inputs or state that must stop preparation before anything is saved.""" + + +# git + + +def git(repo: Path, *args: str, data: bytes | None = None) -> bytes: + result = subprocess.run( + ["git", "-C", str(repo), *args], input=data, capture_output=True + ) + if result.returncode: + message = result.stderr.decode(errors="replace").strip() + raise EditionError(f"git {args[0]} in {repo} failed: {message}") + return result.stdout + + +def text(repo: Path, *args: str) -> str: + return git(repo, *args).decode().strip() + + +def commit_of(repo: Path, ref: str, what: str) -> str: + try: + return text(repo, "rev-parse", "--verify", "--quiet", f"{ref}^{{commit}}") + except EditionError: + raise EditionError(f"{what}: {ref} is not a commit in {repo}; fetch it first") from None + + +def tag(repo: Path, name: str, what: str) -> tuple[str, str]: + """The object a tag points at and the commit it names.""" + ref = f"refs/tags/{name}" + try: + obj = text(repo, "rev-parse", "--verify", "--quiet", ref) + except EditionError: + raise EditionError(f"{what}: {repo} has no tag {name}; fetch the release tags first") from None + return obj, commit_of(repo, ref, what) + + +def pinned(repo: Path, ref: str, what: str) -> str: + """A tag or a full commit hash; a branch can move, so it is not accepted.""" + if COMMIT_RE.fullmatch(ref): + return commit_of(repo, ref, what) + if not text(repo, "tag", "--list", ref): + raise EditionError( + f"{what}: {ref} is neither a tag in {repo} nor a full commit hash; " + "name a pinned revision, not a branch" + ) + return tag(repo, ref, what)[1] + + +def bundled_lancedb(enterprise_repo: Path, commit: str, what: str) -> str: + """The LanceDB commit an Enterprise release bundles, from its gitlink.""" + listing = git(enterprise_repo, "ls-tree", "-z", commit, "--", BUNDLED_LANCEDB).decode() + meta = listing.split("\t", 1)[0].split() + if len(meta) != 3 or meta[0] != "160000": + raise EditionError( + f"{what} records no bundled LanceDB: {BUNDLED_LANCEDB} is not a gitlink " + f"at {commit}" + ) + return meta[2] + + +def export(repo: Path, commit: str, folder: str, dest: Path, what: str) -> None: + """Write the files under `folder` at `commit` into `dest`, byte for byte.""" + listing = git(repo, "ls-tree", "-r", "-z", "--full-tree", commit, "--", folder) + paths, blobs = [], [] + for entry in listing.decode().split("\0"): + if not entry: + continue + meta, path = entry.split("\t", 1) + mode, kind, blob = meta.split() + if not path.startswith(folder + "/"): + continue + if any(part in ("", ".", "..") for part in path.split("/")): + raise EditionError(f"{what}: {path!r} is not a plain path inside {folder}") + if mode == "120000" or kind != "blob": + raise EditionError( + f"{what}: {path} is a {'symlink' if mode == '120000' else kind}; " + "an edition reads only regular files" + ) + paths.append(path[len(folder) + 1 :]) + blobs.append(blob) + if not paths: + raise EditionError( + f"{what} has no {folder}: an edition is built only from a release's own " + "documentation sources" + ) + out = git(repo, "cat-file", "--batch", data="".join(f"{b}\n" for b in blobs).encode()) + offset = 0 + for path, blob in zip(paths, blobs): + header_end = out.index(b"\n", offset) + header = out[offset:header_end].split() + if len(header) != 3: + raise EditionError(f"{what}: {repo} lacks {folder}/{path} ({blob}); fetch it first") + size = int(header[2]) + target = dest / path + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(out[header_end + 1 : header_end + 1 + size]) + offset = header_end + 1 + size + 1 + + +# release inputs + + +def source(name: str, commit: str) -> dict: + repository, folder = SOURCES[name] + return {"repository": repository, "path": folder, "commit": commit} + + +def is_ancestor(repo: Path, commit: str, of: str) -> bool: + result = subprocess.run( + ["git", "-C", str(repo), "merge-base", "--is-ancestor", commit, of], capture_output=True + ) + if result.returncode not in (0, 1): + raise EditionError(f"git merge-base in {repo} failed: {result.stderr.decode(errors='replace').strip()}") + return result.returncode == 0 + + +def docs_main(repo: Path) -> str: + """Fetch canonical main; an existing tracking ref may come from a fork.""" + result = subprocess.run( + ["git", "-C", str(repo), "remote", "get-url", "--all", DOCS_REMOTE], capture_output=True + ) + # As fetching would use them, with any `insteadOf` rewriting applied. + urls = result.stdout.decode().splitlines() if result.returncode == 0 else [] + others = [url for url in urls if not DOCS_URLS.fullmatch(url)] + if not urls or others: + raise EditionError( + f"this checkout's {DOCS_REMOTE} is {'not supported' if urls else 'not set'}, not " + f"{SOURCES['build'][0]}; a fork's main can hold commits {SOURCES['build'][0]} never has, " + "so prepare an edition in a clone of https://github.com/lancedb/docs.git, over HTTPS " + "or SSH" + ) + try: + git(repo, "fetch", "--quiet", "--no-tags", DOCS_REMOTE, f"+refs/heads/main:{DOCS_MAIN}") + except EditionError: + raise EditionError( + f"could not fetch {SOURCES['build'][0]} main; check network access and Git authentication" + ) from None + return commit_of(repo, DOCS_MAIN, SOURCES["build"][0]) + + +def docs_commit(repo: Path, ref: str | None) -> str: + """Select docs inputs from main and require this checkout's scripts to match.""" + what = f"{SOURCES['build'][0]} --docs" + main = docs_main(repo) + commit = main if ref is None else commit_of(repo, ref, what) + if not is_ancestor(repo, commit, main): + raise EditionError( + f"{what}: {ref} is {commit}, which {SOURCES['build'][0]} main ({main}) does not " + "contain; name a commit of main, because merging gives a branch's commits new hashes" + ) + changed = text(repo, "status", "--porcelain", "--", "scripts") + if changed: + raise EditionError( + "commit or stash the changes to scripts/ first; they are the assembler that " + "builds the edition:\n" + changed + ) + if text(repo, "ls-tree", "-d", "HEAD", "--", "scripts") != text(repo, "ls-tree", "-d", commit, "--", "scripts"): + raise EditionError( + f"this checkout's scripts/ are not those of {commit}, which the record would name; " + "check out a branch whose scripts/ match it, or name the commit of main they come " + "from with --docs" + ) + return commit + + +def release_inputs(args: argparse.Namespace) -> dict: + product, version = args.product, args.version + if not VERSION_RE.fullmatch(version): + raise EditionError(f"{version!r} is not a stable release version such as 1.2.3") + lancedb, enterprise_repo = args.lancedb.resolve(), args.enterprise_repo.resolve() + if product == "oss": + if args.bundled_lancedb: + raise EditionError("--bundled-lancedb applies to Enterprise releases only") + if not args.enterprise: + raise EditionError( + "an OSS edition includes the Enterprise pages, so name the Enterprise " + "tag or commit they come from with --enterprise" + ) + release_tag = f"v{version}" + commit = tag(lancedb, release_tag, f"lancedb/lancedb {release_tag}")[1] + enterprise = pinned(enterprise_repo, args.enterprise, "Enterprise --enterprise") + release = { + "repository": SOURCES["lancedb"][0], + "tag": release_tag, + "commit": commit, + "enterprise_pages": {"ref": args.enterprise, "commit": enterprise}, + } + shared = commit + else: + if args.enterprise: + raise EditionError( + "--enterprise applies to OSS releases; an Enterprise release reads its own pages" + ) + if args.bundled_lancedb and not COMMIT_RE.fullmatch(args.bundled_lancedb): + raise EditionError( + f"--bundled-lancedb takes the full 40-character commit hash the release " + f"bundles, not {args.bundled_lancedb!r}" + ) + release_tag = f"v{version}" + what = f"Enterprise {release_tag}" + enterprise = tag(enterprise_repo, release_tag, what)[1] + qualified = QUALIFIED_TAG.format(version=version) + try: + qualification, qualified_commit = tag(enterprise_repo, qualified, what) + except EditionError: + raise EditionError( + f"{what} is not qualified: {enterprise_repo} has no tag {qualified}" + ) from None + if text(enterprise_repo, "cat-file", "-t", qualification) != "tag": + raise EditionError(f"{what}: {qualified} is not an annotated tag") + if qualified_commit != enterprise: + raise EditionError( + f"{what} is {enterprise}, but {qualified} qualifies {qualified_commit}" + ) + shared = bundled_lancedb(enterprise_repo, enterprise, what) + if args.bundled_lancedb and args.bundled_lancedb != shared: + raise EditionError( + f"--bundled-lancedb {args.bundled_lancedb} disagrees with {what}, which " + f"bundles LanceDB {shared} at {BUNDLED_LANCEDB}" + ) + commit_of(lancedb, shared, f"the LanceDB that {what} bundles") + release = { + "repository": SOURCES["enterprise"][0], + "tag": release_tag, + "commit": enterprise, + "qualification": {"tag": qualified, "object": qualification}, + "bundled_lancedb": shared, + } + return { + "format": FORMAT, + "product": product, + "version": version, + "edition": f"editions/{product}/{version}", + "release": release, + "sources": { + "lancedb": source("lancedb", shared), + "enterprise": source("enterprise", enterprise), + }, + } + + +def resolve_release(args: argparse.Namespace, repo: Path) -> dict: + requested = release_inputs(args) + requested["sources"]["build"] = source("build", docs_commit(repo, args.docs)) + return requested + + +def identity(record: dict) -> dict: + """Exclude docs main so preparing again after it advances keeps the saved edition.""" + release = record["release"] + return { + "release": release.get("commit"), + "qualification": (release.get("qualification") or {}).get("object"), + "bundled_lancedb": release.get("bundled_lancedb"), + "enterprise_pages": (release.get("enterprise_pages") or {}).get("commit"), + "lancedb": record["sources"]["lancedb"]["commit"], + "enterprise": record["sources"]["enterprise"]["commit"], + } + + +# records + + +def record_path(repo: Path, product: str, version: str) -> Path: + return repo / "releases" / f"{product}-{version}.yaml" + + +def load_record(path: Path) -> dict: + """A release record that holds everything `prepare` writes.""" + name = f"releases/{path.name}" + try: + record = yaml.safe_load(path.read_text(encoding="utf-8")) + except (OSError, yaml.YAMLError) as exc: + raise EditionError(f"{name}: unreadable: {exc}") from None + problems = record_problems(record, path.name) + if problems: + raise EditionError(f"{name}: " + "; ".join(problems)) + return record + + +MISSING = object() + + +def record_problems(record: object, filename: str) -> list[str]: + """Validate fields offline; commit existence is not checked. + + Compare fields only after validating their formats to avoid duplicate errors. + """ + problems: list[str] = [] + + def lookup(path: str) -> object: + value = record + for key in path.split("."): + if not isinstance(value, dict) or key not in value: + return MISSING + value = value[key] + return value + + def fields(path: str, required: tuple[str, ...], optional: tuple[str, ...] = ()) -> bool: + """Whether `path` is a mapping; reports any field it should not have.""" + value = lookup(path) if path else record + if value is MISSING: + problems.append(f"{path} is missing") + return False + if not isinstance(value, dict): + problems.append(f"{path or 'the record'} is not a mapping") + return False + for key in sorted(set(value) - set(required) - set(optional), key=str): + problems.append(f"{path + '.' if path else ''}{key} is not a field here") + return True + + def check(path: str, valid, expected: str) -> object: + """The value at `path` if `valid` accepts it, else None and a problem.""" + value = lookup(path) + if value is MISSING: + problems.append(f"{path} is missing") + elif not valid(value): + problems.append(f"{path} is {value!r}, not {expected}") + else: + return value + return None + + def commit(value: object) -> bool: + return isinstance(value, str) and COMMIT_RE.fullmatch(value) is not None + + def checksum(value: object) -> bool: + return isinstance(value, str) and candidate.CHECKSUM_RE.fullmatch(value) is not None + + def line(value: object) -> bool: + return isinstance(value, str) and bool(value.strip()) and "\n" not in value.strip() + + top = ("format", "product", "version", "edition", "release", "sources", "prepared", "corrections") + if not fields("", top): + return problems + if check("format", lambda v: v == FORMAT and not isinstance(v, bool), str(FORMAT)) is None: + return problems + product = check("product", lambda v: v in PRODUCTS, " or ".join(PRODUCTS)) + version = check( + "version", lambda v: isinstance(v, str) and VERSION_RE.fullmatch(v), "a version such as 1.2.3" + ) + if product and version: + check("edition", lambda v: v == f"editions/{product}/{version}", f"editions/{product}/{version}") + if filename != f"{product}-{version}.yaml": + problems.append(f"product and version name {product}-{version}.yaml, not {filename}") + + own = "lancedb" if product == "oss" else "enterprise" + extra = ("enterprise_pages",) if product == "oss" else ("qualification", "bundled_lancedb") + if product and fields("release", ("repository", "tag", "commit", *extra)): + check("release.repository", lambda v: v == SOURCES[own][0], SOURCES[own][0]) + if version: + check("release.tag", lambda v: v == f"v{version}", f"v{version}") + check("release.commit", commit, "a commit hash") + if product == "oss": + if fields("release.enterprise_pages", ("ref", "commit")): + check("release.enterprise_pages.ref", line, "the tag or commit named with --enterprise") + check("release.enterprise_pages.commit", commit, "a commit hash") + else: + if fields("release.qualification", ("tag", "object")): + if version: + tag_name = QUALIFIED_TAG.format(version=version) + check("release.qualification.tag", lambda v: v == tag_name, tag_name) + check("release.qualification.object", commit, "a tag object hash") + check("release.bundled_lancedb", commit, "a commit hash") + + if fields("sources", tuple(SOURCES), ("openapi",)): + for name, (repository, folder) in SOURCES.items(): + if fields(f"sources.{name}", ("repository", "path", "commit")): + check(f"sources.{name}.repository", lambda v, r=repository: v == r, repository) + check(f"sources.{name}.path", lambda v, f=folder: v == f, folder) + check(f"sources.{name}.commit", commit, "a commit hash") + if lookup("sources.openapi") is not MISSING and fields( + "sources.openapi", ("repository", "release", "path", "sha256") + ): + for key in ("repository", "release", "path"): + check(f"sources.openapi.{key}", line, "a name") + check("sources.openapi.sha256", checksum, "a SHA-256 checksum") + + if product: + pairs = ( + [("sources.lancedb.commit", "release.commit"), + ("sources.enterprise.commit", "release.enterprise_pages.commit")] + if product == "oss" + else [("sources.enterprise.commit", "release.commit"), + ("sources.lancedb.commit", "release.bundled_lancedb")] + ) + for source_path, release_path in pairs: + a, b = lookup(source_path), lookup(release_path) + if commit(a) and commit(b) and a != b: + problems.append(f"{source_path} is {a}, but {release_path} is {b}") + + if fields("prepared", ("files", "sha256")): + check( + "prepared.files", + lambda v: isinstance(v, int) and not isinstance(v, bool) and v > 0, + "a number of files", + ) + check("prepared.sha256", checksum, "a SHA-256 checksum") + + corrections = check("corrections", lambda v: isinstance(v, list), "a list") + accepted = lookup("prepared.sha256") + for index, entry in enumerate(corrections or []): + where = f"corrections[{index}]" + if not isinstance(entry, dict): + problems.append(f"{where} is not a mapping") + accepted = None + continue + for key in sorted(set(entry) - {"summary", "files", "sha256"}, key=str): + problems.append(f"{where}.{key} is not a field here") + if not line(entry.get("summary")): + problems.append(f"{where}.summary must be one line of text") + files = entry.get("files", []) + if not isinstance(files, list) or not all(isinstance(f, str) and f for f in files): + problems.append(f"{where}.files must be a list of the edition's paths") + sha256 = entry.get("sha256") + if not checksum(sha256): + problems.append( + f"{where}.sha256 is {'missing' if sha256 is None else repr(sha256)}, not the " + "SHA-256 checksum of the corrected edition that `check` reports" + ) + elif sha256 == accepted: + problems.append(f"{where}.sha256 is the checksum before it; a correction changes the edition") + accepted = sha256 + return problems + + +def saved_hashes(repo: Path, edition: str) -> dict[str, str]: + """The edition's files as Git would save them: ignored local files, such as + `.DS_Store`, are not part of it. Preparation refuses an edition that has + any ignored path of its own.""" + hashes = candidate.file_hashes(repo / edition) + listing = "".join(f"{edition}/{p}\0" for p in hashes) + ignored = ignored_paths(repo, listing, index=True) + return {p: h for p, h in hashes.items() if f"{edition}/{p}" not in ignored} + + +def ignored_paths(repo: Path, listing: str, index: bool = False) -> set[str]: + """The NUL-separated paths in `listing` that .gitignore leaves out. + + With `index`, files Git already tracks are never ignored, as for `git add`. + """ + result = subprocess.run( + ["git", "-C", str(repo), "check-ignore", "-z", "--stdin", *([] if index else ["--no-index"])], + input=listing.encode(), capture_output=True, + ) + if result.returncode not in (0, 1): + raise EditionError(f"git check-ignore failed: {result.stderr.decode().strip()}") + return {name for name in result.stdout.decode().split("\0") if name} + + +def snapshot_state(repo: Path, record: dict) -> str: + """Require the latest recorded checksum, including corrections.""" + current = candidate.site_checksum(saved_hashes(repo, record["edition"])) + corrections = record["corrections"] + accepted = corrections[-1]["sha256"] if corrections else record["prepared"]["sha256"] + if current != accepted: + since = "its latest recorded correction" if corrections else "it was prepared" + raise EditionError( + f"{record['edition']} has changed since {since}; its files now have checksum " + f"{current}. For a reviewed correction, add an entry under `corrections` in " + f"releases/{record['product']}-{record['version']}.yaml with `sha256: {current}`" + ) + if not corrections: + return "as prepared" + count = len(corrections) + return f"corrected ({count} correction{'s' if count > 1 else ''})" + + +# prepare + + +def existing(repo: Path, requested: dict) -> str: + """What to say about a release that already has a record or an edition.""" + product, version = requested["product"], requested["version"] + path = record_path(repo, product, version) + folder = repo / requested["edition"] + if not path.exists(): + raise EditionError( + f"{requested['edition']} exists without releases/{path.name}; an edition " + "is never replaced, so remove an incomplete one in a reviewed change first" + ) + record = load_record(path) + if not folder.is_dir(): + raise EditionError( + f"releases/{path.name} exists, but {requested['edition']} does not; restore " + "it from Git history rather than preparing it again" + ) + recorded, wanted = identity(record), identity(requested) + if recorded != wanted: + changes = "; ".join( + f"{key} recorded {recorded[key]}, requested {wanted[key]}" + for key in recorded + if recorded[key] != wanted[key] + ) + raise EditionError( + f"{product} {version} is already saved from different inputs ({changes}); " + "an edition is never replaced, so review the difference instead" + ) + try: + state = snapshot_state(repo, record) + except EditionError as exc: + raise EditionError(f"{product} {version} is already saved; nothing changed, but {exc}") from None + return ( + f"{product} {version} is already saved from these release inputs: " + f"{requested['edition']} is {state}; nothing changed" + ) + + +def build_config(raw: dict, roots: dict[str, Path], output: Path, path: Path) -> Path: + """assemble.yaml as committed, reading the exported roots into `output`.""" + raw = dict(raw, roots=[dict(entry) for entry in raw.get("roots", [])]) + names = [entry.get("name") for entry in raw.get("roots", [])] + if sorted(names) != sorted(SOURCES): + raise EditionError( + f"assemble.yaml has roots {names}, but edition.py records the sources of " + f"{sorted(SOURCES)}; add every root to SOURCES" + ) + for entry in raw["roots"]: + entry["path"] = str(roots[entry["name"]]) + raw["output"] = str(output) + if raw.get("openapi"): + # The spec is a file of the docs root; check the exported copy. + dest = PurePosixPath(raw["openapi"]["dest"]).relative_to(SOURCES["build"][1]) + raw["openapi"] = dict(raw["openapi"], dest=str(roots["build"] / dest)) + path.write_text(yaml.safe_dump(raw, sort_keys=False), encoding="utf-8") + return path + + +def openapi_source(raw: dict, hashes: dict[str, str]) -> dict | None: + """The released spec the docs root ships, and the hash of the saved copy.""" + spec = raw.get("openapi") + if not spec: + return None + rel = PurePosixPath(spec["dest"]).relative_to(SOURCES["build"][1]).as_posix() + return { + "repository": spec["repo"], + "release": spec["release"], + "path": spec["source"], + "sha256": hashes.get(rel), + } + + +def place(staging: Path, site: Path, record: str, repo: Path, edition: Path, path: Path) -> None: + """Move the edition into place, then create its record; undo both on failure.""" + created = [d for d in (edition.parent.parent, edition.parent, path.parent) if not d.exists()] + moved = claimed = False + try: + for directory in (edition.parent, path.parent): + directory.mkdir(parents=True, exist_ok=True) + if os.path.lexists(edition) or os.path.lexists(path): + raise EditionError(f"{edition.relative_to(repo)} appeared while preparing; nothing saved") + os.rename(site, edition) + moved = True + # Claim the record's name, then put the whole record there at once. + os.close(os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL)) + claimed = True + draft = staging / "record.yaml" + draft.write_text(record, encoding="utf-8") + os.replace(draft, path) + except BaseException: + if claimed: + path.unlink() + if moved: + os.rename(edition, site) + for directory in reversed(created): + if directory.exists() and not any(directory.iterdir()): + directory.rmdir() + raise + + +def prepare(args: argparse.Namespace, repo: Path = REPO_ROOT) -> str: + requested = release_inputs(args) + edition = repo / requested["edition"] + path = record_path(repo, requested["product"], requested["version"]) + if os.path.lexists(edition) or os.path.lexists(path): + return existing(repo, requested) + + requested["sources"]["build"] = source("build", docs_commit(repo, args.docs)) + (repo / "build").mkdir(exist_ok=True) + staging = Path(tempfile.mkdtemp(prefix="edition-", dir=repo / "build")).resolve() + try: + roots = {} + for name, (repository, folder) in SOURCES.items(): + roots[name] = staging / "roots" / name + commit = requested["sources"][name]["commit"] + source_repo = {"lancedb": args.lancedb, "enterprise": args.enterprise_repo}.get(name, repo) + export(source_repo.resolve(), commit, folder, roots[name], f"{repository} {commit}") + site = staging / "site" + build = requested["sources"]["build"]["commit"] + raw = yaml.safe_load(git(repo, "show", f"{build}:assemble.yaml").decode()) + config = assemble.load_config(build_config(raw, roots, site, staging / "assemble.yaml")) + _count, warnings = assemble.assemble_site(config) + for warning in warnings: + print(f"warning: {warning}", file=sys.stderr) + + hashes = candidate.file_hashes(site) + control = sorted(p for p in hashes if PurePosixPath(p).name in GIT_CONTROL) + if control: + raise EditionError(f"the edition would hold {control}, which change how Git stores it") + listing = "".join(f"{requested['edition']}/{p}\0" for p in hashes) + f"releases/{path.name}\0" + ignored = sorted(ignored_paths(repo, listing)) + if ignored: + raise EditionError(f".gitignore would leave out {ignored[:5]}; the edition must be saved whole") + + openapi = openapi_source(raw, hashes) + record = { + **requested, + "sources": {**requested["sources"], **({"openapi": openapi} if openapi else {})}, + "prepared": {"files": len(hashes), "sha256": candidate.site_checksum(hashes)}, + "corrections": [], + } + place(staging, site, HEADER + yaml.safe_dump(record, sort_keys=False), repo, edition, path) + except (assemble.AssembleError, candidate.CandidateError) as exc: + raise EditionError(str(exc)) from None + finally: + shutil.rmtree(staging, ignore_errors=True) + try: + (repo / "build").rmdir() + except OSError: + pass + sources = " ".join(f"{k}={v['commit']}" for k, v in requested["sources"].items()) + return ( + f"prepared {requested['product']} {requested['version']}: {len(hashes)} files, " + f"sha256 {record['prepared']['sha256']}, in {requested['edition']} with " + f"releases/{path.name}\nsources: {sources}" + ) + + +# check + + +def differences(was: object, now: object, path: str) -> list[str]: + if isinstance(was, dict) and isinstance(now, dict): + return [ + change + for key in sorted(set(was) | set(now), key=str) + for change in differences( + was.get(key, MISSING), now.get(key, MISSING), f"{path}.{key}" if path else str(key) + ) + ] + if type(was) is type(now) and was == now: + return [] + return [f"{path} {'was removed' if now is MISSING else 'was added' if was is MISSING else 'changed'}"] + + +def preserved(repo: Path, base: str) -> list[str]: + """Require base editions and records to remain; only append corrections. + + Compare parsed records so formatting changes are harmless. + """ + problems = [] + listing = git(repo, "ls-tree", "-z", "--name-only", base, "--", "releases/").decode() + for rel in sorted(name for name in listing.split("\0") if name.endswith(".yaml")): + path = repo / rel + if not path.is_file(): + problems.append(f"{rel} is on the base {base} but has been removed; a saved release keeps its record") + continue + try: + was = yaml.safe_load(git(repo, "show", f"{base}:{rel}")) + now = yaml.safe_load(path.read_text(encoding="utf-8")) + except (OSError, yaml.YAMLError): + was = now = None + if not isinstance(was, dict) or not isinstance(now, dict): + # `check` reports a malformed record by itself, and one on the base + # can only be repaired by changing it. + continue + changes = differences( + {k: v for k, v in was.items() if k != "corrections"}, + {k: v for k, v in now.items() if k != "corrections"}, + "", + ) + before, after = was.get("corrections"), now.get("corrections") + after = after if isinstance(after, list) else [] + for index, entry in enumerate(before if isinstance(before, list) else []): + changes += differences(entry, after[index] if index < len(after) else MISSING, f"corrections[{index}]") + if changes: + problems.append( + f"{rel} differs from the base {base}: {', '.join(changes)}; a saved release's " + "record only gains corrections, at the end" + ) + listing = git(repo, "ls-tree", "-z", "-d", "--name-only", base, "--", + *(f"editions/{product}/" for product in PRODUCTS)).decode() + for rel in sorted(name for name in listing.split("\0") if name): + _editions, product, version = rel.split("/") + # A missing edition whose record remains is reported with the record. + if not (repo / rel).is_dir() and not record_path(repo, product, version).is_file(): + problems.append(f"{rel} is on the base {base} but has been removed; a saved edition is never removed") + return problems + + +def check(repo: Path = REPO_ROOT, base: str | None = None) -> tuple[list[str], list[str]]: + """Return edition statuses and errors, optionally enforcing preservation from base.""" + lines, problems, paired = [], [], set() + for path in sorted((repo / "releases").glob("*.yaml")): + # The file name says which edition is this record's, even when the + # record itself is malformed. + paired.add("editions/" + path.stem.replace("-", "/", 1)) + try: + record = load_record(path) + if not (repo / record["edition"]).is_dir(): + raise EditionError(f"releases/{path.name}: {record['edition']} is missing") + lines.append(f"{record['product']} {record['version']}: {snapshot_state(repo, record)}") + except (EditionError, candidate.CandidateError) as exc: + problems.append(str(exc)) + editions = repo / "editions" + entries = sorted([*editions.glob("*"), *editions.glob("*/*")]) if editions.is_dir() else [] + names = [entry.relative_to(repo).as_posix() for entry in entries] + ignored = ignored_paths(repo, "".join(f"{name}\0" for name in names), index=True) + for entry, rel in zip(entries, names): + if rel in ignored: + continue + if entry.parent == editions: + if entry.name not in PRODUCTS or not entry.is_dir(): + problems.append(f"{rel} is not a product folder") + elif rel not in paired: + problems.append(f"{rel} has no release record") + if base: + problems += preserved(repo, commit_of(repo, base, "--base")) + return lines, problems + + +def parse(argv: list[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__.split("\n")[1]) + commands = parser.add_subparsers(dest="command", required=True) + prep = commands.add_parser("prepare", help="save one release's edition") + prep.add_argument("product", choices=PRODUCTS) + prep.add_argument("version", help="the release version, such as 1.2.3") + prep.add_argument("--enterprise", metavar="REF", + help="OSS only: the Enterprise tag or commit for the Enterprise pages") + prep.add_argument("--bundled-lancedb", metavar="COMMIT", + help="Enterprise only: the LanceDB commit the release is expected to bundle") + prep.add_argument("--lancedb", type=Path, default=REPO_ROOT.parent / "lancedb", + help="a lancedb/lancedb repository (default: ../lancedb)") + prep.add_argument("--enterprise-repo", type=Path, default=REPO_ROOT.parent / "enterprise", + help="an Enterprise source repository (default: ../enterprise)") + prep.add_argument("--docs", metavar="REF", + help="the commit of lancedb/docs main to read docs/ and assemble.yaml from " + "(default: main as fetched from origin when preparing)") + checking = commands.add_parser("check", help="check the saved editions against their records") + checking.add_argument("--base", metavar="COMMIT", + help="also require the editions and records on COMMIT, such as a pull " + "request's base, to be kept") + return parser.parse_args(argv) + + +def main(argv: list[str] | None = None) -> int: + args = parse(argv) + try: + if args.command == "prepare": + print(prepare(args)) + return 0 + lines, problems = check(base=args.base) + except (EditionError, candidate.CandidateError) as exc: + print(f"edition: {exc}", file=sys.stderr) + return 1 + if lines or not problems: + print("\n".join(lines) or "no saved editions") + for problem in problems: + print(f"edition: {problem}", file=sys.stderr) + return 1 if problems else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/tests/test_assemble.py b/scripts/tests/test_assemble.py index 9feef853..a92d0a8e 100644 --- a/scripts/tests/test_assemble.py +++ b/scripts/tests/test_assemble.py @@ -3,7 +3,7 @@ Each test builds a small site in a temporary directory: a reference root that owns `enterprise/security` and the navigation, and a private overlay root at -`sophon/docs/web` beside an internal folder that must never be published. +`enterprise_repo/docs/web` beside an internal folder that must never be published. Run with `make test-assemble`. """ @@ -52,23 +52,23 @@ def write(path: Path, text: str) -> Path: def site(base: Path, *, checkout: bool = True) -> dict[str, Path]: - """A reference root, a Sophon-like checkout, and an internal folder.""" + """A reference root, an Enterprise checkout, and an internal folder.""" reference = base / "lancedb/docs/web" write(reference / "docs.json", json.dumps(NAVIGATION)) write(reference / "enterprise/security.mdx", "Open-source security page\n") write(reference / "static/styles/style.css", "body {}\n") - sophon = base / "sophon" + enterprise_repo = base / "enterprise_repo" if checkout: - (sophon / ".git").mkdir(parents=True) - write(sophon / "docs/ci.md", INTERNAL) + (enterprise_repo / ".git").mkdir(parents=True) + write(enterprise_repo / "docs/ci.md", INTERNAL) internal = base / "internal" write(internal / "enterprise/security.mdx", INTERNAL) write(internal / "web/enterprise/security.mdx", INTERNAL) return { "reference": reference, - "sophon": sophon, - "public": sophon / "docs/web", + "enterprise_repo": enterprise_repo, + "public": enterprise_repo / "docs/web", "internal": internal, } @@ -137,11 +137,11 @@ def test_symlinked_private_root_is_refused(base, checkout): @pytest.mark.parametrize("checkout", [True, False]) def test_symlinked_parent_of_private_root_is_refused(base, checkout): paths = site(base, checkout=checkout) - (paths["sophon"] / "docs/ci.md").unlink() - (paths["sophon"] / "docs").rmdir() - (paths["sophon"] / "docs").symlink_to(paths["internal"], target_is_directory=True) + (paths["enterprise_repo"] / "docs/ci.md").unlink() + (paths["enterprise_repo"] / "docs").rmdir() + (paths["enterprise_repo"] / "docs").symlink_to(paths["internal"], target_is_directory=True) - with pytest.raises(assemble.AssembleError, match=r"sophon/docs is a symlink"): + with pytest.raises(assemble.AssembleError, match=r"enterprise_repo/docs is a symlink"): build(config(base, paths["reference"], paths["public"])) @@ -149,10 +149,10 @@ def test_symlinked_checkout_is_refused(base): paths = site(base) public_root(paths) elsewhere = base / "elsewhere" - paths["sophon"].rename(elsewhere) - paths["sophon"].symlink_to(elsewhere, target_is_directory=True) + paths["enterprise_repo"].rename(elsewhere) + paths["enterprise_repo"].symlink_to(elsewhere, target_is_directory=True) - with pytest.raises(assemble.AssembleError, match=r"sophon is a symlink"): + with pytest.raises(assemble.AssembleError, match=r"enterprise_repo is a symlink"): build(config(base, paths["reference"], paths["public"])) @@ -161,7 +161,7 @@ def test_symlink_above_the_checkout_is_allowed(base): paths = site(real) public_root(paths) (base / "linked").symlink_to(real, target_is_directory=True) - overlay = base / "linked/sophon/docs/web" + overlay = base / "linked/enterprise_repo/docs/web" output = build(config(base, paths["reference"], overlay)) diff --git a/scripts/tests/test_candidate.py b/scripts/tests/test_candidate.py index 05e31c93..1db38007 100644 --- a/scripts/tests/test_candidate.py +++ b/scripts/tests/test_candidate.py @@ -719,7 +719,7 @@ def test_independent_review_probes(tmp_path, remote, case): def assemble_sources(base: Path, index: bytes) -> dict: - """lancedb, sophon and docs checkouts with the shape assemble.yaml reads.""" + """LanceDB, Enterprise and docs checkouts with the shape assemble.yaml reads.""" navigation = { "navigation": { "tabs": [ @@ -739,7 +739,7 @@ def assemble_sources(base: Path, index: bytes) -> dict: "docs/web/index.mdx": index, "docs/web/enterprise/security.mdx": b"Open-source security\n", }, - "sophon": {"docs/web/enterprise/security.mdx": b"Enterprise security\n"}, + "enterprise_repo": {"docs/web/enterprise/security.mdx": b"Enterprise security\n"}, "docs": {"docs/geneva/udfs.mdx": b"Geneva UDFs\n"}, }.items(): repo = base / name @@ -760,7 +760,7 @@ def assemble(base: Path, repos: dict, output: Path) -> str: path: {repos["lancedb"] / "docs/web"} role: reference - name: enterprise - path: {repos["sophon"] / "docs/web"} + path: {repos["enterprise_repo"] / "docs/web"} role: overlay private: true - name: build diff --git a/scripts/tests/test_edition.py b/scripts/tests/test_edition.py new file mode 100644 index 00000000..12553d91 --- /dev/null +++ b/scripts/tests/test_edition.py @@ -0,0 +1,1217 @@ +"""Tests for preparing saved editions from release records. + +Disposable repositories stand in for LanceDB, Enterprise and this +repository. Their versions are fixtures, not releases: OSS 0.94.0 to 0.99.0 and +Enterprise 8.1.0 to 8.10.0, each built to exercise one case. The docs fixture +uses this repository's own assemble.yaml and .gitignore, and every edition is +built by the real assembler. + +Run with `make test-assemble`. +""" + +import copy +import hashlib +import json +import os +import re +import shutil +import subprocess +import sys +from dataclasses import dataclass +from pathlib import Path + +import pytest +import yaml + +SCRIPTS = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(SCRIPTS)) + +import edition # noqa: E402 + +# Allow local stand-ins without adding a production URL-check bypass. +LANCEDB_DOCS_URLS = edition.DOCS_URLS +TEST_DOCS_URLS = re.compile(rf"{LANCEDB_DOCS_URLS.pattern}|/.+/lancedb-docs\.git", re.IGNORECASE) + +NAVIGATION = { + "navigation": { + "tabs": [ + { + "tab": "Documentation", + "groups": [ + {"group": "Start", "pages": ["index", "guide"]}, + {"group": "Enterprise", "pages": ["enterprise/security"]}, + ], + } + ] + } +} +GENEVA_NAV = {"tabs": [{"tab": "Geneva", "groups": [{"group": "Geneva", "pages": ["geneva/index"]}]}]} +MARKER = bytes(range(256)) +INTERNAL = b"Internal engineering notes\n" + + +@dataclass(frozen=True) +class Gitlink: + commit: str + + +@dataclass(frozen=True) +class Symlink: + target: str + + +def environment() -> dict[str, str]: + env = {k: v for k, v in os.environ.items() if not k.startswith("GIT_")} + env.update( + GIT_AUTHOR_NAME="Test", + GIT_AUTHOR_EMAIL="test@example.com", + GIT_COMMITTER_NAME="Test", + GIT_COMMITTER_EMAIL="test@example.com", + GIT_CONFIG_GLOBAL=os.devnull, + GIT_CONFIG_NOSYSTEM="1", + GIT_TERMINAL_PROMPT="0", + ) + return env + + +@pytest.fixture(autouse=True) +def isolated_git(monkeypatch): + """The builder's own git calls see no user configuration either.""" + for key in [k for k in os.environ if k.startswith("GIT_")]: + monkeypatch.delenv(key) + for key, value in environment().items(): + if key.startswith("GIT_"): + monkeypatch.setenv(key, value) + + +@pytest.fixture(autouse=True) +def stand_in_for_lancedb_docs(monkeypatch): + monkeypatch.setattr(edition, "DOCS_URLS", TEST_DOCS_URLS) + + +def git(cwd: Path, *args: str, data: bytes | None = None) -> str: + return subprocess.run( + ["git", *args], cwd=cwd, env=environment(), input=data, check=True, capture_output=True + ).stdout.decode().strip() + + +def snapshot(repo: Path, files: dict, message: str) -> str: + """Commit exactly `files` on main, whatever the working tree holds.""" + lines = [] + for path, content in sorted(files.items()): + if isinstance(content, Gitlink): + lines.append(f"160000 commit {content.commit}\t{path}") + continue + data = content.target.encode() if isinstance(content, Symlink) else content + blob = git(repo, "hash-object", "-w", "--stdin", data=data) + lines.append(f"{'120000' if isinstance(content, Symlink) else '100644'} blob {blob}\t{path}") + index = repo / ".git" / "fixture-index" + env = {**environment(), "GIT_INDEX_FILE": str(index)} + subprocess.run(["git", "read-tree", "--empty"], cwd=repo, env=env, check=True) + subprocess.run(["git", "update-index", "--index-info"], cwd=repo, env=env, check=True, + input="".join(f"{line}\n" for line in lines).encode()) + tree = subprocess.run(["git", "write-tree"], cwd=repo, env=env, check=True, + capture_output=True).stdout.decode().strip() + index.unlink() + parent = subprocess.run(["git", "rev-parse", "--verify", "--quiet", "HEAD"], cwd=repo, + env=environment(), capture_output=True).stdout.decode().strip() + commit = git(repo, "commit-tree", tree, *(["-p", parent] if parent else []), "-m", message) + git(repo, "update-ref", "refs/heads/main", commit) + return commit + + +def annotated(repo: Path, name: str, commit: str) -> str: + git(repo, "tag", "-a", "-m", name, name, commit) + return git(repo, "rev-parse", f"refs/tags/{name}") + + +def new_repo(path: Path) -> Path: + path.mkdir(parents=True) + git(path, "init", "-q", "-b", "main") + return path + + +def oss_pages(label: str) -> dict: + """Minimal docs/web fixture, including the required Enterprise placeholder.""" + web = "docs/web/" + guide = ( + "---\ntitle: Guide\n---\n\n" + 'import { PyGuide } from "/snippets/guide.mdx";\n\n' + f"Shared guide as of {label}.\n\n" + '\n{PyGuide}\n\n\n' + "{/*\nDraft for the next release.\n\n## Coming soon {#coming-soon}\n*/}\n" + ) + return { + "README.md": b"Not documentation\n", + "python/lancedb.py": b"print('library code')\n", + web + "docs.json": json.dumps(NAVIGATION).encode(), + web + "index.mdx": f"# LanceDB {label}\n".encode(), + web + "guide.mdx": guide.encode(), + web + "snippets/guide.mdx": f"export const PyGuide = {json.dumps(f'print({label!r})' + chr(10))};\n".encode(), + web + "static/assets/marker.svg": MARKER, + web + "enterprise/security.mdx": b"Open-source security placeholder\n", + web + ".mintignore": b"draft.mdx\n", + web + "draft.mdx": b"# Not released yet\n", + } + + +def enterprise_pages(label: str, bundled: str | None, **extra) -> dict: + files = { + "README.md": b"Not documentation\n", + "docs/ci.md": INTERNAL, + "docs/internal/notes.mdx": INTERNAL, + "docs/web/enterprise/security.mdx": f"Enterprise security for {label}\n".encode(), + **extra, + } + if bundled: + files["src/lancedb"] = Gitlink(bundled) + return files + + +@pytest.fixture(scope="module") +def sources(tmp_path_factory) -> dict: + """LanceDB and Enterprise repositories with tagged fixture releases. Shared, so + tests only read them.""" + base = tmp_path_factory.mktemp("sources").resolve() + lancedb = new_repo(base / "lancedb") + c = {} + c["no-docs"] = snapshot(lancedb, {"README.md": b"Before docs/web\n"}, "no docs yet") + git(lancedb, "tag", "v0.97.0", c["no-docs"]) + c["bundled"] = snapshot(lancedb, oss_pages("the bundled revision"), "bundled by 8.1") + c["ignored"] = snapshot(lancedb, {**oss_pages("0.95.0"), "docs/web/scratch/notes.mdx": b"x\n"}, "0.95") + git(lancedb, "tag", "v0.95.0", c["ignored"]) + control = {"docs/web/.gitattributes": b"* -text\n", "docs/web/static/.gitignore": b"*.svg\n"} + c["control"] = snapshot(lancedb, {**oss_pages("0.94.0"), **control}, "0.94") + git(lancedb, "tag", "v0.94.0", c["control"]) + c["0.98.0"] = snapshot(lancedb, oss_pages("0.98.0"), "0.98") + annotated(lancedb, "v0.98.0", c["0.98.0"]) + c["0.99.0"] = snapshot(lancedb, oss_pages("0.99.0"), "0.99") + annotated(lancedb, "v0.99.0", c["0.99.0"]) + c["main"] = snapshot(lancedb, oss_pages("unreleased main"), "main moves on") + # A working tree that differs from every commit. + (lancedb / "docs/web").mkdir(parents=True) + (lancedb / "docs/web/guide.mdx").write_text("Uncommitted text\n") + + enterprise_repo = new_repo(base / "enterprise_repo") + e = {} + + def release(version: str, files: dict, qualified: str | None = "self") -> str: + commit = snapshot(enterprise_repo, files, version) + annotated(enterprise_repo, f"v{version}", commit) + if qualified == "lightweight": + git(enterprise_repo, "tag", f"release-qualified/v{version}", commit) + elif qualified: + annotated(enterprise_repo, f"release-qualified/v{version}", commit if qualified == "self" else qualified) + e[version] = commit + return commit + + release("8.1.0", enterprise_pages("8.1", c["bundled"])) + release("8.2.0", enterprise_pages("8.2", c["0.99.0"])) + release("8.3.0", enterprise_pages("8.3", c["0.99.0"]), qualified=None) + release("8.4.0", enterprise_pages("8.4", c["0.99.0"]), qualified=e["8.3.0"]) + release("8.5.0", enterprise_pages("8.5", c["no-docs"])) + release("8.6.0", enterprise_pages("8.6", None)) + release("8.7.0", enterprise_pages("8.7", c["0.99.0"], **{"docs/web/.notes.mdx": INTERNAL})) + release("8.8.0", enterprise_pages("8.8", c["0.99.0"], **{"docs/web/enterprise/ci.md": Symlink("../../ci.md")})) + release("8.9.0", enterprise_pages("8.9", "1" * 40)) + release("8.10.0", enterprise_pages("8.10", c["0.99.0"]), qualified="lightweight") + e["main"] = snapshot(enterprise_repo, enterprise_pages("unreleased main", c["main"]), "main moves on") + return {"lancedb": lancedb, "enterprise_repo": enterprise_repo, "oss": c, "enterprise": e} + + +@pytest.fixture +def docs(tmp_path) -> Path: + return docs_checkout(tmp_path.resolve() / "docs") + + +def docs_checkout(path: Path) -> Path: + """Docs checkout with a local bare origin standing in for canonical main.""" + repo = new_repo(path) + for name in ("assemble.yaml", ".gitignore"): + shutil.copy(SCRIPTS.parent / name, repo / name) + files = { + "docs/docs.nav.json": json.dumps(GENEVA_NAV), + "docs/geneva/index.mdx": "# Geneva\n", + "docs/api-reference/rest/openapi.yml": "openapi: 3.0.0\n", + } + for rel, content in files.items(): + (repo / rel).parent.mkdir(parents=True, exist_ok=True) + (repo / rel).write_text(content) + git(repo, "add", "-A") + git(repo, "commit", "-q", "-m", "docs") + origin = path.parent / f"{path.name}-upstream" / "lancedb-docs.git" + git(path.parent, "init", "-q", "--bare", "-b", "main", str(origin)) + git(repo, "remote", "add", "origin", str(origin)) + on_main(repo) + return repo + + +def on_main(docs: Path) -> str: + """Make HEAD available through the canonical-main stand-in.""" + git(docs, "push", "-q", "--force", "origin", "HEAD:refs/heads/main") + return git(docs, "rev-parse", "HEAD") + + +def prepare(docs: Path, sources: dict, *argv: str, lancedb: Path | None = None) -> str: + args = edition.parse([ + "prepare", *argv, + "--lancedb", str(lancedb or sources["lancedb"]), "--enterprise-repo", str(sources["enterprise_repo"]), + ]) + return edition.prepare(args, repo=docs) + + +def saved(root: Path) -> dict[str, bytes]: + """Every file under `root`, hidden ones included.""" + return { + p.relative_to(root).as_posix(): p.read_bytes() + for p in sorted(root.rglob("*")) + if p.is_file() + } + + +def checksum(files: dict[str, bytes]) -> str: + listing = "".join(f"{hashlib.sha256(files[p]).hexdigest()} {p}\n" for p in sorted(files)) + return hashlib.sha256(listing.encode()).hexdigest() + + +def record(docs: Path, name: str) -> dict: + return yaml.safe_load((docs / "releases" / f"{name}.yaml").read_text()) + + +def state(docs: Path) -> dict[str, bytes | None]: + """What preparation may change: editions, records and leftovers in build/, + directories included.""" + found = {} + for top in ("editions", "releases", "build"): + root = docs / top + if root.exists(): + found[top] = None + for path in sorted(root.rglob("*")): + found[f"{top}/{path.relative_to(root).as_posix()}"] = ( + path.read_bytes() if path.is_file() else None + ) + return found + + +def correct(docs: Path, name: str, page: str, text: str) -> str: + """Apply the README's edit-and-record correction workflow.""" + product, version = name.split("-") + (docs / "editions" / product / version / page).write_text(text) + return record_correction(docs, name, page) + + +def record_correction(docs: Path, name: str, page: str) -> str: + """Add a correction entry with the checksum `check` reports for the edition.""" + product, version = name.split("-") + _lines, problems = edition.check(docs) + reported = [p for p in problems if p.startswith(f"editions/{product}/{version} has changed")] + sha256 = re.search(r"`sha256: ([0-9a-f]{64})`", reported[0])[1] + add_correction(docs, name, page, sha256) + return sha256 + + +def add_correction(docs: Path, name: str, page: str, sha256: str) -> None: + path = docs / "releases" / f"{name}.yaml" + entry = f"- summary: Fix {page}.\n files: [{page}]\n sha256: {sha256}\n" + path.write_text(path.read_text().replace("corrections: []\n", "corrections:\n") + entry) + + +def commit_all(docs: Path, message: str) -> str: + git(docs, "add", "-A") + git(docs, "commit", "-q", "-m", message) + return git(docs, "rev-parse", "HEAD") + + +# preparing editions + + +def test_oss_edition_is_built_from_its_tag_and_named_enterprise_pages(docs, sources): + out = prepare(docs, sources, "oss", "0.98.0", "--enterprise", "v8.1.0") + + files = saved(docs / "editions/oss/0.98.0") + assert sorted(files) == [ + ".mintignore", "api-reference/rest/openapi.yml", "docs.json", "draft.mdx", + "enterprise/security.mdx", "geneva/index.mdx", "guide.mdx", "index.mdx", + "static/assets/marker.svg", + ] + guide = files["guide.mdx"].decode() + assert "Shared guide as of 0.98.0." in guide + assert "unreleased main" not in guide and "Uncommitted" not in guide + assert "```python Python icon=\"python\"\nprint('0.98.0')\n```" in guide + assert "/snippets/" not in guide + assert "{/*\nDraft for the next release." in guide + assert files[".mintignore"] == b"draft.mdx\n" + assert files["static/assets/marker.svg"] == MARKER + assert files["enterprise/security.mdx"] == b"Enterprise security for 8.1\n" + assert json.loads(files["docs.json"])["navigation"]["tabs"][1]["tab"] == "Geneva" + + oss, ent = sources["oss"], sources["enterprise"] + rec = record(docs, "oss-0.98.0") + assert rec["release"] == { + "repository": "lancedb/lancedb", + "tag": "v0.98.0", + "commit": oss["0.98.0"], + "enterprise_pages": {"ref": "v8.1.0", "commit": ent["8.1.0"]}, + } + assert rec["sources"] == { + "lancedb": {"repository": "lancedb/lancedb", "path": "docs/web", "commit": oss["0.98.0"]}, + "enterprise": {"repository": "lancedb/sophon", "path": "docs/web", "commit": ent["8.1.0"]}, + "build": {"repository": "lancedb/docs", "path": "docs", "commit": git(docs, "rev-parse", "origin/main")}, + "openapi": { + "repository": "lance-format/lance-namespace", + "release": yaml.safe_load((docs / "assemble.yaml").read_text())["openapi"]["release"], + "path": "docs/src/spec.yaml", + "sha256": hashlib.sha256(b"openapi: 3.0.0\n").hexdigest(), + }, + } + assert rec["prepared"] == {"files": len(files), "sha256": checksum(files)} + assert rec["corrections"] == [] + assert "prepared oss 0.98.0: 9 files" in out + assert not (docs / "build").exists() + + +def test_enterprise_edition_reads_the_lancedb_it_bundles(docs, sources): + bundled = sources["oss"]["bundled"] + prepare(docs, sources, "enterprise", "8.1.0", "--bundled-lancedb", bundled) + + files = saved(docs / "editions/enterprise/8.1.0") + # The bundled revision has no OSS release tag or saved edition. + assert "Shared guide as of the bundled revision." in files["guide.mdx"].decode() + assert files["enterprise/security.mdx"] == b"Enterprise security for 8.1\n" + assert not (docs / "editions/oss").exists() + assert not any(INTERNAL in data for data in files.values()) + + rec = record(docs, "enterprise-8.1.0") + enterprise_repo = sources["enterprise_repo"] + assert rec["release"] == { + "repository": "lancedb/sophon", + "tag": "v8.1.0", + "commit": sources["enterprise"]["8.1.0"], + "qualification": { + "tag": "release-qualified/v8.1.0", + "object": git(enterprise_repo, "rev-parse", "refs/tags/release-qualified/v8.1.0"), + }, + "bundled_lancedb": bundled, + } + assert rec["sources"]["lancedb"]["commit"] == bundled + assert rec["sources"]["enterprise"]["commit"] == sources["enterprise"]["8.1.0"] + + +@pytest.mark.parametrize( + "argv, error", + [ + (["oss", "0.97.0", "--enterprise", "v8.1.0"], "has no docs/web"), + (["oss", "0.96.0", "--enterprise", "v8.1.0"], "has no tag v0.96.0"), + (["oss", "1.0.0-beta.1", "--enterprise", "v8.1.0"], "not a stable release version"), + (["oss", "0.98.0"], "name the Enterprise tag or commit"), + (["oss", "0.98.0", "--enterprise", "main"], "not a branch"), + (["oss", "0.98.0", "--enterprise", "v8.1.0", "--bundled-lancedb", "1" * 40], "Enterprise releases only"), + (["oss", "0.95.0", "--enterprise", "v8.1.0"], ".gitignore would leave out"), + (["oss", "0.94.0", "--enterprise", "v8.1.0"], + r"would hold \['.gitattributes', 'static/.gitignore'\], which change how Git stores it"), + (["enterprise", "8.1.0", "--enterprise", "v8.2.0"], "applies to OSS releases"), + (["enterprise", "8.3.0"], "is not qualified"), + (["enterprise", "8.4.0"], "but release-qualified/v8.4.0 qualifies"), + (["enterprise", "8.10.0"], "release-qualified/v8.10.0 is not an annotated tag"), + (["enterprise", "8.5.0"], "has no docs/web"), + (["enterprise", "8.6.0"], "records no bundled LanceDB"), + (["enterprise", "8.1.0", "--bundled-lancedb", "1" * 40], "disagrees with Enterprise v8.1.0"), + (["enterprise", "8.9.0"], "fetch it first"), + (["enterprise", "8.7.0"], r"\.notes\.mdx is hidden"), + (["enterprise", "8.8.0"], "ci.md is a symlink"), + ], +) +def test_preparation_stops_and_saves_nothing(docs, sources, argv, error): + with pytest.raises(edition.EditionError, match=error): + prepare(docs, sources, *argv) + + assert state(docs) == {} + + +def test_bundled_lancedb_is_named_in_full(docs, sources): + short = sources["oss"]["bundled"][:8] + with pytest.raises(edition.EditionError, match=f"takes the full 40-character commit hash .*, not '{short}'"): + prepare(docs, sources, "enterprise", "8.1.0", "--bundled-lancedb", short) + assert state(docs) == {} + + +def test_a_root_without_recorded_provenance_stops_preparation(docs, sources): + config = yaml.safe_load((docs / "assemble.yaml").read_text()) + config["roots"].append({"name": "notes", "path": "notes", "role": "reference"}) + (docs / "assemble.yaml").write_text(yaml.safe_dump(config, sort_keys=False)) + commit_all(docs, "another root") + on_main(docs) + + with pytest.raises(edition.EditionError, match=r"assemble.yaml has roots \['lancedb', 'enterprise', " + r"'build', 'notes'\], but edition.py records the sources of"): + prepare(docs, sources, "oss", "0.98.0", "--enterprise", "v8.1.0") + assert state(docs) == {} + + +# the docs input commit + + +def test_docs_inputs_come_from_main_not_from_the_branch(docs, sources): + main = git(docs, "rev-parse", "origin/main") + spec = yaml.safe_load((docs / "assemble.yaml").read_text())["openapi"]["release"] + # A pull request's branch changes a docs-owned page and assemble.yaml in a + # commit main does not have, and the page again without committing it. + git(docs, "switch", "-q", "-c", "editions") + (docs / "docs/geneva/index.mdx").write_text("# Geneva, on a branch\n") + config = (docs / "assemble.yaml").read_text() + (docs / "assemble.yaml").write_text(config.replace(f"release: {spec}\n", "release: v9.9.9\n")) + commit_all(docs, "unmerged changes") + (docs / "docs/geneva/index.mdx").write_text("# Geneva, uncommitted\n") + + prepare(docs, sources, "oss", "0.98.0", "--enterprise", "v8.1.0") + git(docs, "add", "editions", "releases") + git(docs, "commit", "-q", "-m", "oss 0.98.0") + prepare(docs, sources, "enterprise", "8.1.0") + + for name in ("oss-0.98.0", "enterprise-8.1.0"): + rec = record(docs, name) + assert rec["sources"]["build"]["commit"] == main + assert rec["sources"]["openapi"]["release"] == spec + assert (docs / rec["edition"] / "geneva/index.mdx").read_text() == "# Geneva\n" + + +def test_an_earlier_commit_of_main_can_be_named(docs, sources): + earlier = git(docs, "rev-parse", "HEAD") + (docs / "docs/geneva/index.mdx").write_text("# Geneva, later\n") + commit_all(docs, "a later page") + later = on_main(docs) + + prepare(docs, sources, "oss", "0.98.0", "--enterprise", "v8.1.0", "--docs", earlier) + prepare(docs, sources, "oss", "0.99.0", "--enterprise", "v8.2.0") + + for name, commit, page in (("oss-0.98.0", earlier, "# Geneva\n"), ("oss-0.99.0", later, "# Geneva, later\n")): + rec = record(docs, name) + assert rec["sources"]["build"]["commit"] == commit + assert (docs / rec["edition"] / "geneva/index.mdx").read_text() == page + + +def test_docs_inputs_off_main_or_unlike_the_running_scripts_are_refused(docs, sources): + git(docs, "switch", "-q", "-c", "editions") + (docs / "docs/geneva/index.mdx").write_text("# Geneva, on a branch\n") + head = commit_all(docs, "an unmerged page") + oss = ["oss", "0.98.0", "--enterprise", "v8.1.0"] + + main = git(docs, "rev-parse", "origin/main") + with pytest.raises(edition.EditionError, + match=rf"--docs: HEAD is {head}, which lancedb/docs main \({main}\) does not contain"): + prepare(docs, sources, *oss, "--docs", "HEAD") + + (docs / "scripts").mkdir() + (docs / "scripts/assemble.py").write_text("# a changed assembler\n") + commit_all(docs, "an unmerged assembler") + with pytest.raises(edition.EditionError, match="this checkout's scripts/ are not those of"): + prepare(docs, sources, *oss) + (docs / "scripts/assemble.py").write_text("# changed again\n") + with pytest.raises(edition.EditionError, match="commit or stash the changes to scripts/"): + prepare(docs, sources, *oss) + + git(docs, "remote", "set-url", "origin", str(docs.parent / "missing/lancedb-docs.git")) + with pytest.raises(edition.EditionError, match="could not fetch lancedb/docs main; check network access and Git authentication"): + prepare(docs, sources, *oss) + git(docs, "remote", "remove", "origin") + with pytest.raises(edition.EditionError, match="this checkout's origin is not set, not lancedb/docs"): + prepare(docs, sources, *oss) + assert state(docs) == {} + + +def test_only_the_lancedb_docs_urls_are_accepted_as_origin(docs, monkeypatch): + monkeypatch.setattr(edition, "DOCS_URLS", LANCEDB_DOCS_URLS) + # Nothing here reaches the network: an accepted URL's fetch stops at once. + monkeypatch.setenv("GIT_ALLOW_PROTOCOL", "file") + for url in ("https://github.com/lancedb/docs", "https://github.com/lancedb/docs.git", + "https://github.com/LanceDB/docs.git", "git@github.com:lancedb/docs.git", + "ssh://git@github.com/lancedb/docs.git"): + git(docs, "remote", "set-url", "origin", url) + with pytest.raises(edition.EditionError, match="could not fetch lancedb/docs main; check network access and Git authentication"): + edition.docs_main(docs) + for url in ("https://github.com/someone/docs.git", "git@github.com:someone/docs.git", + "https://github.com/lancedb/docs-fork.git", "https://github.com/lancedb/docs.git.old", + "http://github.com/lancedb/docs.git", "https://github.com.example.com/lancedb/docs.git", + "https://example.com/lancedb/docs.git", "file:///srv/lancedb/docs.git", + "/srv/git/lancedb-docs.git"): + git(docs, "remote", "set-url", "origin", url) + with pytest.raises(edition.EditionError, match="this checkout's origin is not supported, not"): + edition.docs_main(docs) + + + +@pytest.mark.parametrize("rewritten", [False, True]) +def test_rejected_remote_does_not_print_credentials(docs, monkeypatch, rewritten): + secret = "fixture-only-remote-password" + remote = f"https://user:{secret}@github.com/lancedb/docs.git" + monkeypatch.setattr(edition, "DOCS_URLS", LANCEDB_DOCS_URLS) + if rewritten: + git(docs, "remote", "set-url", "origin", "https://github.com/lancedb/docs.git") + git(docs, "config", f"url.{remote}.insteadOf", "https://github.com/lancedb/docs.git") + else: + git(docs, "remote", "set-url", "origin", remote) + with pytest.raises(edition.EditionError, match="origin is not supported") as caught: + edition.docs_main(docs) + assert secret not in str(caught.value) + assert remote not in str(caught.value) + + +def test_fetch_failure_does_not_print_remote_diagnostics(docs, monkeypatch): + secret = "fixture-only-fetch-password" + run = subprocess.run + + def fail_fetch(argv, **kwargs): + if argv[:4] == ["git", "-C", str(docs), "fetch"]: + return subprocess.CompletedProcess(argv, 128, b"", ( + f"fatal: unable to access 'https://user:{secret}@example.com/docs.git': denied" + ).encode()) + return run(argv, **kwargs) + + monkeypatch.setattr(subprocess, "run", fail_fetch) + with pytest.raises(edition.EditionError, match="check network access and Git authentication") as caught: + edition.docs_main(docs) + assert secret not in str(caught.value) + assert "example.com" not in str(caught.value) + + +def test_a_fork_origin_is_refused_before_anything_is_fetched_or_saved(docs, sources): + """The usual fork layout: origin is a fork, upstream is lancedb/docs, and + the fork's main has a docs-owned change lancedb/docs does not have.""" + fork = docs.parent / "fork.git" + git(docs.parent, "clone", "-q", "--bare", git(docs, "remote", "get-url", "origin"), str(fork)) + git(docs, "remote", "rename", "origin", "upstream") + git(docs, "remote", "add", "origin", str(fork)) + (docs / "docs/geneva/index.mdx").write_text("# Only in the fork\n") + fork_main = commit_all(docs, "a change only the fork has") + git(docs, "push", "-q", "origin", "HEAD:refs/heads/main") + assert not edition.is_ancestor(docs, fork_main, "upstream/main") + refs = git(docs, "for-each-ref") + + refused = r"this checkout's origin is not supported, not lancedb/docs; a fork's main" + with pytest.raises(edition.EditionError, match=refused): + prepare(docs, sources, "oss", "0.98.0", "--enterprise", "v8.1.0") + with pytest.raises(edition.EditionError, match=refused): + prepare(docs, sources, "oss", "0.98.0", "--enterprise", "v8.1.0", "--docs", "upstream/main") + git(docs, "remote", "set-url", "origin", git(docs, "remote", "get-url", "upstream")) + git(docs, "remote", "set-url", "--add", "origin", str(fork)) + with pytest.raises(edition.EditionError, match=refused): + prepare(docs, sources, "oss", "0.98.0", "--enterprise", "v8.1.0") + # The URL is the one fetching would use, after any rewriting. + git(docs, "config", "--unset-all", "remote.origin.url") + git(docs, "remote", "set-url", "origin", "https://github.com/lancedb/docs.git") + git(docs, "config", f"url.{fork}.insteadOf", "https://github.com/lancedb/docs.git") + with pytest.raises(edition.EditionError, match=refused): + prepare(docs, sources, "oss", "0.98.0", "--enterprise", "v8.1.0") + assert state(docs) == {} + assert git(docs, "for-each-ref") == refs + + +def test_a_main_not_fetched_from_lancedb_docs_is_not_trusted(docs, sources): + main = git(docs, "rev-parse", "origin/main") + git(docs, "switch", "-q", "-c", "editions") + (docs / "docs/geneva/index.mdx").write_text("# Geneva, on a branch\n") + branch = commit_all(docs, "an unmerged page") + # As if origin/main had been fetched from elsewhere, or set by hand. + git(docs, "update-ref", "refs/remotes/origin/main", branch) + + with pytest.raises(edition.EditionError, match=rf"which lancedb/docs main \({main}\) does not contain"): + prepare(docs, sources, "oss", "0.98.0", "--enterprise", "v8.1.0", "--docs", branch) + assert state(docs) == {} + git(docs, "update-ref", "refs/remotes/origin/main", branch) + prepare(docs, sources, "oss", "0.98.0", "--enterprise", "v8.1.0") + assert record(docs, "oss-0.98.0")["sources"]["build"]["commit"] == main + assert (docs / "editions/oss/0.98.0/geneva/index.mdx").read_text() == "# Geneva\n" + assert git(docs, "rev-parse", "origin/main") == main + + +def shadowed_main(docs: Path, kind: str) -> tuple[str, str]: + """A local branch or tag named origin/main on an unmerged commit, which the + shorthand origin/main resolves to before the remote-tracking ref.""" + main = git(docs, "rev-parse", "refs/remotes/origin/main") + git(docs, "switch", "-q", "-c", "editions") + (docs / "docs/geneva/index.mdx").write_text("# Geneva, unmerged\n") + unmerged = commit_all(docs, "an unmerged page") + git(docs, kind, "origin/main", unmerged) + assert git(docs, "rev-parse", "origin/main") == unmerged + return main, unmerged + + +@pytest.mark.parametrize("kind", ["branch", "tag"]) +def test_a_local_ref_named_origin_main_is_not_canonical_main(docs, sources, kind): + main, _unmerged = shadowed_main(docs, kind) + + prepare(docs, sources, "oss", "0.98.0", "--enterprise", "v8.1.0") + + rec = record(docs, "oss-0.98.0") + assert rec["sources"]["build"]["commit"] == main + assert (docs / rec["edition"] / "geneva/index.mdx").read_text() == "# Geneva\n" + + +def test_main_is_fetched_where_it_is_read_when_origin_tracks_another_branch(docs, sources): + """Git updates no tracking ref from the configured refspec here, so the + fetch itself must write refs/remotes/origin/main.""" + main = git(docs, "rev-parse", "refs/remotes/origin/main") + git(docs, "config", "remote.origin.fetch", "+refs/heads/other:refs/remotes/origin/other") + git(docs, "switch", "-q", "-c", "editions") + (docs / "docs/geneva/index.mdx").write_text("# Geneva, unmerged\n") + git(docs, "update-ref", "refs/remotes/origin/main", commit_all(docs, "an unmerged page")) + + prepare(docs, sources, "oss", "0.98.0", "--enterprise", "v8.1.0") + + rec = record(docs, "oss-0.98.0") + assert rec["sources"]["build"]["commit"] == main + assert (docs / rec["edition"] / "geneva/index.mdx").read_text() == "# Geneva\n" + + +@pytest.mark.parametrize("kind", ["branch", "tag"]) +def test_naming_an_unmerged_commit_is_refused_despite_a_local_origin_main(docs, sources, kind): + main, unmerged = shadowed_main(docs, kind) + + for ref in (unmerged, "origin/main"): + with pytest.raises(edition.EditionError, + match=rf"--docs: {ref} is {unmerged}, which lancedb/docs main \({main}\) does not contain"): + prepare(docs, sources, "oss", "0.98.0", "--enterprise", "v8.1.0", "--docs", ref) + assert state(docs) == {} + + +# protecting saved editions + + +def test_conflicting_inputs_for_a_saved_release_stop(docs, sources, tmp_path): + prepare(docs, sources, "oss", "0.98.0", "--enterprise", "v8.1.0") + before = state(docs) + + with pytest.raises(edition.EditionError, match="enterprise_pages recorded .*, requested"): + prepare(docs, sources, "oss", "0.98.0", "--enterprise", "v8.2.0") + + retagged = tmp_path / "retagged" + git(tmp_path, "clone", "-q", str(sources["lancedb"]), str(retagged)) + git(retagged, "tag", "-f", "-a", "-m", "moved", "v0.98.0", sources["oss"]["0.99.0"]) + with pytest.raises(edition.EditionError, match="already saved from different inputs .*release recorded"): + prepare(docs, sources, "oss", "0.98.0", "--enterprise", "v8.1.0", lancedb=retagged) + + assert state(docs) == before + + + +@pytest.mark.parametrize("scripts_state", ["uncommitted", "branch", "main"]) +def test_saved_edition_does_not_need_the_current_assembler(docs, sources, monkeypatch, scripts_state): + prepare(docs, sources, "oss", "0.98.0", "--enterprise", "v8.1.0") + correct(docs, "oss-0.98.0", "guide.mdx", "Corrected guide\n") + before = state(docs) + (docs / "scripts").mkdir() + (docs / "scripts/assemble.py").write_text("# a later assembler\n") + if scripts_state != "uncommitted": + commit_all(docs, "change assembler") + if scripts_state == "main": + on_main(docs) + + def must_not_fetch(_repo): + pytest.fail("an existing edition must not fetch docs main") + + monkeypatch.setattr(edition, "docs_main", must_not_fetch) + out = prepare(docs, sources, "oss", "0.98.0", "--enterprise", "v8.1.0") + assert "is corrected (1 correction); nothing changed" in out + assert state(docs) == before + (docs / "editions/oss/0.98.0/guide.mdx").write_text("Unrecorded edit\n") + with pytest.raises(edition.EditionError, match="has changed since its latest recorded correction"): + prepare(docs, sources, "oss", "0.98.0", "--enterprise", "v8.1.0") + with pytest.raises(edition.EditionError, match="enterprise_pages recorded"): + prepare(docs, sources, "oss", "0.98.0", "--enterprise", "v8.2.0") + + +def test_duplicate_preparation_changes_nothing_even_after_a_correction(docs, sources): + prepare(docs, sources, "oss", "0.98.0", "--enterprise", "v8.1.0") + out = prepare(docs, sources, "oss", "0.98.0", "--enterprise", "v8.1.0") + assert "editions/oss/0.98.0 is as prepared; nothing changed" in out + + correct(docs, "oss-0.98.0", "guide.mdx", "Corrected guide\n") + # A commit reference and a tag naming it are the same release input. + (docs / "docs/geneva/index.mdx").write_text("# Geneva, later\n") + git(docs, "commit", "-q", "-am", "later") + on_main(docs) + before = state(docs) + out = prepare(docs, sources, "oss", "0.98.0", "--enterprise", sources["enterprise"]["8.1.0"]) + + assert "is corrected (1 correction); nothing changed" in out + assert state(docs) == before + assert (docs / "editions/oss/0.98.0/guide.mdx").read_text() == "Corrected guide\n" + + +def test_a_newer_edition_leaves_a_corrected_older_one_alone(docs, sources): + prepare(docs, sources, "oss", "0.98.0", "--enterprise", "v8.1.0") + correct(docs, "oss-0.98.0", "guide.mdx", "Corrected guide\n") + before = state(docs) + + prepare(docs, sources, "oss", "0.99.0", "--enterprise", "v8.2.0") + prepare(docs, sources, "enterprise", "8.2.0") + + after = state(docs) + assert {k: v for k, v in after.items() if k in before} == before + assert "Shared guide as of 0.99.0." in (docs / "editions/oss/0.99.0/guide.mdx").read_text() + assert "Shared guide as of 0.99.0." in (docs / "editions/enterprise/8.2.0/guide.mdx").read_text() + + +def test_a_failed_preparation_leaves_saved_editions_intact(docs, sources, monkeypatch): + prepare(docs, sources, "oss", "0.98.0", "--enterprise", "v8.1.0") + before = state(docs) + + with pytest.raises(edition.EditionError, match="has no docs/web"): + prepare(docs, sources, "enterprise", "8.5.0") + assert state(docs) == before + + def fail(*_args): + raise OSError("disk full") + + # The edition cannot be moved into place, after its folders were made. + with monkeypatch.context() as patch: + patch.setattr(edition.os, "rename", fail) + with pytest.raises(OSError, match="disk full"): + prepare(docs, sources, "enterprise", "8.1.0") + assert state(docs) == before + + # The last step fails: the record cannot be put in place after the edition + # has been moved there. + with monkeypatch.context() as patch: + patch.setattr(edition.os, "replace", fail) + with pytest.raises(OSError, match="disk full"): + prepare(docs, sources, "enterprise", "8.1.0") + assert state(docs) == before + + +def test_an_edition_or_a_record_without_its_pair_is_refused(docs, sources): + orphan = docs / "editions/oss/0.99.0" + orphan.mkdir(parents=True) + (orphan / "index.mdx").write_text("half made\n") + with pytest.raises(edition.EditionError, match="exists without releases/oss-0.99.0.yaml"): + prepare(docs, sources, "oss", "0.99.0", "--enterprise", "v8.2.0") + + prepare(docs, sources, "oss", "0.98.0", "--enterprise", "v8.1.0") + shutil.rmtree(docs / "editions/oss/0.98.0") + with pytest.raises(edition.EditionError, match="restore it from Git history"): + prepare(docs, sources, "oss", "0.98.0", "--enterprise", "v8.1.0") + assert not (docs / "editions/oss/0.98.0").exists() + + +# checking saved editions + + +def test_check_pairs_records_and_requires_corrections_to_be_listed(docs, sources): + assert edition.check(docs) == ([], []) + + prepare(docs, sources, "oss", "0.98.0", "--enterprise", "v8.1.0") + # Finder files are not part of what Git saves. + for folder in ("editions", "editions/oss", "editions/oss/0.98.0"): + (docs / folder / ".DS_Store").write_bytes(b"\0") + assert edition.check(docs) == (["oss 0.98.0: as prepared"], []) + + (docs / "editions/oss/0.98.0/guide.mdx").write_text("Edited without a record\n") + lines, problems = edition.check(docs) + assert lines == [] and "has changed since it was prepared" in problems[0] + + correct(docs, "oss-0.98.0", "guide.mdx", "Corrected guide\n") + (docs / "editions/enterprise/8.1.0").mkdir(parents=True) + (docs / "editions/cloud").mkdir() + lines, problems = edition.check(docs) + assert lines == ["oss 0.98.0: corrected (1 correction)"] + assert problems == [ + "editions/cloud is not a product folder", + "editions/enterprise/8.1.0 has no release record", + ] + + +def test_command_prepares_from_main_and_checks_against_the_base(docs, sources): + """End to end, as a person runs it: the scripts are part of the checkout.""" + (docs / "scripts").mkdir() + for name in ("assemble.py", "candidate.py", "edition.py"): + shutil.copy(SCRIPTS / name, docs / "scripts" / name) + git(docs, "add", "-A") + git(docs, "commit", "-q", "-m", "scripts") + main = on_main(docs) + + # Run the checkout's CLI with the local canonical-main stand-in. + runner = ( + "import re, sys; sys.path.insert(0, 'scripts'); import edition; " + "edition.DOCS_URLS = re.compile(sys.argv[1], re.IGNORECASE); raise SystemExit(edition.main(sys.argv[2:]))" + ) + + def run(*argv: str) -> subprocess.CompletedProcess: + return subprocess.run( + [sys.executable, "-B", "-c", runner, TEST_DOCS_URLS.pattern, *argv], + cwd=docs, env=environment(), capture_output=True, text=True, + ) + + repos = ["--lancedb", str(sources["lancedb"]), "--enterprise-repo", str(sources["enterprise_repo"])] + result = run("prepare", "oss", "0.98.0", "--enterprise", "v8.1.0", *repos) + assert result.returncode == 0, result.stderr + assert record(docs, "oss-0.98.0")["sources"]["build"]["commit"] == main + + with (docs / "scripts/assemble.py").open("a") as script: + script.write("# a local change\n") + result = run("prepare", "oss", "0.99.0", "--enterprise", "v8.2.0", *repos) + assert result.returncode == 1 + assert "commit or stash the changes to scripts/" in result.stderr + assert not (docs / "editions/oss/0.99.0").exists() + git(docs, "checkout", "--", "scripts/assemble.py") + + result = run("check") + assert (result.returncode, result.stdout) == (0, "oss 0.98.0: as prepared\n") + base = commit_all(docs, "oss 0.98.0") + assert run("check", "--base", base).returncode == 0 + + (docs / "editions/oss/0.98.0/index.mdx").write_text("A correction\n") + result = run("check") + assert (result.returncode, result.stdout) == (1, "") + reported = re.fullmatch( + r"edition: editions/oss/0\.98\.0 has changed since it was prepared; its files now have " + r"checksum ([0-9a-f]{64})\. For a reviewed correction, add an entry under `corrections` in " + r"releases/oss-0\.98\.0\.yaml with `sha256: \1`\n", + result.stderr, + )[1] + + # The prepared checksum rewritten to match: only the base shows the change. + rewrite(docs, "oss-0.98.0", change("prepared.sha256", reported)) + assert run("check").returncode == 0 + result = run("check", "--base", base) + assert (result.returncode, result.stdout) == (1, "oss 0.98.0: as prepared\n") + assert result.stderr == f"edition: {kept(base, 'releases/oss-0.98.0.yaml', 'prepared.sha256 changed')}\n" + + result = run("check", "--base", "1" * 40) + assert (result.returncode, result.stdout) == (1, "") + assert result.stderr.startswith(f"edition: --base: {'1' * 40} is not a commit in ") + + +# corrections are bound to the edition's checksum + + +def test_a_later_unrecorded_change_fails_after_a_recorded_correction(docs, sources): + """A recorded correction to guide.mdx, then an unrelated page deleted + without updating the record.""" + prepare(docs, sources, "oss", "0.98.0", "--enterprise", "v8.1.0") + folder = docs / "editions/oss/0.98.0" + (folder / "guide.mdx").write_text("Reviewed correction\n") + add_correction(docs, "oss-0.98.0", "guide.mdx", checksum(saved(folder))) + assert edition.check(docs) == (["oss 0.98.0: corrected (1 correction)"], []) + (folder / "enterprise/security.mdx").unlink() + + lines, problems = edition.check(docs) + + assert lines == [] + assert len(problems) == 1 + assert problems[0].startswith( + "editions/oss/0.98.0 has changed since its latest recorded correction; its files now " + "have checksum " + ) + + +def test_each_correction_records_the_snapshot_it_accepts(docs, sources): + prepare(docs, sources, "oss", "0.98.0", "--enterprise", "v8.1.0") + prepared = record(docs, "oss-0.98.0")["prepared"] + first = correct(docs, "oss-0.98.0", "guide.mdx", "Corrected guide\n") + assert edition.check(docs) == (["oss 0.98.0: corrected (1 correction)"], []) + + page = docs / "editions/oss/0.98.0/index.mdx" + page.write_text("Unreviewed edit\n") + assert "has changed since its latest recorded correction" in edition.check(docs)[1][0] + page.unlink() + assert "has changed since its latest recorded correction" in edition.check(docs)[1][0] + + second = record_correction(docs, "oss-0.98.0", "index.mdx") + assert edition.check(docs) == (["oss 0.98.0: corrected (2 corrections)"], []) + rec = record(docs, "oss-0.98.0") + assert rec["prepared"] == prepared + assert [c["sha256"] for c in rec["corrections"]] == [first, second] + assert second == checksum(saved(docs / "editions/oss/0.98.0")) + + before = state(docs) + out = prepare(docs, sources, "oss", "0.98.0", "--enterprise", "v8.1.0") + assert out.endswith("editions/oss/0.98.0 is corrected (2 corrections); nothing changed") + assert state(docs) == before + + (docs / "editions/oss/0.98.0/guide.mdx").write_text("Another unreviewed edit\n") + before = state(docs) + with pytest.raises(edition.EditionError, match="already saved; nothing changed, but .* has changed"): + prepare(docs, sources, "oss", "0.98.0", "--enterprise", "v8.1.0") + assert state(docs) == before + + +# merged editions and records are kept + + +@pytest.fixture +def base(docs, sources) -> str: + """Main after OSS 0.98.0 and Enterprise 8.1.0 were merged, and then a + correction to OSS 0.98.0: the base a pull request is checked against.""" + prepare(docs, sources, "oss", "0.98.0", "--enterprise", "v8.1.0") + prepare(docs, sources, "enterprise", "8.1.0") + commit_all(docs, "editions") + correct(docs, "oss-0.98.0", "guide.mdx", "Corrected guide\n") + return commit_all(docs, "a correction") + + +KEPT = ["enterprise 8.1.0: as prepared", "oss 0.98.0: corrected (1 correction)"] + + +def kept(base: str, rel: str, *changes: str) -> str: + """The problem `check --base` reports for a record changed since `base`.""" + return ( + f"{rel} differs from the base {base}: {', '.join(changes)}; a saved release's " + "record only gains corrections, at the end" + ) + + +def test_unchanged_editions_and_a_new_one_pass_against_the_base(docs, sources, base): + assert edition.check(docs, base=base) == (KEPT, []) + + # A new edition, and a record written again with other formatting and no + # header comment. + prepare(docs, sources, "oss", "0.99.0", "--enterprise", "v8.2.0") + rewrite(docs, "enterprise-8.1.0", lambda rec: None) + assert edition.check(docs, base=base) == ([*KEPT, "oss 0.99.0: as prepared"], []) + + (docs / "editions/oss/0.99.0/index.mdx").write_text("Edited without a record\n") + lines, problems = edition.check(docs, base=base) + assert lines == KEPT + assert len(problems) == 1 + assert problems[0].startswith("editions/oss/0.99.0 has changed since it was prepared") + + +def test_prepared_rewritten_with_the_snapshot_fails_against_the_base(docs, base): + folder = docs / "editions/enterprise/8.1.0" + (folder / "index.mdx").write_text("Edited\n") + (folder / "guide.mdx").unlink() + files = saved(folder) + rewrite(docs, "enterprise-8.1.0", change("prepared", {"files": len(files), "sha256": checksum(files)})) + + # The edition matches its record, so on its own the change passes. + assert edition.check(docs) == (KEPT, []) + assert edition.check(docs, base=base) == (KEPT, [ + kept(base, "releases/enterprise-8.1.0.yaml", "prepared.files changed", "prepared.sha256 changed"), + ]) + + +def test_editing_or_removing_a_recorded_correction_fails_against_the_base(docs, base): + folder = docs / "editions/oss/0.98.0" + # A page deleted, and the existing correction's checksum rewritten to match. + (folder / "enterprise/security.mdx").unlink() + + def rewritten(rec): + rec["corrections"][0]["sha256"] = checksum(saved(folder)) + + rewrite(docs, "oss-0.98.0", rewritten) + assert edition.check(docs) == (KEPT, []) + assert edition.check(docs, base=base) == (KEPT, [ + kept(base, "releases/oss-0.98.0.yaml", "corrections[0].sha256 changed"), + ]) + + # The correction undone and its entry removed: the edition is as prepared. + git(docs, "reset", "-q", "--hard", base) + git(docs, "checkout", "HEAD~1", "--", "editions/oss/0.98.0/guide.mdx") + rewrite(docs, "oss-0.98.0", change("corrections", [])) + lines = [KEPT[0], "oss 0.98.0: as prepared"] + assert edition.check(docs) == (lines, []) + assert edition.check(docs, base=base) == (lines, [ + kept(base, "releases/oss-0.98.0.yaml", "corrections[0] was removed"), + ]) + + +def test_removing_or_replacing_a_saved_release_fails_against_the_base(docs, sources, base): + shutil.rmtree(docs / "editions/enterprise/8.1.0") + (docs / "releases/enterprise-8.1.0.yaml").unlink() + assert edition.check(docs) == ([KEPT[1]], []) + assert edition.check(docs, base=base) == ([KEPT[1]], [ + f"releases/enterprise-8.1.0.yaml is on the base {base} but has been removed; a saved " + "release keeps its record", + f"editions/enterprise/8.1.0 is on the base {base} but has been removed; a saved edition " + "is never removed", + ]) + + git(docs, "reset", "-q", "--hard", base) + shutil.rmtree(docs / "editions/enterprise/8.1.0") + assert edition.check(docs, base=base) == ([KEPT[1]], [ + "releases/enterprise-8.1.0.yaml: editions/enterprise/8.1.0 is missing", + ]) + + # Removed with its record, then prepared again from other inputs. + git(docs, "reset", "-q", "--hard", base) + shutil.rmtree(docs / "editions/oss/0.98.0") + (docs / "releases/oss-0.98.0.yaml").unlink() + prepare(docs, sources, "oss", "0.98.0", "--enterprise", "v8.2.0") + assert edition.check(docs) == ([KEPT[0], "oss 0.98.0: as prepared"], []) + assert edition.check(docs, base=base)[1] == [kept( + base, "releases/oss-0.98.0.yaml", "prepared.sha256 changed", "release.enterprise_pages.commit changed", + "release.enterprise_pages.ref changed", "sources.enterprise.commit changed", "corrections[0] was removed", + )] + + +def test_an_appended_correction_passes_against_the_base(docs, base): + rec = record(docs, "oss-0.98.0") + correct(docs, "oss-0.98.0", "index.mdx", "Corrected index\n") + + assert edition.check(docs, base=base) == ([KEPT[0], "oss 0.98.0: corrected (2 corrections)"], []) + assert record(docs, "oss-0.98.0")["corrections"][0] == rec["corrections"][0] + + +# release records keep their provenance + + +def rewrite(docs: Path, name: str, change) -> None: + path = docs / "releases" / f"{name}.yaml" + rec = yaml.safe_load(path.read_text()) + change(rec) + path.write_text(yaml.safe_dump(rec)) + + +def test_records_without_their_pinned_sources_fail(docs, sources): + """An OSS record without its docs source and with a malformed commit, and + an Enterprise record without its qualification and bundled LanceDB.""" + prepare(docs, sources, "oss", "0.98.0", "--enterprise", "v8.1.0") + prepare(docs, sources, "enterprise", "8.1.0") + + def oss(rec): + del rec["sources"]["build"] + rec["sources"]["lancedb"]["commit"] = "not-a-commit" + + def enterprise(rec): + del rec["release"]["qualification"] + del rec["release"]["bundled_lancedb"] + + rewrite(docs, "oss-0.98.0", oss) + rewrite(docs, "enterprise-8.1.0", enterprise) + + assert edition.check(docs) == ([], [ + "releases/enterprise-8.1.0.yaml: release.qualification is missing; " + "release.bundled_lancedb is missing", + "releases/oss-0.98.0.yaml: sources.lancedb.commit is 'not-a-commit', not a commit " + "hash; sources.build is missing", + ]) + + +@pytest.fixture(scope="module") +def records(tmp_path_factory, sources) -> dict[str, dict]: + """The records the builder writes, which are the contract.""" + docs = docs_checkout(tmp_path_factory.mktemp("records").resolve() / "docs") + with pytest.MonkeyPatch.context() as patch: + patch.setattr(edition, "DOCS_URLS", TEST_DOCS_URLS) + prepare(docs, sources, "oss", "0.98.0", "--enterprise", "v8.1.0") + prepare(docs, sources, "enterprise", "8.1.0") + return {name: record(docs, name) for name in ("oss-0.98.0", "enterprise-8.1.0")} + + +def change(path: str, value=KeyError): + def apply(rec): + *parents, key = path.split(".") + for parent in parents: + rec = rec[parent] + if value is KeyError: + del rec[key] + else: + rec[key] = value + return apply + + +A, B, C = "a" * 64, "b" * 64, "2" * 40 + +# One change each, and the one problem it must cause. {release}, {pages}, +# {bundled} and {prepared} stand for the record's own values. +MALFORMED = [ + ("oss-0.98.0", change("format", 2), "format is 2, not 1"), + ("oss-0.98.0", change("product", "cloud"), "product is 'cloud', not oss or enterprise"), + ("oss-0.98.0", change("version", "0.98"), "version is '0.98', not a version such as 1.2.3"), + ("oss-0.98.0", change("edition", "editions/oss/0.99.0"), + "edition is 'editions/oss/0.99.0', not editions/oss/0.98.0"), + ("oss-0.98.0", change("notes", "x"), "notes is not a field here"), + ("oss-0.98.0", change("release.repository", "lancedb/sophon"), + "release.repository is 'lancedb/sophon', not lancedb/lancedb"), + ("oss-0.98.0", change("release.tag", "v0.99.0"), "release.tag is 'v0.99.0', not v0.98.0"), + ("oss-0.98.0", change("release.commit", "abc123"), "release.commit is 'abc123', not a commit hash"), + ("oss-0.98.0", change("release.commit", C), + "sources.lancedb.commit is {release}, but release.commit is " + C), + ("oss-0.98.0", change("release.enterprise_pages"), "release.enterprise_pages is missing"), + ("oss-0.98.0", change("release.enterprise_pages.ref", ""), + "release.enterprise_pages.ref is '', not the tag or commit named with --enterprise"), + ("oss-0.98.0", change("release.enterprise_pages.commit", C), + "sources.enterprise.commit is {pages}, but release.enterprise_pages.commit is " + C), + ("oss-0.98.0", change("release.bundled_lancedb", C), "release.bundled_lancedb is not a field here"), + ("oss-0.98.0", change("sources.build"), "sources.build is missing"), + ("oss-0.98.0", change("sources.build.commit", "HEAD"), "sources.build.commit is 'HEAD', not a commit hash"), + ("oss-0.98.0", change("sources.enterprise.repository", "lancedb/lancedb"), + "sources.enterprise.repository is 'lancedb/lancedb', not lancedb/sophon"), + ("oss-0.98.0", change("sources.lancedb.path", "docs"), "sources.lancedb.path is 'docs', not docs/web"), + ("oss-0.98.0", change("sources.geneva", {}), "sources.geneva is not a field here"), + ("oss-0.98.0", change("sources.openapi.sha256", "x"), + "sources.openapi.sha256 is 'x', not a SHA-256 checksum"), + ("oss-0.98.0", change("prepared"), "prepared is missing"), + ("oss-0.98.0", change("prepared.files", "9"), "prepared.files is '9', not a number of files"), + ("oss-0.98.0", change("prepared.sha256", "0" * 63), + f"prepared.sha256 is {'0' * 63!r}, not a SHA-256 checksum"), + ("oss-0.98.0", change("corrections", {}), "corrections is {{}}, not a list"), + ("oss-0.98.0", change("corrections", [{"summary": "Fix.", "files": ["guide.mdx"]}]), + "corrections[0].sha256 is missing, not the SHA-256 checksum of the corrected edition that " + "`check` reports"), + ("oss-0.98.0", change("corrections", [{"summary": "Fix.", "sha256": "{prepared}"}]), + "corrections[0].sha256 is the checksum before it; a correction changes the edition"), + ("oss-0.98.0", change("corrections", [{"summary": "Fix.", "sha256": A}, {"summary": "Again.", "sha256": A}]), + "corrections[1].sha256 is the checksum before it; a correction changes the edition"), + ("oss-0.98.0", change("corrections", [{"sha256": A}]), "corrections[0].summary must be one line of text"), + ("oss-0.98.0", change("corrections", [{"summary": "One\nTwo", "sha256": A}]), + "corrections[0].summary must be one line of text"), + ("oss-0.98.0", change("corrections", [{"summary": "Fix.", "files": "guide.mdx", "sha256": A}]), + "corrections[0].files must be a list of the edition's paths"), + ("oss-0.98.0", change("corrections", [{"summary": "Fix.", "sha256": A, "approved": True}]), + "corrections[0].approved is not a field here"), + ("enterprise-8.1.0", change("release.qualification"), "release.qualification is missing"), + ("enterprise-8.1.0", change("release.qualification.tag", "release-qualified/v8.2.0"), + "release.qualification.tag is 'release-qualified/v8.2.0', not release-qualified/v8.1.0"), + ("enterprise-8.1.0", change("release.qualification.object", "annotated"), + "release.qualification.object is 'annotated', not a tag object hash"), + ("enterprise-8.1.0", change("release.bundled_lancedb"), "release.bundled_lancedb is missing"), + ("enterprise-8.1.0", change("release.bundled_lancedb", C), + "sources.lancedb.commit is {bundled}, but release.bundled_lancedb is " + C), + ("enterprise-8.1.0", change("release.commit", C), + "sources.enterprise.commit is {release}, but release.commit is " + C), + ("enterprise-8.1.0", change("release.repository", "lancedb/lancedb"), + "release.repository is 'lancedb/lancedb', not lancedb/sophon"), + ("enterprise-8.1.0", change("release.enterprise_pages", {"ref": "v8.1.0", "commit": C}), + "release.enterprise_pages is not a field here"), +] + + +@pytest.mark.parametrize( + "name, mutate, problem", MALFORMED, + ids=[f"{n}:{p.replace('{{}}', '{}')[:60]}" for n, _m, p in MALFORMED], +) +def test_a_malformed_record_names_the_field(records, tmp_path, name, mutate, problem): + rec = copy.deepcopy(records[name]) + values = { + "release": rec["release"]["commit"], + "pages": (rec["release"].get("enterprise_pages") or {}).get("commit"), + "bundled": rec["release"].get("bundled_lancedb"), + "prepared": rec["prepared"]["sha256"], + } + mutate(rec) + for entry in rec.get("corrections") if isinstance(rec.get("corrections"), list) else []: + if entry.get("sha256") == "{prepared}": + entry["sha256"] = values["prepared"] + path = tmp_path / "releases" / f"{name}.yaml" + path.parent.mkdir() + path.write_text(yaml.safe_dump(rec)) + + with pytest.raises(edition.EditionError) as raised: + edition.load_record(path) + assert str(raised.value) == f"releases/{name}.yaml: " + problem.format(**values) + + +def test_records_the_builder_writes_are_valid(records, tmp_path): + """Including an Enterprise record whose bundled LanceDB no release tags, + and a record with two corrections.""" + corrected = copy.deepcopy(records["oss-0.98.0"]) + corrected["corrections"] = [ + {"summary": "Fix the guide.", "files": ["guide.mdx"], "sha256": A}, + {"summary": "Remove a page.", "sha256": B}, + ] + for name, rec in [*records.items(), ("oss-0.98.0", corrected)]: + assert edition.record_problems(rec, f"{name}.yaml") == [] + assert edition.record_problems(records["enterprise-8.1.0"], "enterprise-8.2.0.yaml") == [ + "product and version name enterprise-8.1.0.yaml, not enterprise-8.2.0.yaml" + ] diff --git a/skills/docs-writer/SKILL.md b/skills/docs-writer/SKILL.md index c8c1909a..32d96050 100644 --- a/skills/docs-writer/SKILL.md +++ b/skills/docs-writer/SKILL.md @@ -146,7 +146,7 @@ Docs must reflect what the code actually does. Before writing or editing any cla Resolution order for a repo name the user mentions: -1. Sibling checkout of `/Users/prrao/code/docs`: e.g., `../lancedb`, `../sophon`, `../geneva`. Check with `ls ../` first. +1. Sibling checkout of `/Users/prrao/code/docs`: e.g., `../lancedb`, `../enterprise`, `../geneva`. Check with `ls ../` first. 2. Anywhere under `/Users/prrao/code/`: try `ls /Users/prrao/code/ | grep -i `. 3. If neither exists locally, ask the user for the path (or a clone URL) before proceeding — do **not** guess or substitute a different repo. @@ -180,6 +180,6 @@ Prefer these over ad-hoc emphasis: - Inventing export names that don't match the `{Py|Ts|Rs}{TitleCase}` formula. - Using a relative import path like `'../snippets/indexing.mdx'` (must be `'/snippets/indexing.mdx'`). - Writing Python-only examples for a feature that exists in all three SDKs. -- Documenting parameters that don't exist in the source repo (e.g., `../lancedb`, `../sophon`, `../geneva`). +- Documenting parameters that don't exist in the source repo (e.g., `../lancedb`, `../enterprise`, `../geneva`). - Ignoring a user's explicit instruction to cross-check against a named repo — that instruction is mandatory, not advisory. - Running `python scripts/mdx_snippets_gen.py` without the `-s tests/` flag (the script needs a source dir). diff --git a/workflows/docs-audit/manifests/indexing.toml b/workflows/docs-audit/manifests/indexing.toml index fde90864..b2c14dec 100644 --- a/workflows/docs-audit/manifests/indexing.toml +++ b/workflows/docs-audit/manifests/indexing.toml @@ -1,5 +1,5 @@ name = "indexing" -description = "Audit the indexing docs against user-facing evidence from lancedb, docs snippets/tests, and sophon." +description = "Audit the indexing docs against user-facing evidence from lancedb, docs snippets/tests, and Enterprise." docs_repo = "docs" rotation_unit = "page" keywords = [ diff --git a/workflows/docs-audit/manifests/namespaces.toml b/workflows/docs-audit/manifests/namespaces.toml index 4ee819c7..a0b55dc2 100644 --- a/workflows/docs-audit/manifests/namespaces.toml +++ b/workflows/docs-audit/manifests/namespaces.toml @@ -1,5 +1,5 @@ name = "namespaces" -description = "Audit namespace and catalog-model docs against user-facing evidence from lancedb SDKs, docs snippets/tests, and sophon enterprise namespace surfaces." +description = "Audit namespace and catalog-model docs against user-facing evidence from lancedb SDKs, docs snippets/tests, and Enterprise namespace surfaces." docs_repo = "docs" rotation_unit = "page" keywords = [ diff --git a/workflows/docs-audit/manifests/search.toml b/workflows/docs-audit/manifests/search.toml index 9469a394..6f160372 100644 --- a/workflows/docs-audit/manifests/search.toml +++ b/workflows/docs-audit/manifests/search.toml @@ -1,5 +1,5 @@ name = "search" -description = "Audit the search docs against user-facing evidence from lancedb, docs snippets/tests, and sophon." +description = "Audit the search docs against user-facing evidence from lancedb, docs snippets/tests, and Enterprise." docs_repo = "docs" rotation_unit = "page" keywords = [ diff --git a/workflows/docs-audit/manifests/storage.toml b/workflows/docs-audit/manifests/storage.toml index 95764110..0524b0b0 100644 --- a/workflows/docs-audit/manifests/storage.toml +++ b/workflows/docs-audit/manifests/storage.toml @@ -1,5 +1,5 @@ name = "storage" -description = "Audit the storage docs against user-facing evidence from lancedb, docs snippets/tests, and sophon." +description = "Audit the storage docs against user-facing evidence from lancedb, docs snippets/tests, and Enterprise." docs_repo = "docs" rotation_unit = "page" keywords = [ diff --git a/workflows/docs-audit/manifests/table-operations.toml b/workflows/docs-audit/manifests/table-operations.toml index 9b9e9ac8..fe511a75 100644 --- a/workflows/docs-audit/manifests/table-operations.toml +++ b/workflows/docs-audit/manifests/table-operations.toml @@ -1,5 +1,5 @@ name = "table-operations" -description = "Audit the table operations docs against user-facing evidence from lancedb, docs snippets/tests, and sophon." +description = "Audit the table operations docs against user-facing evidence from lancedb, docs snippets/tests, and Enterprise." docs_repo = "docs" rotation_unit = "page" keywords = [