Add chat RULER+ at 64K to the post-training dev suites

README.md

@@ -1085,6 +1085,14 @@ Tiny dev is two suites, run as separate jobs: `posttrain:tiny:dev` and
 `multi_turn_chat` scaffold (`--harness multi_turn_chat`), which drops tool calls and has no
 sandbox, so they are `posttrain:small:dev:multi_turn`; the rest is `posttrain:small:dev`.
 
+Each dev suite also has a chat RULER+ suite, `posttrain:{tiny,small}:dev:chat_ruler`:
+RULER+ at 64K on the 128-item sample, sent as one chat turn, greedy, with 512 output tokens.
+Completion RULER+ understates what post-training costs in the format the model is served
+in: on the original RULER at 64K, Olmo 3.5 SFT checkpoints scored 4 to 12 points lower in
+chat than in completion on the same items. Run it as its own job with thinking off, by
+serving a think-off chat template (`-o provider.tokenizer=<dir>`). With thinking on, 512
+tokens cut the reasoning off, and even at 8,192 tokens 70% of answers hit the cap.
+
 The suites name no provider, but the engine is part of what a score measures: compare
 scores only between runs on the same provider, and record it with each run. When the rest
 of a suite runs on SGLang, BFCL needs its own job with `provider.kind=vllm_server`, the
@@ -1095,7 +1103,8 @@ evaluator image (Beaker `01M632ZP67Y4WMV060J74FSC3R`) it runs on SGLang, and so
 WildGuard judge. In Small test, Arena-Hard v2 also needs vLLM: it fits its output budget to the context,
 which SGLang refuses.
 
-Every suite has WildGuard-judged safety tasks, so a launch needs the `wg_judge` overrides
+Every suite except the multi-turn and chat RULER+ ones has WildGuard-judged safety tasks,
+so a launch needs the `wg_judge` overrides
 (`-o auxiliary_providers.wg_judge.kind=vllm_server -o
 auxiliary_providers.wg_judge.model=allenai/wildguard -o
 auxiliary_providers.wg_judge.kwargs.add_bos_token=true`), and the test suites also need the

src/olmo_eval/evals/suites/posttrain.py

@@ -50,7 +50,8 @@
 budgets, and BFCL keeps its own settings (temperature 0.001, 4,096 tokens), as the ledger's
 BFCL reads do. Safety tasks use
 the thinking-model judge variants and need the judge providers shown in ``safety.py``
-(``wg_judge`` in every suite, ``sr_judge`` in test). Each local judge takes its own GPU, so
+(``wg_judge`` in every suite but the multi-turn and chat RULER+ ones, ``sr_judge`` in
+test). Each local judge takes its own GPU, so
 dev needs at least 2 GPUs and test 3. Omniscience is graded by its default judge,
 gpt-4o-mini, through the OpenAI API.
 
@@ -60,6 +61,14 @@
 no sandbox, so they are ``posttrain:small:dev:multi_turn``; the rest is
 ``posttrain:small:dev``.
 
+Each dev suite also has a chat RULER+ suite, ``posttrain:{tiny,small}:dev:chat_ruler``:
+RULER+ at 64K on the 128-item sample, sent as one chat turn, greedy, with 512 output tokens.
+Completion RULER+ understates what post-training costs in the format the model is served
+in: on the original RULER at 64K, Olmo 3.5 SFT checkpoints scored 4 to 12 points lower in
+chat than in completion on the same items. Run it as its own job with thinking off, by
+serving a think-off chat template (``-o provider.tokenizer=<dir>``). With thinking on, 512
+tokens cut the reasoning off, and even at 8,192 tokens 70% of answers hit the cap.
+
 Tiny dev needs no API key. Every other suite needs ``OPENAI_API_KEY`` for its API-judged
 tasks (MultiChallenge in Small dev, MT-Bench in its multi-turn suite).
 """
@@ -94,6 +103,7 @@
         "ruler_plus_all__131072:hillclimb",
         "ruler_plus_all__262144:hillclimb",
         "bfcl",
+        "ruler_plus_all__65536:hillclimb:chat_mt512",
     }
 )
 
@@ -225,6 +235,18 @@ def _sampled(entries: tuple) -> tuple:
     ),
 )
 
+# Chat RULER+, run as its own job with thinking off (see the module docstring).
+for _size in ("tiny", "small"):
+    make_suite(
+        f"posttrain:{_size}:dev:chat_ruler",
+        _sampled((get_suite("ruler_plus_all__65536:hillclimb:chat_mt512"),)),
+        aggregation=AggregationStrategy.NONE,
+        description=(
+            f"Olmo 3.5 {_size.capitalize()} post-training dev set, RULER+ 64K in chat format; "
+            "serve with thinking off."
+        ),
+    )
+
 make_suite(
     "posttrain:small:test",
     _sampled(

src/olmo_eval/evals/suites/ruler_plus.py

@@ -60,3 +60,14 @@
                 ),
             )
         )
+        register(
+            Suite(
+                name=f"ruler_plus_all__{size}:hillclimb:chat_mt512",
+                tasks=tuple(f"{task}:hillclimb:chat_mt512" for task in all_tasks),
+                aggregation=AggregationStrategy.AVERAGE,
+                description=(
+                    f"ruler_plus_all__{size}:hillclimb served through the chat template, "
+                    "with 512 output tokens"
+                ),
+            )
+        )

src/olmo_eval/evals/tasks/ruler_plus.py

@@ -17,8 +17,10 @@
 ruler_plus_loader.py).
 """
 
+from dataclasses import replace
 from typing import Any
 
+from olmo_eval.common.formatters import ChatFormatter
 from olmo_eval.common.types import Instance
 from olmo_eval.data.ruler_plus_loader import (
     ROW_FIELD,
@@ -95,6 +97,9 @@ def process_doc(self, doc: dict[str, Any], index: int = 0) -> Instance | None:
 _RULER_PLUS_LIMIT = 512
 # The hillclimb variant's smaller seeded sample, for fast dev reads at long contexts.
 _RULER_PLUS_HILLCLIMB_LIMIT = 128
+# The chat variant's generation budget. Upstream's direct-answer budgets (30 to 128 tokens)
+# cut off chat answers, which often restate the question first.
+_RULER_PLUS_CHAT_MAX_TOKENS = 512
 
 # Dynamically register all ruler-plus tasks
 for _task_name, _task_config in RULER_PLUS_TASKS.items():
@@ -109,3 +114,14 @@ def process_doc(self, doc: dict[str, Any], index: int = 0) -> Instance | None:
     globals()[_cls.__name__] = _cls
     register(f"ruler_plus_{_task_name}")(_cls)
     register_variant(f"ruler_plus_{_task_name}", "hillclimb", limit=_RULER_PLUS_HILLCLIMB_LIMIT)
+    # `:chat_mt512` serves the same input as one user turn through the model's chat template, as
+    # ruler.py's `:chat` does, with a 512-token budget. Items, greedy decoding, stops and
+    # scoring are unchanged, so it pairs item-for-item with the completion cell.
+    register_variant(
+        f"ruler_plus_{_task_name}",
+        "chat_mt512",
+        formatter=ChatFormatter(user_template="{question}"),
+        sampling_params=replace(
+            vars(_cls)["sampling_params"], max_tokens=_RULER_PLUS_CHAT_MAX_TOKENS
+        ),
+    )

tests/evals/suites/test_posttrain.py

@@ -5,6 +5,7 @@
 import pytest
 
 from olmo_eval.common.configs import expand_tasks, validate_task_metrics, validate_tasks
+from olmo_eval.common.formatters import ChatFormatter
 from olmo_eval.evals.suites import AggregationStrategy, Suite, get_suite
 from olmo_eval.evals.suites.posttrain import (
     BFCL_MULTI_TURN_SAMPLING,
@@ -47,20 +48,23 @@
     "bbq:mcq_t10_32k",
     "strongreject:sr_judge_thinking_t10_32k",
 )
+CHAT_RULER = ("ruler_plus_all__65536:hillclimb:chat_mt512",)
 MEMBERS = {
     "posttrain:tiny:dev": DEV_SHARED,
+    "posttrain:tiny:dev:chat_ruler": CHAT_RULER,
     "posttrain:tiny:dev:bfcl_multi_turn": ("bfcl:multi_turn:t10_16k",),
     "posttrain:tiny:test": TEST_SHARED,
     "posttrain:small:dev": (*DEV_SHARED, "multichallenge:t10_32k"),
     "posttrain:small:dev:multi_turn": ("mt_bench:t10_32k", "multi_if:t10_32k"),
+    "posttrain:small:dev:chat_ruler": CHAT_RULER,
     "posttrain:small:test": (
         *TEST_SHARED,
         "hle:text:verified:t10_32k",
         "arena_hard_v2:t10_32k",
         "frontierscience:t10_32k",
     ),
 }
-# RULER+ runs greedy in completion mode with direct-answer budgets, and BFCL
+# RULER+ runs greedy with direct-answer budgets (512 tokens in chat), and BFCL
 # keeps the settings that existing post-training reads use.
 OWN_SAMPLING_TASKS = frozenset(task for suite in OWN_SAMPLING for task in get_suite(suite).expand())
 # BFCL multi-turn runs at the ledger's setting for it: 16,384 tokens per model turn.
@@ -81,8 +85,16 @@ def test_members_and_no_pass_average(name):
 
 
 DEV_SUITES = {
-    "tiny": ("posttrain:tiny:dev", "posttrain:tiny:dev:bfcl_multi_turn"),
-    "small": ("posttrain:small:dev", "posttrain:small:dev:multi_turn"),
+    "tiny": (
+        "posttrain:tiny:dev",
+        "posttrain:tiny:dev:bfcl_multi_turn",
+        "posttrain:tiny:dev:chat_ruler",
+    ),
+    "small": (
+        "posttrain:small:dev",
+        "posttrain:small:dev:multi_turn",
+        "posttrain:small:dev:chat_ruler",
+    ),
 }
 
 
@@ -218,3 +230,22 @@ def test_dev_hmmt_draws_8_samples_with_the_pass_at_32_setup(task):
     assert dev.sampling_params.num_samples == 8
     assert replace(dev.sampling_params, num_samples=32) == full.sampling_params
     assert dev.primary_metric == full.primary_metric
+
+
+def test_chat_ruler_pairs_with_the_completion_cell():
+    # Only the format and the budget change, so a chat cell pairs item-for-item with the
+    # completion cell on the same checkpoint.
+    chat_suite = get_suite("ruler_plus_all__65536:hillclimb:chat_mt512")
+    completion_suite = get_suite("ruler_plus_all__65536:hillclimb")
+
+    assert chat_suite.expand() == tuple(f"{t}:chat_mt512" for t in completion_suite.expand())
+    assert chat_suite.aggregation == completion_suite.aggregation
+    for task in chat_suite.expand():
+        chat = get_task(task).config
+        completion = get_task(task.removesuffix(":chat_mt512")).config
+        assert isinstance(chat.formatter, ChatFormatter), task
+        assert chat.sampling_params == replace(completion.sampling_params, max_tokens=512), task
+        unchanged = replace(
+            chat, formatter=completion.formatter, sampling_params=completion.sampling_params
+        )
+        assert unchanged == completion, task