diff --git a/README.md b/README.md index 5286e86ea..d12bd6ba0 100644 --- a/README.md +++ b/README.md @@ -1085,6 +1085,14 @@ Tiny dev is two suites, run as separate jobs: `posttrain:tiny:dev` and `multi_turn_chat` scaffold (`--harness multi_turn_chat`), which drops tool calls and has no sandbox, so they are `posttrain:small:dev:multi_turn`; the rest is `posttrain:small:dev`. +Each dev suite also has a chat RULER+ suite, `posttrain:{tiny,small}:dev:chat_ruler`: +RULER+ at 64K on the 128-item sample, sent as one chat turn, greedy, with 512 output tokens. +Completion RULER+ understates what post-training costs in the format the model is served +in: on the original RULER at 64K, Olmo 3.5 SFT checkpoints scored 4 to 12 points lower in +chat than in completion on the same items. Run it as its own job with thinking off, by +serving a think-off chat template (`-o provider.tokenizer=`). With thinking on, 512 +tokens cut the reasoning off, and even at 8,192 tokens 70% of answers hit the cap. + The suites name no provider, but the engine is part of what a score measures: compare scores only between runs on the same provider, and record it with each run. When the rest of a suite runs on SGLang, BFCL needs its own job with `provider.kind=vllm_server`, the @@ -1095,7 +1103,8 @@ evaluator image (Beaker `01M632ZP67Y4WMV060J74FSC3R`) it runs on SGLang, and so WildGuard judge. In Small test, Arena-Hard v2 also needs vLLM: it fits its output budget to the context, which SGLang refuses. -Every suite has WildGuard-judged safety tasks, so a launch needs the `wg_judge` overrides +Every suite except the multi-turn and chat RULER+ ones has WildGuard-judged safety tasks, +so a launch needs the `wg_judge` overrides (`-o auxiliary_providers.wg_judge.kind=vllm_server -o auxiliary_providers.wg_judge.model=allenai/wildguard -o auxiliary_providers.wg_judge.kwargs.add_bos_token=true`), and the test suites also need the diff --git a/src/olmo_eval/evals/suites/posttrain.py b/src/olmo_eval/evals/suites/posttrain.py index 65bdb0f24..9bf31073a 100644 --- a/src/olmo_eval/evals/suites/posttrain.py +++ b/src/olmo_eval/evals/suites/posttrain.py @@ -50,7 +50,8 @@ budgets, and BFCL keeps its own settings (temperature 0.001, 4,096 tokens), as the ledger's BFCL reads do. Safety tasks use the thinking-model judge variants and need the judge providers shown in ``safety.py`` -(``wg_judge`` in every suite, ``sr_judge`` in test). Each local judge takes its own GPU, so +(``wg_judge`` in every suite but the multi-turn and chat RULER+ ones, ``sr_judge`` in +test). Each local judge takes its own GPU, so dev needs at least 2 GPUs and test 3. Omniscience is graded by its default judge, gpt-4o-mini, through the OpenAI API. @@ -60,6 +61,14 @@ no sandbox, so they are ``posttrain:small:dev:multi_turn``; the rest is ``posttrain:small:dev``. +Each dev suite also has a chat RULER+ suite, ``posttrain:{tiny,small}:dev:chat_ruler``: +RULER+ at 64K on the 128-item sample, sent as one chat turn, greedy, with 512 output tokens. +Completion RULER+ understates what post-training costs in the format the model is served +in: on the original RULER at 64K, Olmo 3.5 SFT checkpoints scored 4 to 12 points lower in +chat than in completion on the same items. Run it as its own job with thinking off, by +serving a think-off chat template (``-o provider.tokenizer=``). With thinking on, 512 +tokens cut the reasoning off, and even at 8,192 tokens 70% of answers hit the cap. + Tiny dev needs no API key. Every other suite needs ``OPENAI_API_KEY`` for its API-judged tasks (MultiChallenge in Small dev, MT-Bench in its multi-turn suite). """ @@ -94,6 +103,7 @@ "ruler_plus_all__131072:hillclimb", "ruler_plus_all__262144:hillclimb", "bfcl", + "ruler_plus_all__65536:hillclimb:chat_mt512", } ) @@ -225,6 +235,18 @@ def _sampled(entries: tuple) -> tuple: ), ) +# Chat RULER+, run as its own job with thinking off (see the module docstring). +for _size in ("tiny", "small"): + make_suite( + f"posttrain:{_size}:dev:chat_ruler", + _sampled((get_suite("ruler_plus_all__65536:hillclimb:chat_mt512"),)), + aggregation=AggregationStrategy.NONE, + description=( + f"Olmo 3.5 {_size.capitalize()} post-training dev set, RULER+ 64K in chat format; " + "serve with thinking off." + ), + ) + make_suite( "posttrain:small:test", _sampled( diff --git a/src/olmo_eval/evals/suites/ruler_plus.py b/src/olmo_eval/evals/suites/ruler_plus.py index a931467af..038dac08c 100644 --- a/src/olmo_eval/evals/suites/ruler_plus.py +++ b/src/olmo_eval/evals/suites/ruler_plus.py @@ -60,3 +60,14 @@ ), ) ) + register( + Suite( + name=f"ruler_plus_all__{size}:hillclimb:chat_mt512", + tasks=tuple(f"{task}:hillclimb:chat_mt512" for task in all_tasks), + aggregation=AggregationStrategy.AVERAGE, + description=( + f"ruler_plus_all__{size}:hillclimb served through the chat template, " + "with 512 output tokens" + ), + ) + ) diff --git a/src/olmo_eval/evals/tasks/ruler_plus.py b/src/olmo_eval/evals/tasks/ruler_plus.py index 766f2a9f6..c946decc4 100644 --- a/src/olmo_eval/evals/tasks/ruler_plus.py +++ b/src/olmo_eval/evals/tasks/ruler_plus.py @@ -17,8 +17,10 @@ ruler_plus_loader.py). """ +from dataclasses import replace from typing import Any +from olmo_eval.common.formatters import ChatFormatter from olmo_eval.common.types import Instance from olmo_eval.data.ruler_plus_loader import ( ROW_FIELD, @@ -95,6 +97,9 @@ def process_doc(self, doc: dict[str, Any], index: int = 0) -> Instance | None: _RULER_PLUS_LIMIT = 512 # The hillclimb variant's smaller seeded sample, for fast dev reads at long contexts. _RULER_PLUS_HILLCLIMB_LIMIT = 128 +# The chat variant's generation budget. Upstream's direct-answer budgets (30 to 128 tokens) +# cut off chat answers, which often restate the question first. +_RULER_PLUS_CHAT_MAX_TOKENS = 512 # Dynamically register all ruler-plus tasks for _task_name, _task_config in RULER_PLUS_TASKS.items(): @@ -109,3 +114,14 @@ def process_doc(self, doc: dict[str, Any], index: int = 0) -> Instance | None: globals()[_cls.__name__] = _cls register(f"ruler_plus_{_task_name}")(_cls) register_variant(f"ruler_plus_{_task_name}", "hillclimb", limit=_RULER_PLUS_HILLCLIMB_LIMIT) + # `:chat_mt512` serves the same input as one user turn through the model's chat template, as + # ruler.py's `:chat` does, with a 512-token budget. Items, greedy decoding, stops and + # scoring are unchanged, so it pairs item-for-item with the completion cell. + register_variant( + f"ruler_plus_{_task_name}", + "chat_mt512", + formatter=ChatFormatter(user_template="{question}"), + sampling_params=replace( + vars(_cls)["sampling_params"], max_tokens=_RULER_PLUS_CHAT_MAX_TOKENS + ), + ) diff --git a/tests/evals/suites/test_posttrain.py b/tests/evals/suites/test_posttrain.py index f564d5b95..0b66f6b75 100644 --- a/tests/evals/suites/test_posttrain.py +++ b/tests/evals/suites/test_posttrain.py @@ -5,6 +5,7 @@ import pytest from olmo_eval.common.configs import expand_tasks, validate_task_metrics, validate_tasks +from olmo_eval.common.formatters import ChatFormatter from olmo_eval.evals.suites import AggregationStrategy, Suite, get_suite from olmo_eval.evals.suites.posttrain import ( BFCL_MULTI_TURN_SAMPLING, @@ -47,12 +48,15 @@ "bbq:mcq_t10_32k", "strongreject:sr_judge_thinking_t10_32k", ) +CHAT_RULER = ("ruler_plus_all__65536:hillclimb:chat_mt512",) MEMBERS = { "posttrain:tiny:dev": DEV_SHARED, + "posttrain:tiny:dev:chat_ruler": CHAT_RULER, "posttrain:tiny:dev:bfcl_multi_turn": ("bfcl:multi_turn:t10_16k",), "posttrain:tiny:test": TEST_SHARED, "posttrain:small:dev": (*DEV_SHARED, "multichallenge:t10_32k"), "posttrain:small:dev:multi_turn": ("mt_bench:t10_32k", "multi_if:t10_32k"), + "posttrain:small:dev:chat_ruler": CHAT_RULER, "posttrain:small:test": ( *TEST_SHARED, "hle:text:verified:t10_32k", @@ -60,7 +64,7 @@ "frontierscience:t10_32k", ), } -# RULER+ runs greedy in completion mode with direct-answer budgets, and BFCL +# RULER+ runs greedy with direct-answer budgets (512 tokens in chat), and BFCL # keeps the settings that existing post-training reads use. OWN_SAMPLING_TASKS = frozenset(task for suite in OWN_SAMPLING for task in get_suite(suite).expand()) # BFCL multi-turn runs at the ledger's setting for it: 16,384 tokens per model turn. @@ -81,8 +85,16 @@ def test_members_and_no_pass_average(name): DEV_SUITES = { - "tiny": ("posttrain:tiny:dev", "posttrain:tiny:dev:bfcl_multi_turn"), - "small": ("posttrain:small:dev", "posttrain:small:dev:multi_turn"), + "tiny": ( + "posttrain:tiny:dev", + "posttrain:tiny:dev:bfcl_multi_turn", + "posttrain:tiny:dev:chat_ruler", + ), + "small": ( + "posttrain:small:dev", + "posttrain:small:dev:multi_turn", + "posttrain:small:dev:chat_ruler", + ), } @@ -218,3 +230,22 @@ def test_dev_hmmt_draws_8_samples_with_the_pass_at_32_setup(task): assert dev.sampling_params.num_samples == 8 assert replace(dev.sampling_params, num_samples=32) == full.sampling_params assert dev.primary_metric == full.primary_metric + + +def test_chat_ruler_pairs_with_the_completion_cell(): + # Only the format and the budget change, so a chat cell pairs item-for-item with the + # completion cell on the same checkpoint. + chat_suite = get_suite("ruler_plus_all__65536:hillclimb:chat_mt512") + completion_suite = get_suite("ruler_plus_all__65536:hillclimb") + + assert chat_suite.expand() == tuple(f"{t}:chat_mt512" for t in completion_suite.expand()) + assert chat_suite.aggregation == completion_suite.aggregation + for task in chat_suite.expand(): + chat = get_task(task).config + completion = get_task(task.removesuffix(":chat_mt512")).config + assert isinstance(chat.formatter, ChatFormatter), task + assert chat.sampling_params == replace(completion.sampling_params, max_tokens=512), task + unchanged = replace( + chat, formatter=completion.formatter, sampling_params=completion.sampling_params + ) + assert unchanged == completion, task diff --git a/tests/evals/tasks/task_hashes.json b/tests/evals/tasks/task_hashes.json index 5cf3e9c81..eeede17d6 100644 --- a/tests/evals/tasks/task_hashes.json +++ b/tests/evals/tasks/task_hashes.json @@ -2957,265 +2957,525 @@ "ruler_niah_s_3__8192:chat": "ca11030fd3d27cce", "ruler_niah_s_3__8192:v2": "d18454f098048178", "ruler_plus_cwe__1048576": "cc4f7d9db57fd264", + "ruler_plus_cwe__1048576:chat_mt512": "2707bca947cc25e6", "ruler_plus_cwe__1048576:hillclimb": "374e4fd696e8761c", + "ruler_plus_cwe__1048576:hillclimb:chat_mt512": "5f232465e8f6abf2", "ruler_plus_cwe__131072": "4c8c39bc945f6a4a", + "ruler_plus_cwe__131072:chat_mt512": "58803ddcdb976939", "ruler_plus_cwe__131072:hillclimb": "56a6180879a243a7", + "ruler_plus_cwe__131072:hillclimb:chat_mt512": "da2c1f8669323554", "ruler_plus_cwe__16384": "9fe124efea508acd", + "ruler_plus_cwe__16384:chat_mt512": "6bf9faf86b139b7b", "ruler_plus_cwe__16384:hillclimb": "a23f63c6e346e03e", + "ruler_plus_cwe__16384:hillclimb:chat_mt512": "38fc835038590489", "ruler_plus_cwe__2097152": "76827bdb273c0338", + "ruler_plus_cwe__2097152:chat_mt512": "f1d86233ffd82e66", "ruler_plus_cwe__2097152:hillclimb": "4220786c03d253ba", + "ruler_plus_cwe__2097152:hillclimb:chat_mt512": "fb68874303ecf6d9", "ruler_plus_cwe__262144": "b62a01c36e584952", + "ruler_plus_cwe__262144:chat_mt512": "6f174cb48236c8ea", "ruler_plus_cwe__262144:hillclimb": "6a65a8ad610fbf8e", + "ruler_plus_cwe__262144:hillclimb:chat_mt512": "5215bc4a22aa91c2", "ruler_plus_cwe__32768": "ff234057eaacf90b", + "ruler_plus_cwe__32768:chat_mt512": "d854c4525fd2fe93", "ruler_plus_cwe__32768:hillclimb": "a91b6683441ec5e0", + "ruler_plus_cwe__32768:hillclimb:chat_mt512": "d9a1ad9d8fffb641", "ruler_plus_cwe__4096": "8c1d9c9c60662b70", + "ruler_plus_cwe__4096:chat_mt512": "73c8c862a1b994fe", "ruler_plus_cwe__4096:hillclimb": "8e85b5e04b010c2d", + "ruler_plus_cwe__4096:hillclimb:chat_mt512": "c52de90b7221df2d", "ruler_plus_cwe__524288": "dfbd8a5522e906e6", + "ruler_plus_cwe__524288:chat_mt512": "2696c3b2f02161fb", "ruler_plus_cwe__524288:hillclimb": "6c7795a5b72108b0", + "ruler_plus_cwe__524288:hillclimb:chat_mt512": "d1e530a4d5a57a86", "ruler_plus_cwe__65536": "30a3d9d892fa2ffa", + "ruler_plus_cwe__65536:chat_mt512": "c4af396f3b8ffe9d", "ruler_plus_cwe__65536:hillclimb": "84e98f22a2600ce5", + "ruler_plus_cwe__65536:hillclimb:chat_mt512": "448efed5a808dcb2", "ruler_plus_cwe__8192": "e4c6d7c99bf946f5", + "ruler_plus_cwe__8192:chat_mt512": "535ff6ff2557863b", "ruler_plus_cwe__8192:hillclimb": "62935c696fc8500f", + "ruler_plus_cwe__8192:hillclimb:chat_mt512": "87d1cb1109ff6942", "ruler_plus_fwe__1048576": "3ae9d85ece45a1d0", + "ruler_plus_fwe__1048576:chat_mt512": "6fd2ce7ed12ab2ed", "ruler_plus_fwe__1048576:hillclimb": "9127fa59a2f86cbd", + "ruler_plus_fwe__1048576:hillclimb:chat_mt512": "ba485432cdc0d176", "ruler_plus_fwe__131072": "b041216f734720dd", + "ruler_plus_fwe__131072:chat_mt512": "79bdad591c7d4bd7", "ruler_plus_fwe__131072:hillclimb": "667c11dbcc1b4da3", + "ruler_plus_fwe__131072:hillclimb:chat_mt512": "8c6320a2911af4d2", "ruler_plus_fwe__16384": "67933af2ad59a52a", + "ruler_plus_fwe__16384:chat_mt512": "be3839d718b44059", "ruler_plus_fwe__16384:hillclimb": "e9135e83d228fb28", + "ruler_plus_fwe__16384:hillclimb:chat_mt512": "e2e9fb7afb48c538", "ruler_plus_fwe__2097152": "d78f8699c920efa7", + "ruler_plus_fwe__2097152:chat_mt512": "cb613f488b52301d", "ruler_plus_fwe__2097152:hillclimb": "7894791e8a87d0e3", + "ruler_plus_fwe__2097152:hillclimb:chat_mt512": "61e933bd5e58e0ff", "ruler_plus_fwe__262144": "4d72aefa8316a796", + "ruler_plus_fwe__262144:chat_mt512": "70421e6690b39844", "ruler_plus_fwe__262144:hillclimb": "38bd4f4dad66d413", + "ruler_plus_fwe__262144:hillclimb:chat_mt512": "1e138cb8114402ac", "ruler_plus_fwe__32768": "3b1c841484e54ef5", + "ruler_plus_fwe__32768:chat_mt512": "12a79ea62da425c5", "ruler_plus_fwe__32768:hillclimb": "d0c4942abd473d24", + "ruler_plus_fwe__32768:hillclimb:chat_mt512": "4df910157b4484a0", "ruler_plus_fwe__4096": "d09103c4ac45e997", + "ruler_plus_fwe__4096:chat_mt512": "08b4194d9a0c9763", "ruler_plus_fwe__4096:hillclimb": "a3cebee691273a3c", + "ruler_plus_fwe__4096:hillclimb:chat_mt512": "eb177fd812192a17", "ruler_plus_fwe__524288": "3e8293c5637d5569", + "ruler_plus_fwe__524288:chat_mt512": "6498646c59e3d1ca", "ruler_plus_fwe__524288:hillclimb": "13120609f16fd341", + "ruler_plus_fwe__524288:hillclimb:chat_mt512": "2a380cebc89d74a1", "ruler_plus_fwe__65536": "11a718bf11516c02", + "ruler_plus_fwe__65536:chat_mt512": "545fd47a5a00080c", "ruler_plus_fwe__65536:hillclimb": "9fddcb4fcef849bf", + "ruler_plus_fwe__65536:hillclimb:chat_mt512": "aa7257926c0f7e68", "ruler_plus_fwe__8192": "4a1af55180905fb6", + "ruler_plus_fwe__8192:chat_mt512": "5de707bd2d4ca407", "ruler_plus_fwe__8192:hillclimb": "84b3733347a0de91", + "ruler_plus_fwe__8192:hillclimb:chat_mt512": "b8bd6a8a92b9bc59", "ruler_plus_niah_mk_1__1048576": "ba1cf5ea67b19e21", + "ruler_plus_niah_mk_1__1048576:chat_mt512": "9c8247d655c11d1e", "ruler_plus_niah_mk_1__1048576:hillclimb": "10b032a556421a06", + "ruler_plus_niah_mk_1__1048576:hillclimb:chat_mt512": "50c371adcba0777b", "ruler_plus_niah_mk_1__131072": "c81496e594bf4157", + "ruler_plus_niah_mk_1__131072:chat_mt512": "bbfff8c974f98b05", "ruler_plus_niah_mk_1__131072:hillclimb": "1219108a0042fa15", + "ruler_plus_niah_mk_1__131072:hillclimb:chat_mt512": "7f2e0b78b0b67f7e", "ruler_plus_niah_mk_1__16384": "0068f18283a0f1de", + "ruler_plus_niah_mk_1__16384:chat_mt512": "08cabec5853de522", "ruler_plus_niah_mk_1__16384:hillclimb": "553a7969b5ba659b", + "ruler_plus_niah_mk_1__16384:hillclimb:chat_mt512": "c0183b60a42728d4", "ruler_plus_niah_mk_1__2097152": "7e2d2f96331fa933", + "ruler_plus_niah_mk_1__2097152:chat_mt512": "a17b57aeba262ea9", "ruler_plus_niah_mk_1__2097152:hillclimb": "3ac2c09630a192ad", + "ruler_plus_niah_mk_1__2097152:hillclimb:chat_mt512": "d0100d19880b04c0", "ruler_plus_niah_mk_1__262144": "c9c8be0ee4151656", + "ruler_plus_niah_mk_1__262144:chat_mt512": "505651be291df489", "ruler_plus_niah_mk_1__262144:hillclimb": "f69c1f6073b67731", + "ruler_plus_niah_mk_1__262144:hillclimb:chat_mt512": "c1bc6b64a1a5275a", "ruler_plus_niah_mk_1__32768": "ff9d3efcc6eb3093", + "ruler_plus_niah_mk_1__32768:chat_mt512": "2f77d73dfe714255", "ruler_plus_niah_mk_1__32768:hillclimb": "d8a6cb27271cc34c", + "ruler_plus_niah_mk_1__32768:hillclimb:chat_mt512": "707e30513f54f2a6", "ruler_plus_niah_mk_1__4096": "ee3b05ec2910d347", + "ruler_plus_niah_mk_1__4096:chat_mt512": "8845d1f02dfe55bd", "ruler_plus_niah_mk_1__4096:hillclimb": "e607919eeaf93555", + "ruler_plus_niah_mk_1__4096:hillclimb:chat_mt512": "80571f05ad07c25f", "ruler_plus_niah_mk_1__524288": "2da68ff061fd982f", + "ruler_plus_niah_mk_1__524288:chat_mt512": "89e5011f321b1180", "ruler_plus_niah_mk_1__524288:hillclimb": "0a514d5225f0e2b8", + "ruler_plus_niah_mk_1__524288:hillclimb:chat_mt512": "4e4becd40e3e0264", "ruler_plus_niah_mk_1__65536": "9e05759340505051", + "ruler_plus_niah_mk_1__65536:chat_mt512": "7404d29ebaf0026c", "ruler_plus_niah_mk_1__65536:hillclimb": "8d031e0628577d15", + "ruler_plus_niah_mk_1__65536:hillclimb:chat_mt512": "6289aef3afc22f20", "ruler_plus_niah_mk_1__8192": "f3a19b47382b3b90", + "ruler_plus_niah_mk_1__8192:chat_mt512": "8707b23fb549c23e", "ruler_plus_niah_mk_1__8192:hillclimb": "c69aa93aea3a474c", + "ruler_plus_niah_mk_1__8192:hillclimb:chat_mt512": "68d39411666bc29d", "ruler_plus_niah_mk_2__1048576": "f0675d2021ea495c", + "ruler_plus_niah_mk_2__1048576:chat_mt512": "c66ac32964960332", "ruler_plus_niah_mk_2__1048576:hillclimb": "cbcaff4ea3af0e32", + "ruler_plus_niah_mk_2__1048576:hillclimb:chat_mt512": "c9f98cfd28804ba4", "ruler_plus_niah_mk_2__131072": "47d70655da27e5bb", + "ruler_plus_niah_mk_2__131072:chat_mt512": "59291a2b650ef7cf", "ruler_plus_niah_mk_2__131072:hillclimb": "fbea5285162d6a4a", + "ruler_plus_niah_mk_2__131072:hillclimb:chat_mt512": "6b10dd85bb127cf6", "ruler_plus_niah_mk_2__16384": "ad976fd43575e72b", + "ruler_plus_niah_mk_2__16384:chat_mt512": "21c95069555ea689", "ruler_plus_niah_mk_2__16384:hillclimb": "2caa9313a8a3698c", + "ruler_plus_niah_mk_2__16384:hillclimb:chat_mt512": "00b1cfa6f7cbe806", "ruler_plus_niah_mk_2__2097152": "688a1a0e3fa4df30", + "ruler_plus_niah_mk_2__2097152:chat_mt512": "c86969a08f8a2d6c", "ruler_plus_niah_mk_2__2097152:hillclimb": "8538fa61e59afe5e", + "ruler_plus_niah_mk_2__2097152:hillclimb:chat_mt512": "bad1c28629dd4ba5", "ruler_plus_niah_mk_2__262144": "dcccc479ca1301ea", + "ruler_plus_niah_mk_2__262144:chat_mt512": "b216ff0428dc374e", "ruler_plus_niah_mk_2__262144:hillclimb": "49f905cd2c870831", + "ruler_plus_niah_mk_2__262144:hillclimb:chat_mt512": "4bc9fa1889362e4d", "ruler_plus_niah_mk_2__32768": "5a1c32633e28c09f", + "ruler_plus_niah_mk_2__32768:chat_mt512": "e8cd7a8032aa1b64", "ruler_plus_niah_mk_2__32768:hillclimb": "9b9c457ebc3331d0", + "ruler_plus_niah_mk_2__32768:hillclimb:chat_mt512": "081ab8e3a8631e8d", "ruler_plus_niah_mk_2__4096": "7014a50e93e6abda", + "ruler_plus_niah_mk_2__4096:chat_mt512": "35054825f7c85ba9", "ruler_plus_niah_mk_2__4096:hillclimb": "04606025ef54cbc6", + "ruler_plus_niah_mk_2__4096:hillclimb:chat_mt512": "40cc98f87df088b1", "ruler_plus_niah_mk_2__524288": "2cfe8f498bbbab85", + "ruler_plus_niah_mk_2__524288:chat_mt512": "3e14af64657ec9a8", "ruler_plus_niah_mk_2__524288:hillclimb": "12fde96763a9f8a3", + "ruler_plus_niah_mk_2__524288:hillclimb:chat_mt512": "1e42d672d75cc780", "ruler_plus_niah_mk_2__65536": "8f715342564ffac2", + "ruler_plus_niah_mk_2__65536:chat_mt512": "8a44a08bba1ad135", "ruler_plus_niah_mk_2__65536:hillclimb": "04f1da329f7d8031", + "ruler_plus_niah_mk_2__65536:hillclimb:chat_mt512": "1197aae274d8cf38", "ruler_plus_niah_mk_2__8192": "6817b564d85c9458", + "ruler_plus_niah_mk_2__8192:chat_mt512": "4a0457db916070ae", "ruler_plus_niah_mk_2__8192:hillclimb": "aaa2c122ece36d83", + "ruler_plus_niah_mk_2__8192:hillclimb:chat_mt512": "2c3cf4bdbe23beab", "ruler_plus_niah_mk_3__1048576": "b0506c1ede634c2f", + "ruler_plus_niah_mk_3__1048576:chat_mt512": "fc1ae7b4a8b2f35b", "ruler_plus_niah_mk_3__1048576:hillclimb": "b20011e1937f0079", + "ruler_plus_niah_mk_3__1048576:hillclimb:chat_mt512": "d4092b4686acd26d", "ruler_plus_niah_mk_3__131072": "dd6b78a9021d2c9d", + "ruler_plus_niah_mk_3__131072:chat_mt512": "288ede4bc2b3135a", "ruler_plus_niah_mk_3__131072:hillclimb": "c71d8b78f786fcbb", + "ruler_plus_niah_mk_3__131072:hillclimb:chat_mt512": "2d6e662297b782cb", "ruler_plus_niah_mk_3__16384": "a20279229696fa1f", + "ruler_plus_niah_mk_3__16384:chat_mt512": "947c712d0724aad6", "ruler_plus_niah_mk_3__16384:hillclimb": "f6dd01b030233e8b", + "ruler_plus_niah_mk_3__16384:hillclimb:chat_mt512": "fca64631e2f2530e", "ruler_plus_niah_mk_3__2097152": "6559908e7358ccf8", + "ruler_plus_niah_mk_3__2097152:chat_mt512": "30401860a890822e", "ruler_plus_niah_mk_3__2097152:hillclimb": "b5b397251b8bf156", + "ruler_plus_niah_mk_3__2097152:hillclimb:chat_mt512": "1ea14a68d9f0c3c0", "ruler_plus_niah_mk_3__262144": "a53d8e652c4791e4", + "ruler_plus_niah_mk_3__262144:chat_mt512": "697bd25629c618a7", "ruler_plus_niah_mk_3__262144:hillclimb": "1edac2297848f107", + "ruler_plus_niah_mk_3__262144:hillclimb:chat_mt512": "e798595e8a0ec9ef", "ruler_plus_niah_mk_3__32768": "82c5e67bb1b885c2", + "ruler_plus_niah_mk_3__32768:chat_mt512": "f4214b77a2b3114b", "ruler_plus_niah_mk_3__32768:hillclimb": "6db377e39fb3d62a", + "ruler_plus_niah_mk_3__32768:hillclimb:chat_mt512": "c154cd13e97ed1d3", "ruler_plus_niah_mk_3__4096": "8950fe521f9fa22a", + "ruler_plus_niah_mk_3__4096:chat_mt512": "e4fbc09da60b6a5d", "ruler_plus_niah_mk_3__4096:hillclimb": "a3da5b1000b130f1", + "ruler_plus_niah_mk_3__4096:hillclimb:chat_mt512": "88002ea85c46a6a5", "ruler_plus_niah_mk_3__524288": "04eeeb175b923131", + "ruler_plus_niah_mk_3__524288:chat_mt512": "f58e50c33e27c721", "ruler_plus_niah_mk_3__524288:hillclimb": "0425b72b7591f0cd", + "ruler_plus_niah_mk_3__524288:hillclimb:chat_mt512": "5cf87427fa3796c8", "ruler_plus_niah_mk_3__65536": "7f7c2c658e33679e", + "ruler_plus_niah_mk_3__65536:chat_mt512": "abcbe318e22a747d", "ruler_plus_niah_mk_3__65536:hillclimb": "4e712ce93479ea92", + "ruler_plus_niah_mk_3__65536:hillclimb:chat_mt512": "e2bb96a59013608f", "ruler_plus_niah_mk_3__8192": "675b60361a7a0f7c", + "ruler_plus_niah_mk_3__8192:chat_mt512": "5b90e0e406608e4e", "ruler_plus_niah_mk_3__8192:hillclimb": "983376177284a3d0", + "ruler_plus_niah_mk_3__8192:hillclimb:chat_mt512": "9b01030fcc56fe5e", "ruler_plus_niah_mq__1048576": "5891f6df3762bf7c", + "ruler_plus_niah_mq__1048576:chat_mt512": "f2048c02cd5b11fe", "ruler_plus_niah_mq__1048576:hillclimb": "3bdde64310c718b1", + "ruler_plus_niah_mq__1048576:hillclimb:chat_mt512": "8b19c33db66c646e", "ruler_plus_niah_mq__131072": "af9ddc913d15a815", + "ruler_plus_niah_mq__131072:chat_mt512": "ccc63bfb21409bef", "ruler_plus_niah_mq__131072:hillclimb": "64323fb472b1397d", + "ruler_plus_niah_mq__131072:hillclimb:chat_mt512": "503c8321da8d9631", "ruler_plus_niah_mq__16384": "40309daabb27274f", + "ruler_plus_niah_mq__16384:chat_mt512": "c725d3ef26ec89cb", "ruler_plus_niah_mq__16384:hillclimb": "30f938de770c2786", + "ruler_plus_niah_mq__16384:hillclimb:chat_mt512": "b5b3398069145787", "ruler_plus_niah_mq__2097152": "09e8d3bd705c95c4", + "ruler_plus_niah_mq__2097152:chat_mt512": "f0f2487a7093f95d", "ruler_plus_niah_mq__2097152:hillclimb": "f62656df121f532a", + "ruler_plus_niah_mq__2097152:hillclimb:chat_mt512": "ea51c45b48d02cab", "ruler_plus_niah_mq__262144": "9d7dc00aca507ea5", + "ruler_plus_niah_mq__262144:chat_mt512": "e902a2b3f35af5ec", "ruler_plus_niah_mq__262144:hillclimb": "9f7480182523ebba", + "ruler_plus_niah_mq__262144:hillclimb:chat_mt512": "22cdf7c303e02bc4", "ruler_plus_niah_mq__32768": "9c9d4f3a358ba157", + "ruler_plus_niah_mq__32768:chat_mt512": "1a42b62f856c9bf3", "ruler_plus_niah_mq__32768:hillclimb": "fa7d99b9a956f713", + "ruler_plus_niah_mq__32768:hillclimb:chat_mt512": "9481669b69f5d222", "ruler_plus_niah_mq__4096": "cbed9ad4ebb93d79", + "ruler_plus_niah_mq__4096:chat_mt512": "d3b27b249e23b08a", "ruler_plus_niah_mq__4096:hillclimb": "3908d49fc07f8997", + "ruler_plus_niah_mq__4096:hillclimb:chat_mt512": "04e8219b8474cb58", "ruler_plus_niah_mq__524288": "358e5a8e76ad3abb", + "ruler_plus_niah_mq__524288:chat_mt512": "e858c89c8803711c", "ruler_plus_niah_mq__524288:hillclimb": "144c6de005bb9a56", + "ruler_plus_niah_mq__524288:hillclimb:chat_mt512": "16c6d638dd75855b", "ruler_plus_niah_mq__65536": "2461f59b6a4a7103", + "ruler_plus_niah_mq__65536:chat_mt512": "e427ee93ce8f0cb8", "ruler_plus_niah_mq__65536:hillclimb": "d56811de2b92e185", + "ruler_plus_niah_mq__65536:hillclimb:chat_mt512": "2836c029c193c9c5", "ruler_plus_niah_mq__8192": "ef8ed46d8a7720c4", + "ruler_plus_niah_mq__8192:chat_mt512": "6ce1137bc943d2f3", "ruler_plus_niah_mq__8192:hillclimb": "1830484f519b7979", + "ruler_plus_niah_mq__8192:hillclimb:chat_mt512": "00188945e60ec396", "ruler_plus_niah_mv__1048576": "a016e3fea9355000", + "ruler_plus_niah_mv__1048576:chat_mt512": "950c291cf9c96c22", "ruler_plus_niah_mv__1048576:hillclimb": "e080bff683d59889", + "ruler_plus_niah_mv__1048576:hillclimb:chat_mt512": "615b0a2b50e7c02d", "ruler_plus_niah_mv__131072": "b6a09908e614318f", + "ruler_plus_niah_mv__131072:chat_mt512": "c92b345bff1c695b", "ruler_plus_niah_mv__131072:hillclimb": "0675f39c3ebe81fc", + "ruler_plus_niah_mv__131072:hillclimb:chat_mt512": "5d0dba561491c6d9", "ruler_plus_niah_mv__16384": "fc36e64df0380264", + "ruler_plus_niah_mv__16384:chat_mt512": "4653531e150c28cf", "ruler_plus_niah_mv__16384:hillclimb": "c72b670418c0063c", + "ruler_plus_niah_mv__16384:hillclimb:chat_mt512": "770225716a51ee1c", "ruler_plus_niah_mv__2097152": "bd6e30f24db903ae", + "ruler_plus_niah_mv__2097152:chat_mt512": "9a3446d9cfde9e01", "ruler_plus_niah_mv__2097152:hillclimb": "df15b1d93ebc11a2", + "ruler_plus_niah_mv__2097152:hillclimb:chat_mt512": "b044dae3b0e4c493", "ruler_plus_niah_mv__262144": "36b2a9d918f68d25", + "ruler_plus_niah_mv__262144:chat_mt512": "b74647fd0f0d43e4", "ruler_plus_niah_mv__262144:hillclimb": "9bc161cc453b7a45", + "ruler_plus_niah_mv__262144:hillclimb:chat_mt512": "01508f284fcecc9e", "ruler_plus_niah_mv__32768": "56c72d584207bfcf", + "ruler_plus_niah_mv__32768:chat_mt512": "26a997eb2849e678", "ruler_plus_niah_mv__32768:hillclimb": "46c3cc6bcee6610a", + "ruler_plus_niah_mv__32768:hillclimb:chat_mt512": "b4e8fb92233d3e94", "ruler_plus_niah_mv__4096": "289c833c0ebeccf8", + "ruler_plus_niah_mv__4096:chat_mt512": "623929509cc042bd", "ruler_plus_niah_mv__4096:hillclimb": "2d1369335e20f206", + "ruler_plus_niah_mv__4096:hillclimb:chat_mt512": "79d20c83da21590f", "ruler_plus_niah_mv__524288": "60f1d353e1c3233e", + "ruler_plus_niah_mv__524288:chat_mt512": "cc78d4f6330f589a", "ruler_plus_niah_mv__524288:hillclimb": "081ed640d5780112", + "ruler_plus_niah_mv__524288:hillclimb:chat_mt512": "364ac6ffa78181db", "ruler_plus_niah_mv__65536": "499503cc6ff78211", + "ruler_plus_niah_mv__65536:chat_mt512": "d9ec3a7866a5048c", "ruler_plus_niah_mv__65536:hillclimb": "6e699fe6b3e8de08", + "ruler_plus_niah_mv__65536:hillclimb:chat_mt512": "992ddc0ae18f3dc1", "ruler_plus_niah_mv__8192": "d4648f6faab410e0", + "ruler_plus_niah_mv__8192:chat_mt512": "f77afa85cd6bea3c", "ruler_plus_niah_mv__8192:hillclimb": "e4779cf5182d7e09", + "ruler_plus_niah_mv__8192:hillclimb:chat_mt512": "04818650a0f48227", "ruler_plus_niah_s_1__1048576": "d11f399667567d55", + "ruler_plus_niah_s_1__1048576:chat_mt512": "1dceaebb6c5ab9ad", "ruler_plus_niah_s_1__1048576:hillclimb": "2b18abafebd38ec0", + "ruler_plus_niah_s_1__1048576:hillclimb:chat_mt512": "766502e1bc359be3", "ruler_plus_niah_s_1__131072": "456332799e165bed", + "ruler_plus_niah_s_1__131072:chat_mt512": "6d0109f429cbe6b8", "ruler_plus_niah_s_1__131072:hillclimb": "2d83e79b0d41d452", + "ruler_plus_niah_s_1__131072:hillclimb:chat_mt512": "ee7ebd3e9e5223ea", "ruler_plus_niah_s_1__16384": "5868ec129282a546", + "ruler_plus_niah_s_1__16384:chat_mt512": "f71f72b116ea6f9c", "ruler_plus_niah_s_1__16384:hillclimb": "7877f62353d2a47b", + "ruler_plus_niah_s_1__16384:hillclimb:chat_mt512": "4a54e79717deb66e", "ruler_plus_niah_s_1__2097152": "2104c1028666bc0c", + "ruler_plus_niah_s_1__2097152:chat_mt512": "d6dd8e2fbc4748ed", "ruler_plus_niah_s_1__2097152:hillclimb": "96b964264a377bc5", + "ruler_plus_niah_s_1__2097152:hillclimb:chat_mt512": "af86f2d539d48258", "ruler_plus_niah_s_1__262144": "c8ddee931729e56c", + "ruler_plus_niah_s_1__262144:chat_mt512": "1c5da18b2d71a1e0", "ruler_plus_niah_s_1__262144:hillclimb": "6571a6e2d7eee33d", + "ruler_plus_niah_s_1__262144:hillclimb:chat_mt512": "e35668d3ef7df322", "ruler_plus_niah_s_1__32768": "74ed956e5cff2a9c", + "ruler_plus_niah_s_1__32768:chat_mt512": "516a1b5d612ecd4d", "ruler_plus_niah_s_1__32768:hillclimb": "629ff3d2fe2cfac9", + "ruler_plus_niah_s_1__32768:hillclimb:chat_mt512": "c46cb37798511a13", "ruler_plus_niah_s_1__4096": "8819833b865876f3", + "ruler_plus_niah_s_1__4096:chat_mt512": "4408537437a63e52", "ruler_plus_niah_s_1__4096:hillclimb": "2e5dd3e85e2d86e5", + "ruler_plus_niah_s_1__4096:hillclimb:chat_mt512": "0cc6ece9e87e3b5a", "ruler_plus_niah_s_1__524288": "4b2dd2ad10c77820", + "ruler_plus_niah_s_1__524288:chat_mt512": "6241fb9e814ec330", "ruler_plus_niah_s_1__524288:hillclimb": "7643a1e434a00e5c", + "ruler_plus_niah_s_1__524288:hillclimb:chat_mt512": "45e0bb9f1fa76836", "ruler_plus_niah_s_1__65536": "b19622c7995fc292", + "ruler_plus_niah_s_1__65536:chat_mt512": "96fae385c71b08a3", "ruler_plus_niah_s_1__65536:hillclimb": "83aefbf354ae82a2", + "ruler_plus_niah_s_1__65536:hillclimb:chat_mt512": "d8ef63cc0f810f4c", "ruler_plus_niah_s_1__8192": "42be49df9d1f2ccc", + "ruler_plus_niah_s_1__8192:chat_mt512": "1525c43f7c353a8f", "ruler_plus_niah_s_1__8192:hillclimb": "058a6587a0bd1c47", + "ruler_plus_niah_s_1__8192:hillclimb:chat_mt512": "7e73443aa8cb62c5", "ruler_plus_niah_s_2__1048576": "7b9ac88a1ba6ddd7", + "ruler_plus_niah_s_2__1048576:chat_mt512": "863838b07482757e", "ruler_plus_niah_s_2__1048576:hillclimb": "a397ec34c1d42c74", + "ruler_plus_niah_s_2__1048576:hillclimb:chat_mt512": "4b6d1f348937d7aa", "ruler_plus_niah_s_2__131072": "55d675c8c699addd", + "ruler_plus_niah_s_2__131072:chat_mt512": "a3e110f0ba943c05", "ruler_plus_niah_s_2__131072:hillclimb": "5bb603b16ce8411f", + "ruler_plus_niah_s_2__131072:hillclimb:chat_mt512": "412e08e441d3a67e", "ruler_plus_niah_s_2__16384": "437a17b41ec3d946", + "ruler_plus_niah_s_2__16384:chat_mt512": "9b96c9880c326d1a", "ruler_plus_niah_s_2__16384:hillclimb": "8161d7dc71de34ac", + "ruler_plus_niah_s_2__16384:hillclimb:chat_mt512": "b683f2df3cb670a8", "ruler_plus_niah_s_2__2097152": "0a41f7efbb629dc9", + "ruler_plus_niah_s_2__2097152:chat_mt512": "105636c6c3711e71", "ruler_plus_niah_s_2__2097152:hillclimb": "6741a34c548ccf0d", + "ruler_plus_niah_s_2__2097152:hillclimb:chat_mt512": "b106e1388549527f", "ruler_plus_niah_s_2__262144": "547dfbced629aa0c", + "ruler_plus_niah_s_2__262144:chat_mt512": "1999121ed6bb7162", "ruler_plus_niah_s_2__262144:hillclimb": "b2a9680fe962a987", + "ruler_plus_niah_s_2__262144:hillclimb:chat_mt512": "2716f872f0c7c45f", "ruler_plus_niah_s_2__32768": "672b67ada8cdf338", + "ruler_plus_niah_s_2__32768:chat_mt512": "7de401aa1a4d39a0", "ruler_plus_niah_s_2__32768:hillclimb": "b6e7890815d9a8de", + "ruler_plus_niah_s_2__32768:hillclimb:chat_mt512": "306b06a502754d34", "ruler_plus_niah_s_2__4096": "6cedd00d372d4c9d", + "ruler_plus_niah_s_2__4096:chat_mt512": "723b6a0571422555", "ruler_plus_niah_s_2__4096:hillclimb": "246a99b52d2eaddc", + "ruler_plus_niah_s_2__4096:hillclimb:chat_mt512": "61c3eb4cbbbca1fa", "ruler_plus_niah_s_2__524288": "b99981de17135e70", + "ruler_plus_niah_s_2__524288:chat_mt512": "2421d0087e21aa4f", "ruler_plus_niah_s_2__524288:hillclimb": "8782dfe9b82bc8e1", + "ruler_plus_niah_s_2__524288:hillclimb:chat_mt512": "36c9ac9e68913a07", "ruler_plus_niah_s_2__65536": "a031010e88922fc9", + "ruler_plus_niah_s_2__65536:chat_mt512": "fe1dbb2c36374428", "ruler_plus_niah_s_2__65536:hillclimb": "1c0c07fdca9cccd8", + "ruler_plus_niah_s_2__65536:hillclimb:chat_mt512": "cab0810c26511ac3", "ruler_plus_niah_s_2__8192": "42e0881c73f492b0", + "ruler_plus_niah_s_2__8192:chat_mt512": "c03c0fd591bb687f", "ruler_plus_niah_s_2__8192:hillclimb": "1348d79ede3cfcd8", + "ruler_plus_niah_s_2__8192:hillclimb:chat_mt512": "71275ff21d326ef3", "ruler_plus_niah_s_3__1048576": "7e73681331973777", + "ruler_plus_niah_s_3__1048576:chat_mt512": "e3a8a0b13106a4e7", "ruler_plus_niah_s_3__1048576:hillclimb": "581d16fa0d36928a", + "ruler_plus_niah_s_3__1048576:hillclimb:chat_mt512": "400aefd6d501ae06", "ruler_plus_niah_s_3__131072": "30879ee577dae1fb", + "ruler_plus_niah_s_3__131072:chat_mt512": "89d49669d57b39cf", "ruler_plus_niah_s_3__131072:hillclimb": "5e477d6fd8a24c27", + "ruler_plus_niah_s_3__131072:hillclimb:chat_mt512": "cbed4c6a5b1e2209", "ruler_plus_niah_s_3__16384": "2fd963d0507f9c33", + "ruler_plus_niah_s_3__16384:chat_mt512": "080a258c1a251632", "ruler_plus_niah_s_3__16384:hillclimb": "637f2ca4f25e87f9", + "ruler_plus_niah_s_3__16384:hillclimb:chat_mt512": "9c7b9aad0a1dd977", "ruler_plus_niah_s_3__2097152": "85ac06b8661e1f42", + "ruler_plus_niah_s_3__2097152:chat_mt512": "99319f4308d4a1f9", "ruler_plus_niah_s_3__2097152:hillclimb": "dd5cecd5e983e132", + "ruler_plus_niah_s_3__2097152:hillclimb:chat_mt512": "5e38f93bb2b49edb", "ruler_plus_niah_s_3__262144": "77c8b7d0e9ab0ff9", + "ruler_plus_niah_s_3__262144:chat_mt512": "8d651e14a5a612f7", "ruler_plus_niah_s_3__262144:hillclimb": "37cee56403e41a1c", + "ruler_plus_niah_s_3__262144:hillclimb:chat_mt512": "cbcdd96b07cb5b58", "ruler_plus_niah_s_3__32768": "a81ca65187100991", + "ruler_plus_niah_s_3__32768:chat_mt512": "4f83424b5cc37744", "ruler_plus_niah_s_3__32768:hillclimb": "fc2104bc1cdc68cd", + "ruler_plus_niah_s_3__32768:hillclimb:chat_mt512": "89fbcde804f02594", "ruler_plus_niah_s_3__4096": "88135081c8b8857a", + "ruler_plus_niah_s_3__4096:chat_mt512": "ed250aefa2ea6e1c", "ruler_plus_niah_s_3__4096:hillclimb": "677e2ccb984c65eb", + "ruler_plus_niah_s_3__4096:hillclimb:chat_mt512": "f7bb61a4c39a8fb4", "ruler_plus_niah_s_3__524288": "6634ecb1b762126f", + "ruler_plus_niah_s_3__524288:chat_mt512": "dbcacb54eabda729", "ruler_plus_niah_s_3__524288:hillclimb": "bedce289e5188e64", + "ruler_plus_niah_s_3__524288:hillclimb:chat_mt512": "f53b9d581a402785", "ruler_plus_niah_s_3__65536": "1019eea8fac63dba", + "ruler_plus_niah_s_3__65536:chat_mt512": "8380b3487713b0c7", "ruler_plus_niah_s_3__65536:hillclimb": "46d563031bed2f50", + "ruler_plus_niah_s_3__65536:hillclimb:chat_mt512": "29cdcc1141996b8a", "ruler_plus_niah_s_3__8192": "ae12461da18ead32", + "ruler_plus_niah_s_3__8192:chat_mt512": "f708f6ce1a17d040", "ruler_plus_niah_s_3__8192:hillclimb": "f501674df10a9fe5", + "ruler_plus_niah_s_3__8192:hillclimb:chat_mt512": "2d6dd3914638b6bd", "ruler_plus_qa_1__1048576": "b7cabe550b4b987c", + "ruler_plus_qa_1__1048576:chat_mt512": "ee79f54ef767ac8f", "ruler_plus_qa_1__1048576:hillclimb": "76497f7a2023139b", + "ruler_plus_qa_1__1048576:hillclimb:chat_mt512": "d5951343ec75d67f", "ruler_plus_qa_1__131072": "edca3c91b6084960", + "ruler_plus_qa_1__131072:chat_mt512": "d71b231b8ff7caf3", "ruler_plus_qa_1__131072:hillclimb": "bfcb6d44b8073a72", + "ruler_plus_qa_1__131072:hillclimb:chat_mt512": "1c62cd13d29b7afd", "ruler_plus_qa_1__16384": "8fb3c7be5a1ad0db", + "ruler_plus_qa_1__16384:chat_mt512": "cfe589e8d368ca00", "ruler_plus_qa_1__16384:hillclimb": "579531ed6df63fec", + "ruler_plus_qa_1__16384:hillclimb:chat_mt512": "7b11265fce79b906", "ruler_plus_qa_1__2097152": "799bab5710935d1f", + "ruler_plus_qa_1__2097152:chat_mt512": "4e21e9bef51c80e3", "ruler_plus_qa_1__2097152:hillclimb": "2a2d18f4c422e7c9", + "ruler_plus_qa_1__2097152:hillclimb:chat_mt512": "5f9a17aec2d5d87e", "ruler_plus_qa_1__262144": "d52926316350999f", + "ruler_plus_qa_1__262144:chat_mt512": "a08290501bd1af9a", "ruler_plus_qa_1__262144:hillclimb": "2bf4efc7599d1365", + "ruler_plus_qa_1__262144:hillclimb:chat_mt512": "64dc8b8f363bb515", "ruler_plus_qa_1__32768": "a92ab92982f5906c", + "ruler_plus_qa_1__32768:chat_mt512": "a305f253a958ad2b", "ruler_plus_qa_1__32768:hillclimb": "5a639084f3067a54", + "ruler_plus_qa_1__32768:hillclimb:chat_mt512": "613a596edecfb41f", "ruler_plus_qa_1__4096": "be4abc89ce5fdadb", + "ruler_plus_qa_1__4096:chat_mt512": "8d6b573ac2dca876", "ruler_plus_qa_1__4096:hillclimb": "b19c57fa07e8ebd7", + "ruler_plus_qa_1__4096:hillclimb:chat_mt512": "c8ee95507b9fb89a", "ruler_plus_qa_1__524288": "34b81003a25acb77", + "ruler_plus_qa_1__524288:chat_mt512": "a5ef066d2b241618", "ruler_plus_qa_1__524288:hillclimb": "e79a34032c18b518", + "ruler_plus_qa_1__524288:hillclimb:chat_mt512": "53547dff5b5a0212", "ruler_plus_qa_1__65536": "2c3dd5e2f5a21890", + "ruler_plus_qa_1__65536:chat_mt512": "bcae4d4c3f953448", "ruler_plus_qa_1__65536:hillclimb": "8d83cd3d5cf420a8", + "ruler_plus_qa_1__65536:hillclimb:chat_mt512": "bad09a05e82e7c11", "ruler_plus_qa_1__8192": "1cbc7d1d887ddcb3", + "ruler_plus_qa_1__8192:chat_mt512": "90cd6a40fe6dc36b", "ruler_plus_qa_1__8192:hillclimb": "fd4d10b5e08ed39b", + "ruler_plus_qa_1__8192:hillclimb:chat_mt512": "0b48d68351a0f06f", "ruler_plus_qa_2__1048576": "eb7c56a73a6898fb", + "ruler_plus_qa_2__1048576:chat_mt512": "e2540311fee1e63f", "ruler_plus_qa_2__1048576:hillclimb": "8d7df095569b4686", + "ruler_plus_qa_2__1048576:hillclimb:chat_mt512": "07acad880aebb1c3", "ruler_plus_qa_2__131072": "c97d89add3e95f72", + "ruler_plus_qa_2__131072:chat_mt512": "fbb1a007fbcf6dcb", "ruler_plus_qa_2__131072:hillclimb": "8b550ea8177a6b06", + "ruler_plus_qa_2__131072:hillclimb:chat_mt512": "81c4b2865eb1e5f0", "ruler_plus_qa_2__16384": "32d6a587feb38c1e", + "ruler_plus_qa_2__16384:chat_mt512": "dfffb7cb55bc3435", "ruler_plus_qa_2__16384:hillclimb": "319a249e9c6a43e8", + "ruler_plus_qa_2__16384:hillclimb:chat_mt512": "dce5b4c5030b7870", "ruler_plus_qa_2__2097152": "19b102e523b8b4bd", + "ruler_plus_qa_2__2097152:chat_mt512": "009383f7d94a50cc", "ruler_plus_qa_2__2097152:hillclimb": "558cb993c7ee2b7b", + "ruler_plus_qa_2__2097152:hillclimb:chat_mt512": "006297b17f746d71", "ruler_plus_qa_2__262144": "2d0d9edffbf2c663", + "ruler_plus_qa_2__262144:chat_mt512": "3f8463401663763b", "ruler_plus_qa_2__262144:hillclimb": "be86c6ef9e820d43", + "ruler_plus_qa_2__262144:hillclimb:chat_mt512": "97fdedfc7f6f5227", "ruler_plus_qa_2__32768": "882f22caa00d643f", + "ruler_plus_qa_2__32768:chat_mt512": "0fcaa14bc92254ef", "ruler_plus_qa_2__32768:hillclimb": "ae61eab6c2a9a7a4", + "ruler_plus_qa_2__32768:hillclimb:chat_mt512": "6c902ad318a6f453", "ruler_plus_qa_2__4096": "c1f3b5aaed83e9e6", + "ruler_plus_qa_2__4096:chat_mt512": "62ad085d251a47cc", "ruler_plus_qa_2__4096:hillclimb": "3a84809f2a2a5d93", + "ruler_plus_qa_2__4096:hillclimb:chat_mt512": "605364e41bb5e2e2", "ruler_plus_qa_2__524288": "7dede54e2f3fa4d1", + "ruler_plus_qa_2__524288:chat_mt512": "be63397b35ffcb13", "ruler_plus_qa_2__524288:hillclimb": "c5c4ef1fed63b108", + "ruler_plus_qa_2__524288:hillclimb:chat_mt512": "1bb2ef544765ec29", "ruler_plus_qa_2__65536": "7ce9d1ca30502eb3", + "ruler_plus_qa_2__65536:chat_mt512": "4c120c78c5e631b5", "ruler_plus_qa_2__65536:hillclimb": "03bc36e96e52b77c", + "ruler_plus_qa_2__65536:hillclimb:chat_mt512": "3bad8ceec4dd5a7e", "ruler_plus_qa_2__8192": "d3133baa45864df7", + "ruler_plus_qa_2__8192:chat_mt512": "3315827e5e1c63e7", "ruler_plus_qa_2__8192:hillclimb": "6119ae9e46957551", + "ruler_plus_qa_2__8192:hillclimb:chat_mt512": "b9e1dfac0d26f24e", "ruler_plus_vt__1048576": "fbb07c79ae493e37", + "ruler_plus_vt__1048576:chat_mt512": "ee45b029ab1e4c12", "ruler_plus_vt__1048576:hillclimb": "5eac49198f53eb22", + "ruler_plus_vt__1048576:hillclimb:chat_mt512": "e0a5fd3f0048caa4", "ruler_plus_vt__131072": "c5e7795e99f20316", + "ruler_plus_vt__131072:chat_mt512": "74f3079f32e42f9f", "ruler_plus_vt__131072:hillclimb": "8c50a1645ab5b342", + "ruler_plus_vt__131072:hillclimb:chat_mt512": "e03afb681fd726c9", "ruler_plus_vt__16384": "cc9172665399dd73", + "ruler_plus_vt__16384:chat_mt512": "e7724de96431a6a5", "ruler_plus_vt__16384:hillclimb": "0852d30e450d6d5e", + "ruler_plus_vt__16384:hillclimb:chat_mt512": "bcc9dc0ae6d8b837", "ruler_plus_vt__2097152": "85c1b2b69812ca4c", + "ruler_plus_vt__2097152:chat_mt512": "a263887fe59e15a8", "ruler_plus_vt__2097152:hillclimb": "5dcec0330973c8db", + "ruler_plus_vt__2097152:hillclimb:chat_mt512": "3f0dab38b659cec3", "ruler_plus_vt__262144": "2cd5a0ee9a910474", + "ruler_plus_vt__262144:chat_mt512": "42f67c7ab2da92b5", "ruler_plus_vt__262144:hillclimb": "747c2b61626d942b", + "ruler_plus_vt__262144:hillclimb:chat_mt512": "74743f8066a0e764", "ruler_plus_vt__32768": "61fa04ef20c914ec", + "ruler_plus_vt__32768:chat_mt512": "dbeef6335320423b", "ruler_plus_vt__32768:hillclimb": "02d04fbd5494eb77", + "ruler_plus_vt__32768:hillclimb:chat_mt512": "da1091136452f2dc", "ruler_plus_vt__4096": "b8fb48aa25680729", + "ruler_plus_vt__4096:chat_mt512": "c303ddcae27140a0", "ruler_plus_vt__4096:hillclimb": "63a23a3684fa7765", + "ruler_plus_vt__4096:hillclimb:chat_mt512": "256769318392f063", "ruler_plus_vt__524288": "7ae24fc9f5ced6ac", + "ruler_plus_vt__524288:chat_mt512": "e38246692fde896e", "ruler_plus_vt__524288:hillclimb": "97f2fa8b0728522a", + "ruler_plus_vt__524288:hillclimb:chat_mt512": "65330d84ba16a606", "ruler_plus_vt__65536": "e5f25b02236d9319", + "ruler_plus_vt__65536:chat_mt512": "9f851fa1246d038c", "ruler_plus_vt__65536:hillclimb": "dcfc9099b833bc71", + "ruler_plus_vt__65536:hillclimb:chat_mt512": "aebf7bed33667e07", "ruler_plus_vt__8192": "99b9597442f12770", + "ruler_plus_vt__8192:chat_mt512": "4447ef4b0fc664bc", "ruler_plus_vt__8192:hillclimb": "b18b59201786f3aa", + "ruler_plus_vt__8192:hillclimb:chat_mt512": "26adc4d41129a9bb", "ruler_qa_1__131072": "4a3a59025cee53e9", "ruler_qa_1__131072:chat": "159526598fb0f2f3", "ruler_qa_1__131072:v2": "2036ab2917688332", diff --git a/tests/evals/tasks/test_ruler_chat.py b/tests/evals/tasks/test_ruler_chat.py index b962a86c4..902ddcdef 100644 --- a/tests/evals/tasks/test_ruler_chat.py +++ b/tests/evals/tasks/test_ruler_chat.py @@ -8,7 +8,7 @@ def test_every_ruler_task_has_a_chat_variant(): variants = list_variants() - # RULER+ tasks have their own variants (hillclimb), not chat ones. + # RULER+ tasks have their own chat variant, `:chat_mt512` (test_ruler_plus.py). ruler = [t for t in variants if t.startswith("ruler_") and not t.startswith("ruler_plus_")] assert ruler assert all("chat" in variants[t] for t in ruler) diff --git a/tests/evals/tasks/test_ruler_plus.py b/tests/evals/tasks/test_ruler_plus.py index e673d1c6b..d98320614 100644 --- a/tests/evals/tasks/test_ruler_plus.py +++ b/tests/evals/tasks/test_ruler_plus.py @@ -8,6 +8,7 @@ import pytest from huggingface_hub.errors import RepositoryNotFoundError +from olmo_eval.common.types import Instance, RequestType from olmo_eval.data import ruler_plus_loader from olmo_eval.data.ruler_plus_loader import ( ROW_FIELD, @@ -170,3 +171,16 @@ def test_every_task_and_suite_is_registered(): } for size in CONTEXT_SIZES: assert len(get_suite(f"ruler_plus_all__{size}").tasks) == 13 + + +def test_chat_variant_sends_the_completion_prompt_as_one_user_turn(): + completion = get_task("ruler_plus_niah_s_1__65536") + chat = get_task("ruler_plus_niah_s_1__65536:chat_mt512") + instance = Instance(question="haystack ... The special magic number is:", gold_answer=["42"]) + plain = completion.format_request(instance) + request = chat.format_request(instance) + + assert plain.request_type == RequestType.COMPLETION + assert request.request_type == RequestType.CHAT + assert request.messages == ({"role": "user", "content": plain.prompt},) + assert chat.config.sampling_params.max_tokens == 512