Modularity-eval support: pipeline-parallel GPU planning, namespaced dataset ids, launcher runtime hooks

docs/result-compatibility.md

@@ -16,6 +16,23 @@ Add an entry in the PR that makes the change, newest first. Each entry states:
 - **Comparability**: which earlier results remain comparable and what replaces the old
   behavior, if anything.
 
+## 2026-11-12: Namespaced Hugging Face dataset ids for six tasks (#558)
+
+- **Affected**: every spec of `gsm8k` (except `gsm8k:platinum`, which has its own
+  dataset), `csqa`, `piqa`, `socialiqa`, `winogrande`, `humaneval` and
+  `codex_humaneval`, 60 specs in all, including their `olmo3base`, `olmes`, `bpb`,
+  `mc`/`rc`, `full`, `xlarge`, `chat` and pass@k variants.
+- **Change**: the data is now read from `openai/gsm8k`, `tau/commonsense_qa`,
+  `ybisk/piqa`, `allenai/social_i_qa`, `allenai/winogrande` and
+  `openai/openai_humaneval` instead of the legacy un-namespaced ids, which current
+  `huggingface_hub` no longer resolves from an empty cache. These are the repositories the
+  legacy ids pointed to, with the same splits and row counts. Prompts, few-shot examples,
+  sampling and metrics are unchanged.
+- **Identity**: the data source is part of a task's identity, so all 60 task hashes
+  change.
+- **Comparability**: results from before and after measure the same thing and remain
+  comparable; they differ only in task hash.
+
 ## 2026-11-12: Sandbox HTTP commands stop waiting for background processes (#527)
 
 - **Affected**: independent `execute_command` and `execute_code` calls that go through the

src/olmo_eval/cli/beaker/config_loader.py

@@ -359,8 +359,8 @@ def _detect_gpu_requirement(
         """Detect GPU requirement based on provider and harness configuration.
 
         Calculates total GPUs needed for:
-        - Main model (tensor_parallel_size × num_instances)
-        - Auxiliary providers (each with their own tensor_parallel_size × num_instances)
+        - Main model (tensor_parallel_size × pipeline_parallel_size × num_instances)
+        - Auxiliary providers (each with their own TP × PP × num_instances)
 
         Args:
             model_spec: Model specification (name or preset).
@@ -371,6 +371,7 @@ def _detect_gpu_requirement(
             Total number of GPUs required.
         """
         from olmo_eval.common.configs import get_provider_config
+        from olmo_eval.inference.gpu_planner import gpus_per_instance
 
         # Build effective harness config with overrides applied
         harness_config = self._get_effective_harness_config(harness_name, harness_overrides)
@@ -394,7 +395,7 @@ def _detect_gpu_requirement(
         main_tp = 1
         main_provider = harness_config.provider if harness_config else provider_config
         if main_provider:
-            main_tp = main_provider.kwargs.get("tensor_parallel_size", 1)
+            main_tp = gpus_per_instance(main_provider.kwargs)
             main_instances = main_provider.num_instances
         main_gpus = (
             main_instances * main_tp
@@ -410,8 +411,7 @@ def _detect_gpu_requirement(
                     # API-backed or external server - no GPUs needed
                     continue
                 num_instances = config.num_instances
-                tensor_parallel = config.kwargs.get("tensor_parallel_size", 1)
-                aux_gpus += num_instances * tensor_parallel
+                aux_gpus += num_instances * gpus_per_instance(config.kwargs)
 
         return main_gpus + aux_gpus
 

src/olmo_eval/evals/tasks/csqa.py

@@ -23,7 +23,7 @@
 
 @register("csqa")
 class CommonsenseQA(Task):
-    data_source = DataSource(path="commonsense_qa", split="validation")
+    data_source = DataSource(path="tau/commonsense_qa", split="validation")
     split = Split.VALIDATION
     metrics = (LogprobMCAccuracyMetric(),)
     num_fewshot = 0
@@ -157,7 +157,7 @@ def format_request(self, instance: Instance) -> LMRequest:
     "olmo3base",
     num_fewshot=5,
     fewshot_source="olmes_csqa_fixed",
-    data_source=DataSource(path="commonsense_qa", split="validation"),
+    data_source=DataSource(path="tau/commonsense_qa", split="validation"),
     metrics=(LogprobUncondMCAccuracyMetric(),),
 )
 register_variant(
@@ -166,14 +166,14 @@ def format_request(self, instance: Instance) -> LMRequest:
     formatter=MultipleChoiceFormatter(),
     num_fewshot=5,
     fewshot_source="olmes_csqa_fixed",
-    data_source=DataSource(path="commonsense_qa", split="validation+train"),
+    data_source=DataSource(path="tau/commonsense_qa", split="validation+train"),
     limit=10000,
     seed=1234,
 )
 register_variant(
     "csqa",
     "xlarge",
-    data_source=DataSource(path="commonsense_qa", split="validation+train"),
+    data_source=DataSource(path="tau/commonsense_qa", split="validation+train"),
     num_fewshot=5,
     limit=10000,
     seed=1234,

src/olmo_eval/evals/tasks/gsm8k.py

@@ -55,7 +55,7 @@ def _clean_short_answer(text: str) -> str:
 
 @register("gsm8k")
 class GSM8K(Task):
-    data_source = DataSource(path="gsm8k", subset="main")
+    data_source = DataSource(path="openai/gsm8k", subset="main")
     metrics = (AccuracyMetric(scorer=ExactMatchScorer),)
     num_fewshot = 8
     sampling_params = SamplingParams(

src/olmo_eval/evals/tasks/humaneval.py

@@ -28,7 +28,7 @@ class CodeExecutionScorer3s(CodeExecutionScorer):
 class HumanEval(Task):
     """HumanEval code generation task."""
 
-    data_source = DataSource(path="openai_humaneval")
+    data_source = DataSource(path="openai/openai_humaneval")
     sampling_params = SamplingParams(
         max_tokens=1024,
         temperature=0.0,

src/olmo_eval/evals/tasks/piqa.py

@@ -18,7 +18,7 @@
 
 @register("piqa")
 class PiQA(Task):
-    data_source = DataSource(path="piqa", split="validation", revision="refs/convert/parquet")
+    data_source = DataSource(path="ybisk/piqa", split="validation", revision="refs/convert/parquet")
     split = Split.VALIDATION
     metrics = (LogprobPerTokenMCAccuracyMetric(),)
     num_fewshot = 0

src/olmo_eval/evals/tasks/socialiqa.py

@@ -21,7 +21,7 @@
 @register("socialiqa")
 class SocialIQA(Task):
     data_source = DataSource(
-        path="social_i_qa", split="validation", revision="refs/convert/parquet"
+        path="allenai/social_i_qa", split="validation", revision="refs/convert/parquet"
     )
     split = Split.VALIDATION
     metrics = (LogprobPerCharMCAccuracyMetric(),)
@@ -158,7 +158,7 @@ def format_request(self, instance: Instance) -> LMRequest:
     "mc_olmo3base",
     formatter=MultipleChoiceFormatter(),
     data_source=DataSource(
-        path="social_i_qa", split="train+validation", revision="refs/convert/parquet"
+        path="allenai/social_i_qa", split="train+validation", revision="refs/convert/parquet"
     ),
     num_fewshot=5,
     limit=10000,
@@ -168,7 +168,7 @@ def format_request(self, instance: Instance) -> LMRequest:
     "socialiqa",
     "xlarge",
     data_source=DataSource(
-        path="social_i_qa", split="train+validation", revision="refs/convert/parquet"
+        path="allenai/social_i_qa", split="train+validation", revision="refs/convert/parquet"
     ),
     num_fewshot=5,
     limit=10000,

src/olmo_eval/evals/tasks/winogrande.py

@@ -64,7 +64,7 @@ def _format_mc(sentence: str, options: tuple[str, ...], answer: str | None = Non
 
 @register("winogrande")
 class Winogrande(Task):
-    data_source = DataSource(path="winogrande", subset="winogrande_xl", split="validation")
+    data_source = DataSource(path="allenai/winogrande", subset="winogrande_xl", split="validation")
     split = Split.VALIDATION
     metrics = (LogprobMCAccuracyMetric(),)
     num_fewshot = 0

src/olmo_eval/inference/gpu_planner.py

@@ -40,6 +40,28 @@ def resolve_visible_devices(gpu_ids: list[int]) -> str:
     return ",".join(resolved)
 
 
+def gpus_per_instance(kwargs) -> int:
+    """GPUs one vLLM instance spans: ``tensor_parallel_size`` x ``pipeline_parallel_size``."""
+    kwargs = kwargs or {}
+    return int(kwargs.get("tensor_parallel_size", 1)) * int(kwargs.get("pipeline_parallel_size", 1))
+
+
+def tensor_parallel_for(gpu_ids: list[int] | None, kwargs: dict | None) -> int | None:
+    """Tensor-parallel size of a vLLM engine on ``gpu_ids``: its GPUs over the pipeline stages.
+
+    The GPU planner sizes an instance as tensor_parallel_size x pipeline_parallel_size, so with
+    ``pipeline_parallel_size`` set the instance's GPUs are shared out across pipeline stages.
+    """
+    if not gpu_ids:
+        return None
+    pipeline_parallel = int((kwargs or {}).get("pipeline_parallel_size", 1))
+    if len(gpu_ids) % pipeline_parallel:
+        raise ValueError(
+            f"{len(gpu_ids)} GPU(s) cannot be split into {pipeline_parallel} pipeline stages"
+        )
+    return len(gpu_ids) // pipeline_parallel
+
+
 @dataclass(frozen=True)
 class GPUAllocation:
     """GPU allocation for a single component."""
@@ -86,11 +108,12 @@ class GPUPlanner:
     total_gpus: int
     num_main_workers: int
     main_tensor_parallel: int = 1
+    main_pipeline_parallel: int = 1
     auxiliary_configs: dict[str, ProviderConfig] = field(default_factory=dict)
 
     def plan(self) -> GPUPlan:
         """Compute GPU allocation. Main workers first, then auxiliary."""
-        main_gpus_per_worker = self.main_tensor_parallel
+        main_gpus_per_worker = self.main_tensor_parallel * self.main_pipeline_parallel
         main_total_gpus = self.num_main_workers * main_gpus_per_worker
 
         # Calculate auxiliary requirements
@@ -103,7 +126,7 @@ def plan(self) -> GPUPlan:
                 continue
             num_instances = config.num_instances
             tensor_parallel = config.kwargs.get("tensor_parallel_size", 1)
-            gpus_needed = num_instances * tensor_parallel
+            gpus_needed = num_instances * gpus_per_instance(config.kwargs)
             aux_requirements[name] = (num_instances, tensor_parallel)
             aux_total_gpus += gpus_needed
 
@@ -113,7 +136,7 @@ def plan(self) -> GPUPlan:
             raise RuntimeError(
                 f"Not enough GPUs. Need {total_needed} "
                 f"({main_total_gpus} for {self.num_main_workers} main worker(s) × "
-                f"{main_gpus_per_worker} TP, {aux_total_gpus} for auxiliary), "
+                f"{main_gpus_per_worker} GPU(s) (TP x PP), {aux_total_gpus} for auxiliary), "
                 f"but only {self.total_gpus} available."
             )
 
@@ -136,7 +159,7 @@ def plan(self) -> GPUPlan:
         aux_allocations: dict[str, GPUAllocation] = {}
 
         for name, (num_instances, tensor_parallel) in aux_requirements.items():
-            gpus_needed = num_instances * tensor_parallel
+            gpus_needed = num_instances * gpus_per_instance(self.auxiliary_configs[name].kwargs)
             provider_gpus = [gpu_pool.pop(0) for _ in range(gpus_needed)]
             aux_allocations[name] = GPUAllocation(
                 name=name,
@@ -153,20 +176,23 @@ def from_harness_config(cls, harness_config: object, total_gpus: int) -> GPUPlan
 
         # Calculate main provider GPU requirements based on num_instances
         main_tp = 1
+        main_pp = 1
         main_instances = 0
         if provider:
             requires_local = getattr(provider, "requires_local_gpu", True)
             if requires_local:
                 main_instances = getattr(provider, "num_instances", 1)
                 if hasattr(provider, "kwargs"):
                     main_tp = provider.kwargs.get("tensor_parallel_size", 1)
+                    main_pp = provider.kwargs.get("pipeline_parallel_size", 1)
 
         aux_configs = dict(getattr(harness_config, "auxiliary_providers", {}) or {})
 
         return cls(
             total_gpus=total_gpus,
             num_main_workers=main_instances,
             main_tensor_parallel=main_tp,
+            main_pipeline_parallel=main_pp,
             auxiliary_configs=aux_configs,
         )
 

src/olmo_eval/inference/manager.py

@@ -8,6 +8,7 @@
 from dataclasses import dataclass, field, replace
 from typing import TYPE_CHECKING, Any
 
+from olmo_eval.inference.gpu_planner import gpus_per_instance
 from olmo_eval.inference.providers.vllm_server_utils import VLLMServerProcess
 
 if TYPE_CHECKING:
@@ -101,13 +102,15 @@ def start(self) -> dict[str, list[dict[str, Any]]]:
 
                 elif config.kind in LOCAL_SERVER_KINDS:
                     num_instances = config.num_instances
-                    tensor_parallel = config.kwargs.get("tensor_parallel_size", 1)
-                    gpus_needed = num_instances * tensor_parallel
+                    # an instance spans tensor_parallel_size x pipeline_parallel_size GPUs
+                    instance_gpus_needed = gpus_per_instance(config.kwargs)
+                    gpus_needed = num_instances * instance_gpus_needed
 
                     if len(gpu_pool) < gpus_needed:
                         raise RuntimeError(
                             f"Not enough GPUs for {name!r}. "
-                            f"Needs {gpus_needed} ({num_instances} × {tensor_parallel} TP), "
+                            f"Needs {gpus_needed} "
+                            f"({num_instances} × {instance_gpus_needed} TP x PP), "
                             f"available: {len(gpu_pool)}"
                         )
 
@@ -116,7 +119,7 @@ def start(self) -> dict[str, list[dict[str, Any]]]:
                     safe_model = config.model.replace("/", "_").replace("\\", "_")
 
                     for i in range(num_instances):
-                        instance_gpus = [gpu_pool.pop(0) for _ in range(tensor_parallel)]
+                        instance_gpus = [gpu_pool.pop(0) for _ in range(instance_gpus_needed)]
                         # Per-instance log dir: logs/vllm_server_{model}_{idx}/
                         instance_log_dir = None
                         if self.log_dir:

src/olmo_eval/inference/providers/vllm_server_utils.py

@@ -21,7 +21,7 @@
 from typing import TYPE_CHECKING, Any
 
 from olmo_eval.common.debug import is_debug_provider
-from olmo_eval.inference.gpu_planner import resolve_visible_devices
+from olmo_eval.inference.gpu_planner import resolve_visible_devices, tensor_parallel_for
 
 if TYPE_CHECKING:
     pass
@@ -416,8 +416,8 @@ def __init__(
         Args:
             model_name: HuggingFace model ID or local path
             port: Port to serve on (auto-assigned if None)
-            tensor_parallel_size: Number of GPUs for tensor parallelism.
-                If gpu_ids is provided, this is inferred from len(gpu_ids).
+            tensor_parallel_size: Number of GPUs for tensor parallelism. If gpu_ids is
+                provided, this is len(gpu_ids) divided by kwargs' pipeline_parallel_size.
             gpu_ids: Specific GPU IDs to use. Sets CUDA_VISIBLE_DEVICES.
             startup_timeout: Maximum time to wait for server startup
             log_dir: Directory to write server logs to (if set, logs are persisted)
@@ -429,8 +429,8 @@ def __init__(
         self.model_name = model_name
         self.port = port or _find_free_port()
         self.gpu_ids = gpu_ids
-        # Infer tensor_parallel_size from gpu_ids if provided
-        self.tensor_parallel_size = len(gpu_ids) if gpu_ids else tensor_parallel_size
+        # With gpu_ids, the instance's GPUs are shared out across its pipeline stages
+        self.tensor_parallel_size = tensor_parallel_for(gpu_ids, kwargs) or tensor_parallel_size
         self.startup_timeout = startup_timeout
         self.log_dir = log_dir
         self.owner = owner or get_current_worker_id()

src/olmo_eval/launch/beaker/launcher.py

@@ -973,6 +973,10 @@ def _build_install_cmd(
             script = "/gantry-runtime/src/olmo_eval/launch/beaker/scripts/setup_artifact_registry"
             steps.append(f"source {script}")
 
+        # Optional shell snippet run after every package install (e.g. to re-pin a dependency
+        # that a later per-package install changed).  Not exported: it is a command, not a variable.
+        env_exports = dict(env_exports or {})
+        post_install_cmd = env_exports.pop("OLMO_EVAL_RUNTIME_POST_INSTALL_CMD", None)
         # Export additional environment variables (e.g., UV_CACHE_DIR)
         if env_exports:
             for key, value in env_exports.items():
@@ -993,7 +997,9 @@ def provider_package_spec(package: str) -> str:
 
             _, flags = parse_install_spec(package)
             extra_flags: list[str] = []
-            if "--index-url" not in flags and "--extra-index-url" not in flags:
+            if "--index-url" not in flags:
+                extra_flags.extend(["--index-url", "https://pypi.org/simple"])
+            if "--extra-index-url" not in flags:
                 extra_flags.extend(["--extra-index-url", runtime_torch_index_url])
             if "--index-strategy" not in flags:
                 extra_flags.extend(["--index-strategy", "unsafe-best-match"])
@@ -1006,6 +1012,18 @@ def provider_package_spec(package: str) -> str:
         # overridden packages are symlinked into /opt/vllm-venv below.
         if runtime_torch_version:
             steps.append(build_install_command(runtime_torch_spec(), force_reinstall=True))
+            # Companion packages that must change build together with torch (torchvision/torchaudio
+            # carry the CUDA variant too); otherwise the image's builds end up in the constraints
+            # and conflict with the provider's pins.  Space-separated specs, same index as torch.
+            for extra_spec in (
+                (env_exports or {}).get("OLMO_EVAL_RUNTIME_TORCH_EXTRA_PACKAGES") or ""
+            ).split():
+                spec = (
+                    f"{extra_spec} --index-url {runtime_torch_index_url}"
+                    if runtime_torch_index_url
+                    else extra_spec
+                )
+                steps.append(build_install_command(spec, force_reinstall=True))
 
         # Generate constraints from pre-installed CUDA packages to prevent uv from changing them.
         # CUTLASS DSL is the exception: vLLM pins this package independently of
@@ -1090,6 +1108,9 @@ def provider_package_spec(package: str) -> str:
             for pkg in task_packages:
                 steps.append(build_install_command(pkg, constraints))
 
+        if post_install_cmd:
+            steps.append(post_install_cmd)
+
         # Set up Modal secret for GCP Artifact Registry (Modal sandboxes)
         if setup_modal_gcp_secret:
             script = "/gantry-runtime/src/olmo_eval/launch/beaker/scripts/setup_modal_gcp_secret"
@@ -1117,6 +1138,8 @@ def launch(self, config: BeakerJobConfig, dry_run: bool = False) -> BeakerExperi
             "UV_CACHE_DIR",
             "OLMO_EVAL_RUNTIME_TORCH_VERSION",
             "OLMO_EVAL_RUNTIME_TORCH_INDEX_URL",
+            "OLMO_EVAL_RUNTIME_TORCH_EXTRA_PACKAGES",
+            "OLMO_EVAL_RUNTIME_POST_INSTALL_CMD",
         )
         for key in install_env_keys:
             if key in config.env_vars:

src/olmo_eval/runners/asynq/workers.py

@@ -12,7 +12,7 @@
 from typing import TYPE_CHECKING, Any, cast
 
 from olmo_eval.common.logging import get_logger
-from olmo_eval.inference.gpu_planner import resolve_visible_devices
+from olmo_eval.inference.gpu_planner import resolve_visible_devices, tensor_parallel_for
 from olmo_eval.runners.asynq.runtime import resolved_worker_config
 from olmo_eval.runners.asynq.types import (
     WORKER_FATAL,
@@ -112,7 +112,7 @@ def inference_worker(
         engine_overrides: dict[str, Any] = {}
         if provider_kind in ("vllm", "vllm_server"):
             engine_overrides = dict(
-                tensor_parallel_size=len(gpu_ids) if gpu_ids else None,
+                tensor_parallel_size=tensor_parallel_for(gpu_ids, provider_config.kwargs),
                 load_format=load_format,
                 model_loader_extra_config=extra_loader_config,
                 enable_auto_tool_choice=enable_auto_tool_choice or None,