Repository navigation
Expand file tree
/
Copy pathrunner_common.py
More file actions
276 lines (233 loc) · 10.3 KB
/
Copy pathrunner_common.py
File metadata and controls
276 lines (233 loc) · 10.3 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
#!/usr/bin/env python3
"""
runner_common.py - Shared infrastructure for the experiment runners
(exp_exposure.py, exp_msc.py, exp_message_ablation.py, exp_mitigation_verifier.py,
scripts/validate_mint.py, scripts/generate_mint_passages.py).
Provides
* ChatClient: OpenAI-compatible chat wrapper with retries, per-call seeds,
token accounting, finish_reason capture and a --mock mode.
* load_experiment_config(): reads configs/experiment_config.json.
* JSONL writer for one-row-per-question x agent x turn outputs.
* Endpoint resolution reusing exp2's discovery/autostart helpers.
"""
from __future__ import annotations
import json
import os
import random
import threading
import time
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any, Dict, List, Optional, Tuple
REPO_ROOT = Path(__file__).resolve().parent
CONFIG_PATH = REPO_ROOT / "configs" / "experiment_config.json"
def load_experiment_config() -> dict:
with open(CONFIG_PATH, "r", encoding="utf-8") as f:
return json.load(f)
@dataclass
class TokenLedger:
"""Thread-safe prompt/completion token accounting per (stage) label."""
counts: Dict[str, Dict[str, int]] = field(default_factory=dict)
_lock: threading.Lock = field(default_factory=threading.Lock, repr=False)
def add(self, label: str, prompt_tokens: int, completion_tokens: int, n_calls: int = 1) -> None:
with self._lock:
c = self.counts.setdefault(label, {"prompt_tokens": 0, "completion_tokens": 0, "calls": 0})
c["prompt_tokens"] += int(prompt_tokens or 0)
c["completion_tokens"] += int(completion_tokens or 0)
c["calls"] += n_calls
def as_dict(self) -> dict:
with self._lock:
return json.loads(json.dumps(self.counts))
THINK_END = "</think>"
def split_think_channels(content: Optional[str], reasoning: Optional[str]) -> Tuple[str, Optional[str]]:
"""Normalise (content, reasoning) from an OpenAI-compatible server with a reasoning parser.
vLLM's glm45 parser (GLM-4.7-Flash) has two failure modes:
* the closing tag leaks into `content` ("...nal Solution: 4</think>The answer ...");
* when the model never emits `</think>` the whole output lands in `reasoning` and
`content` is empty, although the answer is at the end of that text.
The answer is everything after the LAST `</think>`; anything before it is thinking.
"""
content = content or ""
reasoning = reasoning or None
if THINK_END in content:
pre, _, post = content.rpartition(THINK_END)
reasoning = ((reasoning or "") + pre).strip() or None
content = post
elif not content.strip() and reasoning and THINK_END in reasoning:
pre, _, post = reasoning.rpartition(THINK_END)
reasoning, content = pre.strip() or None, post
if content.lstrip().startswith("<think>"):
content = content.lstrip()[len("<think>"):]
if reasoning and reasoning.lstrip().startswith("<think>"):
reasoning = reasoning.lstrip()[len("<think>"):].strip() or None
return content, reasoning
@dataclass
class ChatResult:
text: str
finish_reason: Optional[str]
prompt_tokens: int
completion_tokens: int
reasoning_text: Optional[str] = None
error: Optional[str] = None
class ChatClient:
"""Minimal OpenAI-compatible chat client with retries, seeds and token accounting."""
def __init__(
self,
endpoint_url: Optional[str],
model_name: str,
*,
mock: bool = False,
timeout: float = 600.0,
max_retries: int = 5,
ledger: Optional[TokenLedger] = None,
mock_answer_fn=None,
) -> None:
self.model_name = model_name
self.mock = mock
self.timeout = timeout
self.max_retries = max_retries
self.ledger = ledger or TokenLedger()
self._seed_unsupported = threading.Event()
self._mock_answer_fn = mock_answer_fn
self._mock_rng = random.Random(0)
self.client = None
if not mock:
from openai import OpenAI # lazy import
self.client = OpenAI(
base_url=(endpoint_url or "http://127.0.0.1:8080/v1").rstrip("/"),
api_key=os.environ.get("OPENAI_API_KEY") or "EMPTY",
timeout=timeout,
max_retries=0,
)
def chat(
self,
messages: List[dict],
*,
max_tokens: int,
temperature: float,
top_p: float = 1.0,
seed: Optional[int] = None,
label: str = "default",
stop: Optional[List[str]] = None,
) -> ChatResult:
if self.mock:
text = self._mock_answer_fn(messages) if self._mock_answer_fn else "Final Solution: A"
pt = sum(len(str(m.get("content", "")).split()) for m in messages)
ct = len(text.split())
self.ledger.add(label, pt, ct)
return ChatResult(text=text, finish_reason="stop", prompt_tokens=pt, completion_tokens=ct)
delay = 1.0
last_err: Optional[BaseException] = None
for _attempt in range(self.max_retries):
kwargs: Dict[str, Any] = {
"model": self.model_name,
"messages": messages,
"max_tokens": max_tokens,
"temperature": temperature,
"top_p": top_p,
"timeout": self.timeout,
}
if stop:
kwargs["stop"] = stop
if seed is not None and not self._seed_unsupported.is_set():
kwargs["seed"] = seed
try:
resp = self.client.chat.completions.create(**kwargs)
choice = resp.choices[0]
msg = choice.message
content = getattr(msg, "content", None) or ""
reasoning = None
for attr in ("reasoning_content", "reasoning"):
alt = getattr(msg, attr, None)
if isinstance(alt, str) and alt.strip():
reasoning = alt
break
content, reasoning = split_think_channels(content, reasoning)
usage = getattr(resp, "usage", None)
pt = int(getattr(usage, "prompt_tokens", 0) or 0)
ct = int(getattr(usage, "completion_tokens", 0) or 0)
self.ledger.add(label, pt, ct)
return ChatResult(text=content.strip(), finish_reason=getattr(choice, "finish_reason", None),
prompt_tokens=pt, completion_tokens=ct, reasoning_text=reasoning)
except Exception as e: # noqa: BLE001
if "seed" in kwargs and "seed" in str(e).lower():
self._seed_unsupported.set()
print("[WARN] endpoint rejected 'seed'; continuing without per-call seeds")
continue
last_err = e
time.sleep(delay)
delay = min(delay * 2, 30.0)
return ChatResult(text="", finish_reason="error", prompt_tokens=0, completion_tokens=0, error=str(last_err))
def resolve_endpoint(endpoint_url: str, model_name: str, no_autostart: bool) -> str:
"""Reuse exp2's endpoint discovery / autostart logic."""
import exp2 as exp2_base # local import: heavy module
resolved = exp2_base._env_or_default_endpoint(endpoint_url)
if resolved.startswith("http") and not exp2_base._endpoint_alive(resolved, timeout=2.0):
auto = exp2_base._discover_endpoint_from_mallm(model_name)
if auto:
print(f"[INFO] endpoint {resolved!r} unreachable; using discovered endpoint {auto!r}")
resolved = auto
elif not no_autostart:
started = exp2_base._start_model_server(model_name, desired_port=8080)
if started:
resolved = started["endpoint_url"]
if resolved.startswith("http") and not exp2_base._endpoint_alive(resolved, timeout=2.5):
raise SystemExit(f"[FATAL] no reachable model endpoint at {resolved!r}")
return resolved
class JsonlWriter:
"""Append-only JSONL writer (one record per question x agent x turn)."""
def __init__(self, path: Path) -> None:
self.path = Path(path)
self.path.parent.mkdir(parents=True, exist_ok=True)
self._lock = threading.Lock()
def write(self, record: dict) -> None:
line = json.dumps(record, ensure_ascii=False)
with self._lock:
with open(self.path, "a", encoding="utf-8") as f:
f.write(line + "\n")
def write_many(self, records: List[dict]) -> None:
with self._lock:
with open(self.path, "a", encoding="utf-8") as f:
for r in records:
f.write(json.dumps(r, ensure_ascii=False) + "\n")
def read_jsonl(path: Path) -> List[dict]:
out: List[dict] = []
if not Path(path).is_file():
return out
with open(path, "r", encoding="utf-8") as f:
for line in f:
line = line.strip()
if line:
out.append(json.loads(line))
return out
def stable_seed(*parts: Any) -> int:
"""Deterministic 31-bit seed from arbitrary parts (used for per-question assignment)."""
import hashlib
h = hashlib.md5("::".join(str(p) for p in parts).encode("utf-8")).hexdigest()
return int(h[:8], 16) & 0x7FFFFFFF
def write_run_metadata(path: Path, args: Any, extra: Optional[dict] = None) -> None:
"""Dump the effective configuration next to the results (reproducibility)."""
import platform
import sys
from datetime import datetime, timezone
meta = {
"timestamp_utc": datetime.now(timezone.utc).isoformat(),
"argv": sys.argv,
"args": {k: (v if isinstance(v, (int, float, str, bool, list, dict, type(None))) else str(v))
for k, v in vars(args).items()},
"python": sys.version,
"platform": platform.platform(),
"git_commit": _git_commit(),
}
if extra:
meta.update(extra)
Path(path).parent.mkdir(parents=True, exist_ok=True)
with open(path, "w", encoding="utf-8") as f:
json.dump(meta, f, indent=2, ensure_ascii=False)
def _git_commit() -> Optional[str]:
try:
import subprocess
return subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=str(REPO_ROOT), text=True,
stderr=subprocess.DEVNULL).strip()
except Exception:
return None