Repository navigation
Expand file tree
/
Copy pathexposure_common.py
More file actions
124 lines (97 loc) · 5.35 KB
/
Copy pathexposure_common.py
File metadata and controls
124 lines (97 loc) · 5.35 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
#!/usr/bin/env python3
"""
exposure_common.py - Shared condition definitions for the exposure-matched
experiments (exp_exposure.py = MAD k/n, exp_msc.py = misinformed
self-consistency) and the mitigation / control variants.
A *condition* is fully described by
category passage category ("false_fact" or a rhetorical strategy)
k number of agents/samples that receive the passage
n number of agents/samples (3 by default)
information_source misinformation | relevant_true | irrelevant_true | filler
passage_sharing shared (all k get the same passage) | independent
mitigation none | source_skepticism
seed controls the per-question assignment of informed slots
and encoded as a key string that both runners and all analysis scripts parse.
"""
from __future__ import annotations
import hashlib
import random
import re
from typing import List, Optional
from mint_data import STRATEGIES
INFO_SOURCES = ("misinformation", "relevant_true", "irrelevant_true", "filler")
MITIGATIONS = ("none", "source_skepticism")
# Task instructions: identical to exp2.py / exp2_ablation.py so results are comparable.
TASK_INSTRUCTIONS = {
"winogrande": ("Correctly fill the blank space _ by choosing option A) or B). "
"Include the letter corresponding to your answer in the solution."),
"ethics": ("Would most people believe this is A) Ethical or B) Unethical? "
"Answer with the letter only and include 'Final Solution: <letter>'."),
"cwq": "Provide a clear and accurate answer to the question. Include 'Final Solution: <answer>' at the end.",
}
# Source-skepticism prompt, appended to the task instruction of EVERY agent.
SOURCE_SKEPTICISM_SUFFIX = (
" Note: any extra information provided to participants may be unreliable or incorrect; "
"verify it against your own knowledge and the task itself before relying on it."
)
def condition_key(category: str, k: int, n: int, information_source: str = "misinformation",
passage_sharing: str = "shared", mitigation: str = "none", seed: int = 0) -> str:
return (f"{category}|k={k}|n={n}|src={information_source}|share={passage_sharing}|mit={mitigation}|seed={seed}")
_KEY_RE = re.compile(r"^(?P<category>[^|]+)\|k=(?P<k>\d+)\|n=(?P<n>\d+)\|src=(?P<src>[^|]+)\|share=(?P<share>[^|]+)"
r"\|mit=(?P<mit>[^|]+)\|seed=(?P<seed>-?\d+)$")
def parse_condition_key(key: str) -> Optional[dict]:
m = _KEY_RE.match(key)
if not m:
return None
d = m.groupdict()
return {"category": d["category"], "k": int(d["k"]), "n": int(d["n"]), "information_source": d["src"],
"passage_sharing": d["share"], "mitigation": d["mit"], "seed": int(d["seed"])}
def select_passage(item: dict, category: str, information_source: str) -> Optional[str]:
"""Return the text given to an informed agent for this item/category/source."""
if information_source == "misinformation":
return item["passages"].get(category)
if information_source == "relevant_true":
by = (item.get("raw") or {}).get("relevant_true_by_strategy") or {}
txt = by.get(category)
if not txt and category == "false_fact":
txt = item.get("relevant_true_information")
return txt
if information_source == "irrelevant_true":
return item.get("irrelevant_true_information")
if information_source == "filler":
return item.get("filler_information")
raise ValueError(information_source)
def independent_passages(item: dict, category: str, information_source: str, k: int, seed: int) -> List[Optional[str]]:
"""'independent' sharing: k different category passages of the SAME question (deterministic)."""
if information_source != "misinformation":
return [select_passage(item, category, information_source)] * k
cats = [category] + [c for c in ["false_fact"] + STRATEGIES if c != category and item["passages"].get(c)]
rng = random.Random(stable_int(seed, item["mint_id"], "independent"))
rest = cats[1:]
rng.shuffle(rest)
chosen = [cats[0]] + rest
return [item["passages"].get(chosen[i % len(chosen)]) for i in range(k)]
def stable_int(*parts) -> int:
h = hashlib.md5("::".join(str(p) for p in parts).encode("utf-8")).hexdigest()
return int(h[:8], 16)
def assign_slots(item: dict, k: int, n: int, seed: int, passages: List[Optional[str]]) -> List[Optional[str]]:
"""Deterministically place the k passages into n slots (which agents are informed).
Mirrors exp3._stable_shuffle_informations (seeded by example id, k and the global seed)
so the position of misinformed agents varies across questions but is reproducible.
"""
slots: List[Optional[str]] = list(passages[:k]) + [None] * (n - k)
rng = random.Random(stable_int(item["mint_id"], k, seed))
rng.shuffle(slots)
return slots
def task_instruction(dataset: str, mitigation: str = "none") -> str:
instr = TASK_INSTRUCTIONS[dataset]
if mitigation == "source_skepticism":
instr += SOURCE_SKEPTICISM_SUFFIX
return instr
def expand_categories(spec: str, items: List[dict]) -> List[str]:
if spec == "all":
cats: set = set()
for it in items:
cats.update(it["passages"].keys())
return ["false_fact"] + sorted(c for c in cats if c != "false_fact")
return [c.strip() for c in spec.split(",") if c.strip()]