Repository navigation
Expand file tree
/
Copy pathmain code
More file actions
53 lines (46 loc) · 2.07 KB
/
Copy pathmain code
File metadata and controls
53 lines (46 loc) · 2.07 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
_tower = nn.TransformerEncoderLayer(
d_model=d_model, nhead=nhead, dropout=dropout,
self.posterior = nn.Linear(d_model, d_model)
self.prior = nn.Linear(d_model, d_model)
def forward(self, x: torch.Tensor, env: torch.Tensor, culture_id: torch.Tensor):
"""
c_out = self.cognitive_tower(x_embed + culture_vec) # [B, T, D]
e_out = self.emotional_tower(x_embed + env_embed + culture_vec) # [B, T, D]
# Dynamic gating
q_t = torch.mean(env_embed, dim=1) # [B, D]
gate_input = torch.cat([c_out.mean(1), e_out.mean(1), q_t], dim=1) # [B, 3D]
G_t = torch.sigmoid(self.gate_c_out # [B, T, D]
# Heads
lm_logits = self.lm_head(fused) # [B, T, V]
pooled = fused.mean(1) # [B, D]
emo_logits = self.emo_head(pooled) # [B, Emo]
stage_logits = self.stage_head(pooled) # [B, Stage]
safety_score = torch.sigmoid(self.safety_head(pooled)) # [B, 1]
culture_logits = self.culture_head(pooled) # [B, Cultures]
# VIB KL
z_post = self.posterior(pooled) # [B, D]
z_prior = self.prior(env_embed.mean(1)) # [B, D]
kl_div = F.kl_div(
F.log_softmax(z_post, dim=-1),
F.softmax(z_prior, dim=-1),
reduction="batchmean",
)
return {
"lm_logits": lm_logits,
"emo_logits": emo_logits,
"stage_logits": stage_logits,
"safety_score": safety_score,
"culture_logits": culture_logits,
"kl_div": kl_div,
"fused": fused,
}
@torch.no_grad()
def risk_aware_decode(self, lm_logits: torch.Tensor, safety_score: torch.Tensor, beta: float = 0.3):
"""
Adjust logits to suppress unsafe predictions.
lm_logits: [B, T, V], safety_score: [B, 1]
"""
penalty = beta * safety_score.clamp(0, 1) # [B, 1]
penalty = penalty.unsqueeze(-1) # [B, 1, 1]
adjusted = lm_logits -
lm_loss = F.cross_entropy(lm_logits.view(-1, lm_logits.size(-1)), targets["lm"].view(-