Repository navigation
Expand file tree
/
Copy pathconfig.py
More file actions
224 lines (221 loc) · 13.7 KB
/
Copy pathconfig.py
File metadata and controls
224 lines (221 loc) · 13.7 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
import os
from dataclasses import dataclass
from transformers import AutoConfig
@dataclass(slots=True)
class Config:
model: str
max_num_batched_tokens: int = 16384
max_num_seqs: int = 512
max_model_len: int = 4096
gpu_memory_utilization: float = 0.9
tensor_parallel_size: int = 1
# Port for the rank group's dist.init_process_group rendezvous
# (engine/model_runner.py). Was hardcoded to 2333 until 2026-09-06 --
# harmless for a single server instance, but running multiple
# INDEPENDENT server processes on the same machine (e.g. the
# data-parallel-replica topology: N separate tp=1 servers, one per GPU,
# no NCCL between them) had every replica's own process group try to
# rendezvous on the exact same "tcp://localhost:2333", which collides --
# CUDA_VISIBLE_DEVICES isolates which GPU a replica uses, it does nothing
# for this OS-level TCP port. Default unchanged so every existing
# single-server invocation behaves identically; only a multi-replica
# launcher needs to pass a distinct value per instance.
dist_init_port: int = 2333
enforce_eager: bool = False
use_fused_gdr_kernel: bool = False
# Decode-side counterpart to use_fused_gdr_kernel above (that one is
# PREFILL-only -- is_decode_shape forces decode onto the sequential
# per-segment scan regardless of it). This routes the DECODE branch's
# per-request Python loop through fla.ops.gated_delta_rule.
# fused_recurrent_gated_delta_rule -- the single-step recurrent
# counterpart of the chunk kernel prefill uses (models/qwen3_5.py's
# _forward_decode_fused_gdr). REWRITTEN 2026-08-27: the original wired
# phantom, never-committed modules (nanovllm.layers.fused_recurrent /
# gated_delta_net) so this flag was dead; it now calls the real installed
# fla kernel. STILL GPU-UNVALIDATED -- the open question is whether fla's
# recurrent kernel is capturable inside torch.cuda.graph() (the chunk
# kernel is NOT: cudaErrorStreamCaptureInvalidated). Prefer
# use_batched_gdr_decode below (dependency-free, numerically identical,
# CPU-verified, graph-safe by construction) as the primary path; try
# this one on GPU only as an A/B against it -- see THROUGHPUT_PUSH_
# CHECKLIST.md's D2 decision.
use_fused_gdr_decode_kernel: bool = False
# Triton-free batched decode path for Qwen35LinearAttention -- replaces
# forward()'s per-request `for i in range(num_segments)` Python loop (30
# of 40 layers, ~30k tiny kernel launches per captured decode step at
# concurrency 64) with one set of batched tensor ops. Unlike
# use_fused_gdr_decode_kernel above, this has NO missing dependency and
# is numerically identical to the sequential scan (same reduction order),
# CPU-verified (tests/test_qwen35_gdr_decode_batched.py: bitwise@N=1,
# ~1e-8@N=64, no 16-step drift). Still needs a GPU run to confirm it
# CUDA-graph-captures and to measure the actual throughput effect. Off by
# default; the sequential scan stays the ground-truth path.
use_batched_gdr_decode: bool = False
use_vectorized_moe: bool = False
use_moe_w8a8: bool = False
moe_w8a8_weight_group_size: int = 128
# True W8A8 Hopper path (moe_w8a8.cu) -- separate scheme from use_moe_w8a8
# above (2D-blocked FP8, not 1D-grouped INT8), additive not a replacement.
# UNVALIDATED end-to-end -- see w8a8_activation_quant_scoping_memo.md and
# models/qwen3_5.py's NANOVLLM_USE_MOE_W8A8_HOPPER import-site comment.
# Setting this True quantizes weights to FP8 at load time
# (moe_w8a8_hopper_integration.py); the SEPARATE NANOVLLM_USE_MOE_W8A8_HOPPER
# env var controls whether the decode forward path actually calls the
# Hopper kernel with them (mirrors use_moe_w8a8's own
# NANOVLLM_USE_FUSED_MOE_KERNEL split, same reason: two independent
# questions -- "are weights quantized" vs. "which kernel reads them").
# Do not set True before Phase 0 (kernel compile + isolated smoke test)
# has actually run on real Hopper hardware.
use_moe_w8a8_hopper: bool = False
moe_w8a8_hopper_weight_group_size: int = 128
# lm_head INT8 weight-only quantization -- 2026-08-23, see
# tests/lm_head_int8_integration.py's module docstring for the real numbers
# (~485MiB capacity win) and the honest caveat (NOT a confirmed throughput
# win -- lm_head reads its full weight every call, unlike the gathered MoE
# experts, so naive dequant-then-matmul is a plausible bandwidth regression
# until a fused kernel exists). Off by default, independent of use_moe_w8a8.
use_lm_head_int8: bool = False
lm_head_int8_group_size: int = 128
# Folds the eager lm_head GEMM (ParallelLMHead.forward(), currently run
# AFTER graph.replay() every decode step -- see engine/model_runner.py's
# run()) INSIDE the captured CUDA graph instead, alongside the backbone.
# 2026-08-28, CPU window (no GPU access): nsys profiling the same day
# found an unidentified ~2ms/step cost with high variance; this eager
# dispatch (plus the sampler's, addressed separately) is a real
# candidate -- a kernel launch outside the graph pays real CPU-dispatch
# overhead a captured one doesn't. Static audit: ParallelLMHead.forward()
# has no data-dependent branching at decode (context.is_prefill is
# always False during capture -- capture_cudagraph() never captures
# prefill; hasattr(self, "weight_int8") is a load-time module attribute,
# not a runtime tensor check; F.linear and dequantize_weight_int8_grouped
# are both shape-static, no .item()/data-dependent branch).
#
# tp>1 SUPPORTED (2026-08-28): ParallelLMHead.forward()'s own tp>1
# combine allocates FRESH tensors every call
# (`[torch.empty_like(logits) for _ in range(self.tp_size)]` then
# `torch.cat(...)`) -- not capture-safe, and left completely UNCHANGED
# (still the eager-only path for any caller that isn't this one).
# Instead, capture_cudagraph() calls ParallelLMHead._local_logits() (the
# pre-gather per-rank computation, extracted into its own method the
# same day specifically for this reuse) and does its OWN combine against
# static buffers allocated once at capture-setup time: a
# (max_bs, shard_width) buffer per rank for dist.gather's destination
# (only meaningful on rank 0, mirroring ParallelLMHead.forward()'s own
# `if self.tp_rank == 0 else None` convention), and one
# (max_bs, vocab_size) buffer for torch.cat's `out=` parameter --
# dist.gather writes into the persistent gather buffers, torch.cat
# writes into the persistent combined buffer, neither allocates fresh
# memory on replay.
#
# STILL GPU-UNVALIDATED -- same open question as every other graph-
# capture change in this project: does it actually capture cleanly on
# real hardware, now for BOTH the tp=1 and tp>1 code paths (the tp>1
# path additionally exercises dist.gather + torch.cat(out=...) inside a
# captured region, which has no precedent in this codebase to lean on --
# unlike dist.all_reduce, which is already proven capturable at tp=2,
# see project history). Off by default; the eager call after replay()
# stays the ground-truth path.
use_lm_head_in_graph: bool = False
# Debug-only: prints argmax vs sampled token for every prefilled seq on
# every prefill call. Forces an extra argmax() + host-sync .tolist() in
# the hot path even when nobody reads the output -- default off so
# normal runs (including anything being benchmarked) don't pay for it.
debug_print_prefill_samples: bool = False
# Debug-only: prints seq_id/token counts/state_slot on every scheduler
# preemption. Off by default -- under real memory pressure (e.g. a
# concurrency sweep past KV-cache capacity) preemptions can fire in a
# tight burst, and this print previously ran unconditionally inside
# Scheduler.preempt(), which BatchedEngine's background loop calls while
# holding its request-admission lock -- turning a burst of preemptions
# into a burst of serialized stdout syscalls at exactly the moment
# things are already degrading.
debug_print_preemptions: bool = False
# Debug-only: gates the EP forward paths' (_forward_dispatch_ep,
# _forward_gathered_ep, _forward_gathered_ep_w8a8_hopper) independent
# token-id round-trip check, which costs a real extra dist.all_reduce
# on every single EP forward call (prefill AND decode) purely to
# populate self._last_ep_token_id_roundtrip for test verification --
# never read by the numeric output. Off by default so production/
# benchmarked runs don't pay for a second collective on top of the
# functionally-necessary one. tests/moe_ep_dispatch_core.py,
# tests/test_moe_ep_dispatch_decode.py, and
# tests/test_moe_ep_dispatch_edge_cases.py construct Qwen35MoE directly
# with this flag explicitly True, bypassing this Config default.
debug_ep_token_roundtrip: bool = False
# Debug-only: gates StateManager.allocate()'s post-zeroing `.any()`
# assertion, which forces a reduction + host sync on ~63 MB of recurrent
# state per admitted request -- on the scheduler's admission path, which
# BatchedEngine drives while holding its request-admission lock. The
# zeroing itself is unconditional; this only re-checks that `.zero_()`
# took (can fail only on a torch/CUDA bug). Turn on for a
# state-contamination investigation, not for normal/benchmarked runs.
debug_check_state_slot_zeroed: bool = False
hf_config: AutoConfig | None = None
eos: int = -1
kvcache_block_size: int = 256
num_kvcache_blocks: int = -1
def __post_init__(self):
assert os.path.isdir(self.model)
assert self.kvcache_block_size % 256 == 0
assert 1 <= self.tensor_parallel_size <= 8
hf_config = AutoConfig.from_pretrained(self.model)
# VLM checkpoints (e.g. Qwen3.5-MoE) nest language-model fields under
# `text_config`, and recent transformers versions further group RoPE
# hyperparameters (rope_theta, partial_rotary_factor, ...) inside a
# `rope_parameters` dict instead of exposing them as flat attributes.
# Flatten both onto the top-level object so any field downstream code
# reads directly (hidden_size, vocab_size, rope_theta, ...) resolves
# correctly regardless of nesting depth.
#
# Only fill gaps (top-level attribute missing or None) -- never
# overwrite a real top-level value. text_config is itself a full
# PretrainedConfig with its own base-class defaults (e.g.
# `architectures=None`, `model_type="qwen3_5_moe_text"`), and a
# blind overwrite clobbers meaningful top-level fields like
# `architectures` with those meaningless nested defaults.
# layer_types is derived (Qwen3_5MoeConfig.__post_init__ computes it
# from num_hidden_layers + full_attention_interval), not a plain
# config value. text_config is a full PretrainedConfig even when the
# checkpoint's config.json is flat with no "text_config" section --
# it auto-materializes with base-class defaults (num_hidden_layers=40),
# so its layer_types is derived from THAT default count, not the
# top-level checkpoint's real num_hidden_layers. Copying it over
# would silently mismatch _get_layer_types()'s
# `len(layer_types) == num_layers` assert (or worse, if lengths ever
# coincided, apply the wrong attention pattern). Skip it here and let
# _get_layer_types()'s own full_attention_interval fallback
# recompute it against the resolved top-level num_hidden_layers.
text_config = getattr(hf_config, "text_config", None)
if text_config is not None:
for key, value in vars(text_config).items():
if key == "layer_types":
continue
if getattr(hf_config, key, None) is None:
setattr(hf_config, key, value)
rope_parameters = getattr(hf_config, "rope_parameters", None)
if isinstance(rope_parameters, dict):
for key, value in rope_parameters.items():
if getattr(hf_config, key, None) is None:
setattr(hf_config, key, value)
self.hf_config = hf_config
self.max_model_len = min(self.max_model_len, self.hf_config.max_position_embeddings)
# Rides along on hf_config since model_cls() is only ever constructed
# from hf_config, not this Config object -- see Qwen35DecoderLayer's
# getattr(config, "use_fused_gdr_kernel", False) read.
self.hf_config.use_fused_gdr_kernel = self.use_fused_gdr_kernel
self.hf_config.use_fused_gdr_decode_kernel = self.use_fused_gdr_decode_kernel
self.hf_config.use_batched_gdr_decode = self.use_batched_gdr_decode
self.hf_config.use_vectorized_moe = self.use_vectorized_moe
self.hf_config.debug_ep_token_roundtrip = self.debug_ep_token_roundtrip
# NOTE: use_moe_w8a8 / moe_w8a8_weight_group_size deliberately do NOT
# mirror onto hf_config like the two lines above -- unlike
# use_fused_gdr_kernel/use_vectorized_moe (read via
# getattr(config, ...) inside model classes constructed from
# hf_config directly), INT8 quantization is applied as a separate
# pass in engine/model_runner.py (getattr(config, "use_moe_w8a8",
# False), reading THIS Config object, not hf_config) after the model
# is already built, and the MoE forward paths detect int8-ness
# structurally via hasattr(self.experts, "gate_up_proj_int8"), never
# via a config flag. A mirrored hf_config.use_moe_w8a8 previously
# existed here as dead, write-only state (2026-08-26 audit: never
# read anywhere) and was removed.