models/qwen3.6-35b-a3bREADME.md1.1 kB
md
# Qwen3.6 35B-A3B
The text model of Qwen3.6-35B-A3B, a 35B-parameter hybrid mixture of
experts of the Qwen3.5-MoE architecture (`qwen3_5_moe`), about 3B active
per token, without its vision tower or its multi-token-prediction layer.
Its 40 layers come in fours: three of linear attention (Gated DeltaNet) and
one of gated full attention, the same blocks as `qwen3.8-27b` at width
2048 (16 query/key heads and 32 value heads in the linear attention; 16
query heads and 2 key/value heads in the full attention).
Each layer's feed-forward block (`src/moe.linnet`) is a mixture of 256
experts: the router picks 8 per token by its softmax, renormalized over
the chosen, and each expert is a SwiGLU of width 512 whose gate and up
projections the checkpoint stores side by side (`experts.gate_up_proj`,
`[256, 1024, 2048]`). A shared expert of width 512 runs on every token,
scaled by its own sigmoid gate. The chosen experts go through
`std.nn.moe`: a grouped product on a GPU, the dense form elsewhere.
Caches, states and entries are as in `qwen3.8-27b`; `bindings.json` maps
the layers the same way.bindings.json72.7 kB
json
{
"embed_tokens.weight": "model.language_model.embed_tokens.weight",
"norm.weight": "model.language_model.norm.weight",
"lm_head.weight": "lm_head.weight",
"groups.0.linear.0.input_layernorm.weight": "model.language_model.layers.0.input_layernorm.weight",
"groups.0.linear.0.post_attention_layernorm.weight": "model.language_model.layers.0.post_attention_layernorm.weight",
"groups.0.linear.0.mlp.gate.weight": "model.language_model.layers.0.mlp.gate.weight",
"groups.0.linear.0.mlp.gate_up_proj": "model.language_model.layers.0.mlp.experts.gate_up_proj",
"groups.0.linear.0.mlp.down_proj": "model.language_model.layers.0.mlp.experts.down_proj",
"groups.0.linear.0.mlp.shared_expert.gate.weight": "model.language_model.layers.0.mlp.shared_expert.gate_proj.weight",
"groups.0.linear.0.mlp.shared_expert.up.weight": "model.language_model.layers.0.mlp.shared_expert.up_proj.weight",
"groups.0.linear.0.mlp.shared_expert.down.weight": "model.language_model.layers.0.mlp.shared_expert.down_proj.weight",
"groups.0.linear.0.mlp.shared_expert_gate.weight": "model.language_model.layers.0.mlp.shared_expert_gate.weight",
"groups.0.linear.0.linear_attn.in_proj_qkv.weight": "model.language_model.layers.0.linear_attn.in_proj_qkv.weight",
"groups.0.linear.0.linear_attn.in_proj_z.weight": "model.language_model.layers.0.linear_attn.in_proj_z.weight",
"groups.0.linear.0.linear_attn.in_proj_b.weight": "model.language_model.layers.0.linear_attn.in_proj_b.weight",
"groups.0.linear.0.linear_attn.in_proj_a.weight": "model.language_model.layers.0.linear_attn.in_proj_a.weight",
"groups.0.linear.0.linear_attn.out_proj.weight": "model.language_model.layers.0.linear_attn.out_proj.weight",
"groups.0.linear.0.linear_attn.norm.weight": "model.language_model.layers.0.linear_attn.norm.weight",
"groups.0.linear.0.linear_attn.conv_weight": "model.language_model.layers.0.linear_attn.conv1d.weight",
"groups.0.linear.0.linear_attn.A_log": "model.language_model.layers.0.linear_attn.A_log",
"groups.0.linear.0.linear_attn.dt_bias": "model.language_model.layers.0.linear_attn.dt_bias",
"groups.0.linear.1.input_layernorm.weight": "model.language_model.layers.1.input_layernorm.weight",
"groups.0.linear.1.post_attention_layernorm.weight": "model.language_model.layers.1.post_attention_layernorm.weight",
"groups.0.linear.1.mlp.gate.weight": "model.language_model.layers.1.mlp.gate.weight",
"groups.0.linear.1.mlp.gate_up_proj": "model.language_model.layers.1.mlp.experts.gate_up_proj",
"groups.0.linear.1.mlp.down_proj": "model.language_model.layers.1.mlp.experts.down_proj",
"groups.0.linear.1.mlp.shared_expert.gate.weight": "model.language_model.layers.1.mlp.shared_expert.gate_proj.weight",
"groups.0.linear.1.mlp.shared_expert.up.weight": "model.language_model.layers.1.mlp.shared_expert.up_proj.weight",
"groups.0.linear.1.mlp.shared_expert.down.weight": "model.language_model.layers.1.mlp.shared_expert.down_proj.weight",
"groups.0.linear.1.mlp.shared_expert_gate.weight": "model.language_model.layers.1.mlp.shared_expert_gate.weight",
"groups.0.linear.1.linear_attn.in_proj_qkv.weight": "model.language_model.layers.1.linear_attn.in_proj_qkv.weight",
"groups.0.linear.1.linear_attn.in_proj_z.weight": "model.language_model.layers.1.linear_attn.in_proj_z.weight",
"groups.0.linear.1.linear_attn.in_proj_b.weight": "model.language_model.layers.1.linear_attn.in_proj_b.weight",
"groups.0.linear.1.linear_attn.in_proj_a.weight": "model.language_model.layers.1.linear_attn.in_proj_a.weight",
"groups.0.linear.1.linear_attn.out_proj.weight": "model.language_model.layers.1.linear_attn.out_proj.weight",
"groups.0.linear.1.linear_attn.norm.weight": "model.language_model.layers.1.linear_attn.norm.weight",
"groups.0.linear.1.linear_attn.conv_weight": "model.language_model.layers.1.linear_attn.conv1d.weight",
"groups.0.linear.1.linear_attn.A_log": "model.language_model.layers.1.linear_attn.A_log",
"groups.0.linear.1.linear_attn.dt_bias": "model.language_model.layers.1.linear_attn.dt_bias",
"groups.0.linear.2.input_layernorm.weight": "model.language_model.layers.2.input_layernorm.weight",
"groups.0.linear.2.post_attention_layernorm.weight": "model.language_model.layers.2.post_attention_layernorm.weight",
"groups.0.linear.2.mlp.gate.weight": "model.language_model.layers.2.mlp.gate.weight",
"groups.0.linear.2.mlp.gate_up_proj": "model.language_model.layers.2.mlp.experts.gate_up_proj",
"groups.0.linear.2.mlp.down_proj": "model.language_model.layers.2.mlp.experts.down_proj",
"groups.0.linear.2.mlp.shared_expert.gate.weight": "model.language_model.layers.2.mlp.shared_expert.gate_proj.weight",
"groups.0.linear.2.mlp.shared_expert.up.weight": "model.language_model.layers.2.mlp.shared_expert.up_proj.weight",
"groups.0.linear.2.mlp.shared_expert.down.weight": "model.language_model.layers.2.mlp.shared_expert.down_proj.weight",
"groups.0.linear.2.mlp.shared_expert_gate.weight": "model.language_model.layers.2.mlp.shared_expert_gate.weight",
"groups.0.linear.2.linear_attn.in_proj_qkv.weight": "model.language_model.layers.2.linear_attn.in_proj_qkv.weight",
"groups.0.linear.2.linear_attn.in_proj_z.weight": "model.language_model.layers.2.linear_attn.in_proj_z.weight",
"groups.0.linear.2.linear_attn.in_proj_b.weight": "model.language_model.layers.2.linear_attn.in_proj_b.weight",
"groups.0.linear.2.linear_attn.in_proj_a.weight": "model.language_model.layers.2.linear_attn.in_proj_a.weight",
"groups.0.linear.2.linear_attn.out_proj.weight": "model.language_model.layers.2.linear_attn.out_proj.weight",
"groups.0.linear.2.linear_attn.norm.weight": "model.language_model.layers.2.linear_attn.norm.weight",
"groups.0.linear.2.linear_attn.conv_weight": "model.language_model.layers.2.linear_attn.conv1d.weight",
"groups.0.linear.2.linear_attn.A_log": "model.language_model.layers.2.linear_attn.A_log",
"groups.0.linear.2.linear_attn.dt_bias": "model.language_model.layers.2.linear_attn.dt_bias",
"groups.0.full.input_layernorm.weight": "model.language_model.layers.3.input_layernorm.weight",
"groups.0.full.post_attention_layernorm.weight": "model.language_model.layers.3.post_attention_layernorm.weight",
"groups.0.full.mlp.gate.weight": "model.language_model.layers.3.mlp.gate.weight",
"groups.0.full.mlp.gate_up_proj": "model.language_model.layers.3.mlp.experts.gate_up_proj",
"groups.0.full.mlp.down_proj": "model.language_model.layers.3.mlp.experts.down_proj",
"groups.0.full.mlp.shared_expert.gate.weight": "model.language_model.layers.3.mlp.shared_expert.gate_proj.weight",
"groups.0.full.mlp.shared_expert.up.weight": "model.language_model.layers.3.mlp.shared_expert.up_proj.weight",
"groups.0.full.mlp.shared_expert.down.weight": "model.language_model.layers.3.mlp.shared_expert.down_proj.weight",
"groups.0.full.mlp.shared_expert_gate.weight": "model.language_model.layers.3.mlp.shared_expert_gate.weight",
"groups.0.full.self_attn.q_proj.weight": "model.language_model.layers.3.self_attn.q_proj.weight",
"groups.0.full.self_attn.k_proj.weight": "model.language_model.layers.3.self_attn.k_proj.weight",
"groups.0.full.self_attn.v_proj.weight": "model.language_model.layers.3.self_attn.v_proj.weight",
"groups.0.full.self_attn.o_proj.weight": "model.language_model.layers.3.self_attn.o_proj.weight",
"groups.0.full.self_attn.q_norm.weight": "model.language_model.layers.3.self_attn.q_norm.weight",
"groups.0.full.self_attn.k_norm.weight": "model.language_model.layers.3.self_attn.k_norm.weight",
"groups.1.linear.0.input_layernorm.weight": "model.language_model.layers.4.input_layernorm.weight",
"groups.1.linear.0.post_attention_layernorm.weight": "model.language_model.layers.4.post_attention_layernorm.weight",
"groups.1.linear.0.mlp.gate.weight": "model.language_model.layers.4.mlp.gate.weight",
"groups.1.linear.0.mlp.gate_up_proj": "model.language_model.layers.4.mlp.experts.gate_up_proj",
"groups.1.linear.0.mlp.down_proj": "model.language_model.layers.4.mlp.experts.down_proj",
"groups.1.linear.0.mlp.shared_expert.gate.weight": "model.language_model.layers.4.mlp.shared_expert.gate_proj.weight",
"groups.1.linear.0.mlp.shared_expert.up.weight": "model.language_model.layers.4.mlp.shared_expert.up_proj.weight",
"groups.1.linear.0.mlp.shared_expert.down.weight": "model.language_model.layers.4.mlp.shared_expert.down_proj.weight",
"groups.1.linear.0.mlp.shared_expert_gate.weight": "model.language_model.layers.4.mlp.shared_expert_gate.weight",
"groups.1.linear.0.linear_attn.in_proj_qkv.weight": "model.language_model.layers.4.linear_attn.in_proj_qkv.weight",
"groups.1.linear.0.linear_attn.in_proj_z.weight": "model.language_model.layers.4.linear_attn.in_proj_z.weight",
"groups.1.linear.0.linear_attn.in_proj_b.weight": "model.language_model.layers.4.linear_attn.in_proj_b.weight",
"groups.1.linear.0.linear_attn.in_proj_a.weight": "model.language_model.layers.4.linear_attn.in_proj_a.weight",
"groups.1.linear.0.linear_attn.out_proj.weight": "model.language_model.layers.4.linear_attn.out_proj.weight",
"groups.1.linear.0.linear_attn.norm.weight": "model.language_model.layers.4.linear_attn.norm.weight",
"groups.1.linear.0.linear_attn.conv_weight": "model.language_model.layers.4.linear_attn.conv1d.weight",
"groups.1.linear.0.linear_attn.A_log": "model.language_model.layers.4.linear_attn.A_log",
"groups.1.linear.0.linear_attn.dt_bias": "model.language_model.layers.4.linear_attn.dt_bias",
"groups.1.linear.1.input_layernorm.weight": "model.language_model.layers.5.input_layernorm.weight",
"groups.1.linear.1.post_attention_layernorm.weight": "model.language_model.layers.5.post_attention_layernorm.weight",
"groups.1.linear.1.mlp.gate.weight": "model.language_model.layers.5.mlp.gate.weight",
"groups.1.linear.1.mlp.gate_up_proj": "model.language_model.layers.5.mlp.experts.gate_up_proj",
"groups.1.linear.1.mlp.down_proj": "model.language_model.layers.5.mlp.experts.down_proj",
"groups.1.linear.1.mlp.shared_expert.gate.weight": "model.language_model.layers.5.mlp.shared_expert.gate_proj.weight",
"groups.1.linear.1.mlp.shared_expert.up.weight": "model.language_model.layers.5.mlp.shared_expert.up_proj.weight",
"groups.1.linear.1.mlp.shared_expert.down.weight": "model.language_model.layers.5.mlp.shared_expert.down_proj.weight",
"groups.1.linear.1.mlp.shared_expert_gate.weight": "model.language_model.layers.5.mlp.shared_expert_gate.weight",
"groups.1.linear.1.linear_attn.in_proj_qkv.weight": "model.language_model.layers.5.linear_attn.in_proj_qkv.weight",
"groups.1.linear.1.linear_attn.in_proj_z.weight": "model.language_model.layers.5.linear_attn.in_proj_z.weight",
"groups.1.linear.1.linear_attn.in_proj_b.weight": "model.language_model.layers.5.linear_attn.in_proj_b.weight",
"groups.1.linear.1.linear_attn.in_proj_a.weight": "model.language_model.layers.5.linear_attn.in_proj_a.weight",
"groups.1.linear.1.linear_attn.out_proj.weight": "model.language_model.layers.5.linear_attn.out_proj.weight",
"groups.1.linear.1.linear_attn.norm.weight": "model.language_model.layers.5.linear_attn.norm.weight",
"groups.1.linear.1.linear_attn.conv_weight": "model.language_model.layers.5.linear_attn.conv1d.weight",
"groups.1.linear.1.linear_attn.A_log": "model.language_model.layers.5.linear_attn.A_log",
"groups.1.linear.1.linear_attn.dt_bias": "model.language_model.layers.5.linear_attn.dt_bias",
"groups.1.linear.2.input_layernorm.weight": "model.language_model.layers.6.input_layernorm.weight",
"groups.1.linear.2.post_attention_layernorm.weight": "model.language_model.layers.6.post_attention_layernorm.weight",
"groups.1.linear.2.mlp.gate.weight": "model.language_model.layers.6.mlp.gate.weight",
"groups.1.linear.2.mlp.gate_up_proj": "model.language_model.layers.6.mlp.experts.gate_up_proj",
"groups.1.linear.2.mlp.down_proj": "model.language_model.layers.6.mlp.experts.down_proj",
"groups.1.linear.2.mlp.shared_expert.gate.weight": "model.language_model.layers.6.mlp.shared_expert.gate_proj.weight",
"groups.1.linear.2.mlp.shared_expert.up.weight": "model.language_model.layers.6.mlp.shared_expert.up_proj.weight",
"groups.1.linear.2.mlp.shared_expert.down.weight": "model.language_model.layers.6.mlp.shared_expert.down_proj.weight",
"groups.1.linear.2.mlp.shared_expert_gate.weight": "model.language_model.layers.6.mlp.shared_expert_gate.weight",
"groups.1.linear.2.linear_attn.in_proj_qkv.weight": "model.language_model.layers.6.linear_attn.in_proj_qkv.weight",
"groups.1.linear.2.linear_attn.in_proj_z.weight": "model.language_model.layers.6.linear_attn.in_proj_z.weight",
"groups.1.linear.2.linear_attn.in_proj_b.weight": "model.language_model.layers.6.linear_attn.in_proj_b.weight",
"groups.1.linear.2.linear_attn.in_proj_a.weight": "model.language_model.layers.6.linear_attn.in_proj_a.weight",
"groups.1.linear.2.linear_attn.out_proj.weight": "model.language_model.layers.6.linear_attn.out_proj.weight",
"groups.1.linear.2.linear_attn.norm.weight": "model.language_model.layers.6.linear_attn.norm.weight",
"groups.1.linear.2.linear_attn.conv_weight": "model.language_model.layers.6.linear_attn.conv1d.weight",
"groups.1.linear.2.linear_attn.A_log": "model.language_model.layers.6.linear_attn.A_log",
"groups.1.linear.2.linear_attn.dt_bias": "model.language_model.layers.6.linear_attn.dt_bias",
"groups.1.full.input_layernorm.weight": "model.language_model.layers.7.input_layernorm.weight",
"groups.1.full.post_attention_layernorm.weight": "model.language_model.layers.7.post_attention_layernorm.weight",
"groups.1.full.mlp.gate.weight": "model.language_model.layers.7.mlp.gate.weight",
"groups.1.full.mlp.gate_up_proj": "model.language_model.layers.7.mlp.experts.gate_up_proj",
"groups.1.full.mlp.down_proj": "model.language_model.layers.7.mlp.experts.down_proj",
"groups.1.full.mlp.shared_expert.gate.weight": "model.language_model.layers.7.mlp.shared_expert.gate_proj.weight",
"groups.1.full.mlp.shared_expert.up.weight": "model.language_model.layers.7.mlp.shared_expert.up_proj.weight",
"groups.1.full.mlp.shared_expert.down.weight": "model.language_model.layers.7.mlp.shared_expert.down_proj.weight",
"groups.1.full.mlp.shared_expert_gate.weight": "model.language_model.layers.7.mlp.shared_expert_gate.weight",
"groups.1.full.self_attn.q_proj.weight": "model.language_model.layers.7.self_attn.q_proj.weight",
"groups.1.full.self_attn.k_proj.weight": "model.language_model.layers.7.self_attn.k_proj.weight",
"groups.1.full.self_attn.v_proj.weight": "model.language_model.layers.7.self_attn.v_proj.weight",
"groups.1.full.self_attn.o_proj.weight": "model.language_model.layers.7.self_attn.o_proj.weight",
"groups.1.full.self_attn.q_norm.weight": "model.language_model.layers.7.self_attn.q_norm.weight",
"groups.1.full.self_attn.k_norm.weight": "model.language_model.layers.7.self_attn.k_norm.weight",
"groups.2.linear.0.input_layernorm.weight": "model.language_model.layers.8.input_layernorm.weight",
"groups.2.linear.0.post_attention_layernorm.weight": "model.language_model.layers.8.post_attention_layernorm.weight",
"groups.2.linear.0.mlp.gate.weight": "model.language_model.layers.8.mlp.gate.weight",
"groups.2.linear.0.mlp.gate_up_proj": "model.language_model.layers.8.mlp.experts.gate_up_proj",
"groups.2.linear.0.mlp.down_proj": "model.language_model.layers.8.mlp.experts.down_proj",
"groups.2.linear.0.mlp.shared_expert.gate.weight": "model.language_model.layers.8.mlp.shared_expert.gate_proj.weight",
"groups.2.linear.0.mlp.shared_expert.up.weight": "model.language_model.layers.8.mlp.shared_expert.up_proj.weight",
"groups.2.linear.0.mlp.shared_expert.down.weight": "model.language_model.layers.8.mlp.shared_expert.down_proj.weight",
"groups.2.linear.0.mlp.shared_expert_gate.weight": "model.language_model.layers.8.mlp.shared_expert_gate.weight",
"groups.2.linear.0.linear_attn.in_proj_qkv.weight": "model.language_model.layers.8.linear_attn.in_proj_qkv.weight",
"groups.2.linear.0.linear_attn.in_proj_z.weight": "model.language_model.layers.8.linear_attn.in_proj_z.weight",
"groups.2.linear.0.linear_attn.in_proj_b.weight": "model.language_model.layers.8.linear_attn.in_proj_b.weight",
"groups.2.linear.0.linear_attn.in_proj_a.weight": "model.language_model.layers.8.linear_attn.in_proj_a.weight",
"groups.2.linear.0.linear_attn.out_proj.weight": "model.language_model.layers.8.linear_attn.out_proj.weight",
"groups.2.linear.0.linear_attn.norm.weight": "model.language_model.layers.8.linear_attn.norm.weight",
"groups.2.linear.0.linear_attn.conv_weight": "model.language_model.layers.8.linear_attn.conv1d.weight",
"groups.2.linear.0.linear_attn.A_log": "model.language_model.layers.8.linear_attn.A_log",
"groups.2.linear.0.linear_attn.dt_bias": "model.language_model.layers.8.linear_attn.dt_bias",
"groups.2.linear.1.input_layernorm.weight": "model.language_model.layers.9.input_layernorm.weight",
"groups.2.linear.1.post_attention_layernorm.weight": "model.language_model.layers.9.post_attention_layernorm.weight",
"groups.2.linear.1.mlp.gate.weight": "model.language_model.layers.9.mlp.gate.weight",
"groups.2.linear.1.mlp.gate_up_proj": "model.language_model.layers.9.mlp.experts.gate_up_proj",
"groups.2.linear.1.mlp.down_proj": "model.language_model.layers.9.mlp.experts.down_proj",
"groups.2.linear.1.mlp.shared_expert.gate.weight": "model.language_model.layers.9.mlp.shared_expert.gate_proj.weight",
"groups.2.linear.1.mlp.shared_expert.up.weight": "model.language_model.layers.9.mlp.shared_expert.up_proj.weight",
"groups.2.linear.1.mlp.shared_expert.down.weight": "model.language_model.layers.9.mlp.shared_expert.down_proj.weight",
"groups.2.linear.1.mlp.shared_expert_gate.weight": "model.language_model.layers.9.mlp.shared_expert_gate.weight",
"groups.2.linear.1.linear_attn.in_proj_qkv.weight": "model.language_model.layers.9.linear_attn.in_proj_qkv.weight",
"groups.2.linear.1.linear_attn.in_proj_z.weight": "model.language_model.layers.9.linear_attn.in_proj_z.weight",
"groups.2.linear.1.linear_attn.in_proj_b.weight": "model.language_model.layers.9.linear_attn.in_proj_b.weight",
"groups.2.linear.1.linear_attn.in_proj_a.weight": "model.language_model.layers.9.linear_attn.in_proj_a.weight",
"groups.2.linear.1.linear_attn.out_proj.weight": "model.language_model.layers.9.linear_attn.out_proj.weight",
"groups.2.linear.1.linear_attn.norm.weight": "model.language_model.layers.9.linear_attn.norm.weight",
"groups.2.linear.1.linear_attn.conv_weight": "model.language_model.layers.9.linear_attn.conv1d.weight",
"groups.2.linear.1.linear_attn.A_log": "model.language_model.layers.9.linear_attn.A_log",
"groups.2.linear.1.linear_attn.dt_bias": "model.language_model.layers.9.linear_attn.dt_bias",
"groups.2.linear.2.input_layernorm.weight": "model.language_model.layers.10.input_layernorm.weight",
"groups.2.linear.2.post_attention_layernorm.weight": "model.language_model.layers.10.post_attention_layernorm.weight",
"groups.2.linear.2.mlp.gate.weight": "model.language_model.layers.10.mlp.gate.weight",
"groups.2.linear.2.mlp.gate_up_proj": "model.language_model.layers.10.mlp.experts.gate_up_proj",
"groups.2.linear.2.mlp.down_proj": "model.language_model.layers.10.mlp.experts.down_proj",
"groups.2.linear.2.mlp.shared_expert.gate.weight": "model.language_model.layers.10.mlp.shared_expert.gate_proj.weight",
"groups.2.linear.2.mlp.shared_expert.up.weight": "model.language_model.layers.10.mlp.shared_expert.up_proj.weight",
"groups.2.linear.2.mlp.shared_expert.down.weight": "model.language_model.layers.10.mlp.shared_expert.down_proj.weight",
"groups.2.linear.2.mlp.shared_expert_gate.weight": "model.language_model.layers.10.mlp.shared_expert_gate.weight",
"groups.2.linear.2.linear_attn.in_proj_qkv.weight": "model.language_model.layers.10.linear_attn.in_proj_qkv.weight",
"groups.2.linear.2.linear_attn.in_proj_z.weight": "model.language_model.layers.10.linear_attn.in_proj_z.weight",
"groups.2.linear.2.linear_attn.in_proj_b.weight": "model.language_model.layers.10.linear_attn.in_proj_b.weight",
"groups.2.linear.2.linear_attn.in_proj_a.weight": "model.language_model.layers.10.linear_attn.in_proj_a.weight",
"groups.2.linear.2.linear_attn.out_proj.weight": "model.language_model.layers.10.linear_attn.out_proj.weight",
"groups.2.linear.2.linear_attn.norm.weight": "model.language_model.layers.10.linear_attn.norm.weight",
"groups.2.linear.2.linear_attn.conv_weight": "model.language_model.layers.10.linear_attn.conv1d.weight",
"groups.2.linear.2.linear_attn.A_log": "model.language_model.layers.10.linear_attn.A_log",
"groups.2.linear.2.linear_attn.dt_bias": "model.language_model.layers.10.linear_attn.dt_bias",
"groups.2.full.input_layernorm.weight": "model.language_model.layers.11.input_layernorm.weight",
"groups.2.full.post_attention_layernorm.weight": "model.language_model.layers.11.post_attention_layernorm.weight",
"groups.2.full.mlp.gate.weight": "model.language_model.layers.11.mlp.gate.weight",
"groups.2.full.mlp.gate_up_proj": "model.language_model.layers.11.mlp.experts.gate_up_proj",
"groups.2.full.mlp.down_proj": "model.language_model.layers.11.mlp.experts.down_proj",
"groups.2.full.mlp.shared_expert.gate.weight": "model.language_model.layers.11.mlp.shared_expert.gate_proj.weight",
"groups.2.full.mlp.shared_expert.up.weight": "model.language_model.layers.11.mlp.shared_expert.up_proj.weight",
"groups.2.full.mlp.shared_expert.down.weight": "model.language_model.layers.11.mlp.shared_expert.down_proj.weight",
"groups.2.full.mlp.shared_expert_gate.weight": "model.language_model.layers.11.mlp.shared_expert_gate.weight",
"groups.2.full.self_attn.q_proj.weight": "model.language_model.layers.11.self_attn.q_proj.weight",
"groups.2.full.self_attn.k_proj.weight": "model.language_model.layers.11.self_attn.k_proj.weight",
"groups.2.full.self_attn.v_proj.weight": "model.language_model.layers.11.self_attn.v_proj.weight",
"groups.2.full.self_attn.o_proj.weight": "model.language_model.layers.11.self_attn.o_proj.weight",
"groups.2.full.self_attn.q_norm.weight": "model.language_model.layers.11.self_attn.q_norm.weight",
"groups.2.full.self_attn.k_norm.weight": "model.language_model.layers.11.self_attn.k_norm.weight",
"groups.3.linear.0.input_layernorm.weight": "model.language_model.layers.12.input_layernorm.weight",
"groups.3.linear.0.post_attention_layernorm.weight": "model.language_model.layers.12.post_attention_layernorm.weight",
"groups.3.linear.0.mlp.gate.weight": "model.language_model.layers.12.mlp.gate.weight",
"groups.3.linear.0.mlp.gate_up_proj": "model.language_model.layers.12.mlp.experts.gate_up_proj",
"groups.3.linear.0.mlp.down_proj": "model.language_model.layers.12.mlp.experts.down_proj",
"groups.3.linear.0.mlp.shared_expert.gate.weight": "model.language_model.layers.12.mlp.shared_expert.gate_proj.weight",
"groups.3.linear.0.mlp.shared_expert.up.weight": "model.language_model.layers.12.mlp.shared_expert.up_proj.weight",
"groups.3.linear.0.mlp.shared_expert.down.weight": "model.language_model.layers.12.mlp.shared_expert.down_proj.weight",
"groups.3.linear.0.mlp.shared_expert_gate.weight": "model.language_model.layers.12.mlp.shared_expert_gate.weight",
"groups.3.linear.0.linear_attn.in_proj_qkv.weight": "model.language_model.layers.12.linear_attn.in_proj_qkv.weight",
"groups.3.linear.0.linear_attn.in_proj_z.weight": "model.language_model.layers.12.linear_attn.in_proj_z.weight",
"groups.3.linear.0.linear_attn.in_proj_b.weight": "model.language_model.layers.12.linear_attn.in_proj_b.weight",
"groups.3.linear.0.linear_attn.in_proj_a.weight": "model.language_model.layers.12.linear_attn.in_proj_a.weight",
"groups.3.linear.0.linear_attn.out_proj.weight": "model.language_model.layers.12.linear_attn.out_proj.weight",
"groups.3.linear.0.linear_attn.norm.weight": "model.language_model.layers.12.linear_attn.norm.weight",
"groups.3.linear.0.linear_attn.conv_weight": "model.language_model.layers.12.linear_attn.conv1d.weight",
"groups.3.linear.0.linear_attn.A_log": "model.language_model.layers.12.linear_attn.A_log",
"groups.3.linear.0.linear_attn.dt_bias": "model.language_model.layers.12.linear_attn.dt_bias",
"groups.3.linear.1.input_layernorm.weight": "model.language_model.layers.13.input_layernorm.weight",
"groups.3.linear.1.post_attention_layernorm.weight": "model.language_model.layers.13.post_attention_layernorm.weight",
"groups.3.linear.1.mlp.gate.weight": "model.language_model.layers.13.mlp.gate.weight",
"groups.3.linear.1.mlp.gate_up_proj": "model.language_model.layers.13.mlp.experts.gate_up_proj",
"groups.3.linear.1.mlp.down_proj": "model.language_model.layers.13.mlp.experts.down_proj",
"groups.3.linear.1.mlp.shared_expert.gate.weight": "model.language_model.layers.13.mlp.shared_expert.gate_proj.weight",
"groups.3.linear.1.mlp.shared_expert.up.weight": "model.language_model.layers.13.mlp.shared_expert.up_proj.weight",
"groups.3.linear.1.mlp.shared_expert.down.weight": "model.language_model.layers.13.mlp.shared_expert.down_proj.weight",
"groups.3.linear.1.mlp.shared_expert_gate.weight": "model.language_model.layers.13.mlp.shared_expert_gate.weight",
"groups.3.linear.1.linear_attn.in_proj_qkv.weight": "model.language_model.layers.13.linear_attn.in_proj_qkv.weight",
"groups.3.linear.1.linear_attn.in_proj_z.weight": "model.language_model.layers.13.linear_attn.in_proj_z.weight",
"groups.3.linear.1.linear_attn.in_proj_b.weight": "model.language_model.layers.13.linear_attn.in_proj_b.weight",
"groups.3.linear.1.linear_attn.in_proj_a.weight": "model.language_model.layers.13.linear_attn.in_proj_a.weight",
"groups.3.linear.1.linear_attn.out_proj.weight": "model.language_model.layers.13.linear_attn.out_proj.weight",
"groups.3.linear.1.linear_attn.norm.weight": "model.language_model.layers.13.linear_attn.norm.weight",
"groups.3.linear.1.linear_attn.conv_weight": "model.language_model.layers.13.linear_attn.conv1d.weight",
"groups.3.linear.1.linear_attn.A_log": "model.language_model.layers.13.linear_attn.A_log",
"groups.3.linear.1.linear_attn.dt_bias": "model.language_model.layers.13.linear_attn.dt_bias",
"groups.3.linear.2.input_layernorm.weight": "model.language_model.layers.14.input_layernorm.weight",
"groups.3.linear.2.post_attention_layernorm.weight": "model.language_model.layers.14.post_attention_layernorm.weight",
"groups.3.linear.2.mlp.gate.weight": "model.language_model.layers.14.mlp.gate.weight",
"groups.3.linear.2.mlp.gate_up_proj": "model.language_model.layers.14.mlp.experts.gate_up_proj",
"groups.3.linear.2.mlp.down_proj": "model.language_model.layers.14.mlp.experts.down_proj",
"groups.3.linear.2.mlp.shared_expert.gate.weight": "model.language_model.layers.14.mlp.shared_expert.gate_proj.weight",
"groups.3.linear.2.mlp.shared_expert.up.weight": "model.language_model.layers.14.mlp.shared_expert.up_proj.weight",
"groups.3.linear.2.mlp.shared_expert.down.weight": "model.language_model.layers.14.mlp.shared_expert.down_proj.weight",
"groups.3.linear.2.mlp.shared_expert_gate.weight": "model.language_model.layers.14.mlp.shared_expert_gate.weight",
"groups.3.linear.2.linear_attn.in_proj_qkv.weight": "model.language_model.layers.14.linear_attn.in_proj_qkv.weight",
"groups.3.linear.2.linear_attn.in_proj_z.weight": "model.language_model.layers.14.linear_attn.in_proj_z.weight",
"groups.3.linear.2.linear_attn.in_proj_b.weight": "model.language_model.layers.14.linear_attn.in_proj_b.weight",
"groups.3.linear.2.linear_attn.in_proj_a.weight": "model.language_model.layers.14.linear_attn.in_proj_a.weight",
"groups.3.linear.2.linear_attn.out_proj.weight": "model.language_model.layers.14.linear_attn.out_proj.weight",
"groups.3.linear.2.linear_attn.norm.weight": "model.language_model.layers.14.linear_attn.norm.weight",
"groups.3.linear.2.linear_attn.conv_weight": "model.language_model.layers.14.linear_attn.conv1d.weight",
"groups.3.linear.2.linear_attn.A_log": "model.language_model.layers.14.linear_attn.A_log",
"groups.3.linear.2.linear_attn.dt_bias": "model.language_model.layers.14.linear_attn.dt_bias",
"groups.3.full.input_layernorm.weight": "model.language_model.layers.15.input_layernorm.weight",
"groups.3.full.post_attention_layernorm.weight": "model.language_model.layers.15.post_attention_layernorm.weight",
"groups.3.full.mlp.gate.weight": "model.language_model.layers.15.mlp.gate.weight",
"groups.3.full.mlp.gate_up_proj": "model.language_model.layers.15.mlp.experts.gate_up_proj",
"groups.3.full.mlp.down_proj": "model.language_model.layers.15.mlp.experts.down_proj",
"groups.3.full.mlp.shared_expert.gate.weight": "model.language_model.layers.15.mlp.shared_expert.gate_proj.weight",
"groups.3.full.mlp.shared_expert.up.weight": "model.language_model.layers.15.mlp.shared_expert.up_proj.weight",
"groups.3.full.mlp.shared_expert.down.weight": "model.language_model.layers.15.mlp.shared_expert.down_proj.weight",
"groups.3.full.mlp.shared_expert_gate.weight": "model.language_model.layers.15.mlp.shared_expert_gate.weight",
"groups.3.full.self_attn.q_proj.weight": "model.language_model.layers.15.self_attn.q_proj.weight",
"groups.3.full.self_attn.k_proj.weight": "model.language_model.layers.15.self_attn.k_proj.weight",
"groups.3.full.self_attn.v_proj.weight": "model.language_model.layers.15.self_attn.v_proj.weight",
"groups.3.full.self_attn.o_proj.weight": "model.language_model.layers.15.self_attn.o_proj.weight",
"groups.3.full.self_attn.q_norm.weight": "model.language_model.layers.15.self_attn.q_norm.weight",
"groups.3.full.self_attn.k_norm.weight": "model.language_model.layers.15.self_attn.k_norm.weight",
"groups.4.linear.0.input_layernorm.weight": "model.language_model.layers.16.input_layernorm.weight",
"groups.4.linear.0.post_attention_layernorm.weight": "model.language_model.layers.16.post_attention_layernorm.weight",
"groups.4.linear.0.mlp.gate.weight": "model.language_model.layers.16.mlp.gate.weight",
"groups.4.linear.0.mlp.gate_up_proj": "model.language_model.layers.16.mlp.experts.gate_up_proj",
"groups.4.linear.0.mlp.down_proj": "model.language_model.layers.16.mlp.experts.down_proj",
"groups.4.linear.0.mlp.shared_expert.gate.weight": "model.language_model.layers.16.mlp.shared_expert.gate_proj.weight",
"groups.4.linear.0.mlp.shared_expert.up.weight": "model.language_model.layers.16.mlp.shared_expert.up_proj.weight",
"groups.4.linear.0.mlp.shared_expert.down.weight": "model.language_model.layers.16.mlp.shared_expert.down_proj.weight",
"groups.4.linear.0.mlp.shared_expert_gate.weight": "model.language_model.layers.16.mlp.shared_expert_gate.weight",
"groups.4.linear.0.linear_attn.in_proj_qkv.weight": "model.language_model.layers.16.linear_attn.in_proj_qkv.weight",
"groups.4.linear.0.linear_attn.in_proj_z.weight": "model.language_model.layers.16.linear_attn.in_proj_z.weight",
"groups.4.linear.0.linear_attn.in_proj_b.weight": "model.language_model.layers.16.linear_attn.in_proj_b.weight",
"groups.4.linear.0.linear_attn.in_proj_a.weight": "model.language_model.layers.16.linear_attn.in_proj_a.weight",
"groups.4.linear.0.linear_attn.out_proj.weight": "model.language_model.layers.16.linear_attn.out_proj.weight",
"groups.4.linear.0.linear_attn.norm.weight": "model.language_model.layers.16.linear_attn.norm.weight",
"groups.4.linear.0.linear_attn.conv_weight": "model.language_model.layers.16.linear_attn.conv1d.weight",
"groups.4.linear.0.linear_attn.A_log": "model.language_model.layers.16.linear_attn.A_log",
"groups.4.linear.0.linear_attn.dt_bias": "model.language_model.layers.16.linear_attn.dt_bias",
"groups.4.linear.1.input_layernorm.weight": "model.language_model.layers.17.input_layernorm.weight",
"groups.4.linear.1.post_attention_layernorm.weight": "model.language_model.layers.17.post_attention_layernorm.weight",
"groups.4.linear.1.mlp.gate.weight": "model.language_model.layers.17.mlp.gate.weight",
"groups.4.linear.1.mlp.gate_up_proj": "model.language_model.layers.17.mlp.experts.gate_up_proj",
"groups.4.linear.1.mlp.down_proj": "model.language_model.layers.17.mlp.experts.down_proj",
"groups.4.linear.1.mlp.shared_expert.gate.weight": "model.language_model.layers.17.mlp.shared_expert.gate_proj.weight",
"groups.4.linear.1.mlp.shared_expert.up.weight": "model.language_model.layers.17.mlp.shared_expert.up_proj.weight",
"groups.4.linear.1.mlp.shared_expert.down.weight": "model.language_model.layers.17.mlp.shared_expert.down_proj.weight",
"groups.4.linear.1.mlp.shared_expert_gate.weight": "model.language_model.layers.17.mlp.shared_expert_gate.weight",
"groups.4.linear.1.linear_attn.in_proj_qkv.weight": "model.language_model.layers.17.linear_attn.in_proj_qkv.weight",
"groups.4.linear.1.linear_attn.in_proj_z.weight": "model.language_model.layers.17.linear_attn.in_proj_z.weight",
"groups.4.linear.1.linear_attn.in_proj_b.weight": "model.language_model.layers.17.linear_attn.in_proj_b.weight",
"groups.4.linear.1.linear_attn.in_proj_a.weight": "model.language_model.layers.17.linear_attn.in_proj_a.weight",
"groups.4.linear.1.linear_attn.out_proj.weight": "model.language_model.layers.17.linear_attn.out_proj.weight",
"groups.4.linear.1.linear_attn.norm.weight": "model.language_model.layers.17.linear_attn.norm.weight",
"groups.4.linear.1.linear_attn.conv_weight": "model.language_model.layers.17.linear_attn.conv1d.weight",
"groups.4.linear.1.linear_attn.A_log": "model.language_model.layers.17.linear_attn.A_log",
"groups.4.linear.1.linear_attn.dt_bias": "model.language_model.layers.17.linear_attn.dt_bias",
"groups.4.linear.2.input_layernorm.weight": "model.language_model.layers.18.input_layernorm.weight",
"groups.4.linear.2.post_attention_layernorm.weight": "model.language_model.layers.18.post_attention_layernorm.weight",
"groups.4.linear.2.mlp.gate.weight": "model.language_model.layers.18.mlp.gate.weight",
"groups.4.linear.2.mlp.gate_up_proj": "model.language_model.layers.18.mlp.experts.gate_up_proj",
"groups.4.linear.2.mlp.down_proj": "model.language_model.layers.18.mlp.experts.down_proj",
"groups.4.linear.2.mlp.shared_expert.gate.weight": "model.language_model.layers.18.mlp.shared_expert.gate_proj.weight",
"groups.4.linear.2.mlp.shared_expert.up.weight": "model.language_model.layers.18.mlp.shared_expert.up_proj.weight",
"groups.4.linear.2.mlp.shared_expert.down.weight": "model.language_model.layers.18.mlp.shared_expert.down_proj.weight",
"groups.4.linear.2.mlp.shared_expert_gate.weight": "model.language_model.layers.18.mlp.shared_expert_gate.weight",
"groups.4.linear.2.linear_attn.in_proj_qkv.weight": "model.language_model.layers.18.linear_attn.in_proj_qkv.weight",
"groups.4.linear.2.linear_attn.in_proj_z.weight": "model.language_model.layers.18.linear_attn.in_proj_z.weight",
"groups.4.linear.2.linear_attn.in_proj_b.weight": "model.language_model.layers.18.linear_attn.in_proj_b.weight",
"groups.4.linear.2.linear_attn.in_proj_a.weight": "model.language_model.layers.18.linear_attn.in_proj_a.weight",
"groups.4.linear.2.linear_attn.out_proj.weight": "model.language_model.layers.18.linear_attn.out_proj.weight",
"groups.4.linear.2.linear_attn.norm.weight": "model.language_model.layers.18.linear_attn.norm.weight",
"groups.4.linear.2.linear_attn.conv_weight": "model.language_model.layers.18.linear_attn.conv1d.weight",
"groups.4.linear.2.linear_attn.A_log": "model.language_model.layers.18.linear_attn.A_log",
"groups.4.linear.2.linear_attn.dt_bias": "model.language_model.layers.18.linear_attn.dt_bias",
"groups.4.full.input_layernorm.weight": "model.language_model.layers.19.input_layernorm.weight",
"groups.4.full.post_attention_layernorm.weight": "model.language_model.layers.19.post_attention_layernorm.weight",
"groups.4.full.mlp.gate.weight": "model.language_model.layers.19.mlp.gate.weight",
"groups.4.full.mlp.gate_up_proj": "model.language_model.layers.19.mlp.experts.gate_up_proj",
"groups.4.full.mlp.down_proj": "model.language_model.layers.19.mlp.experts.down_proj",
"groups.4.full.mlp.shared_expert.gate.weight": "model.language_model.layers.19.mlp.shared_expert.gate_proj.weight",
"groups.4.full.mlp.shared_expert.up.weight": "model.language_model.layers.19.mlp.shared_expert.up_proj.weight",
"groups.4.full.mlp.shared_expert.down.weight": "model.language_model.layers.19.mlp.shared_expert.down_proj.weight",
"groups.4.full.mlp.shared_expert_gate.weight": "model.language_model.layers.19.mlp.shared_expert_gate.weight",
"groups.4.full.self_attn.q_proj.weight": "model.language_model.layers.19.self_attn.q_proj.weight",
"groups.4.full.self_attn.k_proj.weight": "model.language_model.layers.19.self_attn.k_proj.weight",
"groups.4.full.self_attn.v_proj.weight": "model.language_model.layers.19.self_attn.v_proj.weight",
"groups.4.full.self_attn.o_proj.weight": "model.language_model.layers.19.self_attn.o_proj.weight",
"groups.4.full.self_attn.q_norm.weight": "model.language_model.layers.19.self_attn.q_norm.weight",
"groups.4.full.self_attn.k_norm.weight": "model.language_model.layers.19.self_attn.k_norm.weight",
"groups.5.linear.0.input_layernorm.weight": "model.language_model.layers.20.input_layernorm.weight",
"groups.5.linear.0.post_attention_layernorm.weight": "model.language_model.layers.20.post_attention_layernorm.weight",
"groups.5.linear.0.mlp.gate.weight": "model.language_model.layers.20.mlp.gate.weight",
"groups.5.linear.0.mlp.gate_up_proj": "model.language_model.layers.20.mlp.experts.gate_up_proj",
"groups.5.linear.0.mlp.down_proj": "model.language_model.layers.20.mlp.experts.down_proj",
"groups.5.linear.0.mlp.shared_expert.gate.weight": "model.language_model.layers.20.mlp.shared_expert.gate_proj.weight",
"groups.5.linear.0.mlp.shared_expert.up.weight": "model.language_model.layers.20.mlp.shared_expert.up_proj.weight",
"groups.5.linear.0.mlp.shared_expert.down.weight": "model.language_model.layers.20.mlp.shared_expert.down_proj.weight",
"groups.5.linear.0.mlp.shared_expert_gate.weight": "model.language_model.layers.20.mlp.shared_expert_gate.weight",
"groups.5.linear.0.linear_attn.in_proj_qkv.weight": "model.language_model.layers.20.linear_attn.in_proj_qkv.weight",
"groups.5.linear.0.linear_attn.in_proj_z.weight": "model.language_model.layers.20.linear_attn.in_proj_z.weight",
"groups.5.linear.0.linear_attn.in_proj_b.weight": "model.language_model.layers.20.linear_attn.in_proj_b.weight",
"groups.5.linear.0.linear_attn.in_proj_a.weight": "model.language_model.layers.20.linear_attn.in_proj_a.weight",
"groups.5.linear.0.linear_attn.out_proj.weight": "model.language_model.layers.20.linear_attn.out_proj.weight",
"groups.5.linear.0.linear_attn.norm.weight": "model.language_model.layers.20.linear_attn.norm.weight",
"groups.5.linear.0.linear_attn.conv_weight": "model.language_model.layers.20.linear_attn.conv1d.weight",
"groups.5.linear.0.linear_attn.A_log": "model.language_model.layers.20.linear_attn.A_log",
"groups.5.linear.0.linear_attn.dt_bias": "model.language_model.layers.20.linear_attn.dt_bias",
"groups.5.linear.1.input_layernorm.weight": "model.language_model.layers.21.input_layernorm.weight",
"groups.5.linear.1.post_attention_layernorm.weight": "model.language_model.layers.21.post_attention_layernorm.weight",
"groups.5.linear.1.mlp.gate.weight": "model.language_model.layers.21.mlp.gate.weight",
"groups.5.linear.1.mlp.gate_up_proj": "model.language_model.layers.21.mlp.experts.gate_up_proj",
"groups.5.linear.1.mlp.down_proj": "model.language_model.layers.21.mlp.experts.down_proj",
"groups.5.linear.1.mlp.shared_expert.gate.weight": "model.language_model.layers.21.mlp.shared_expert.gate_proj.weight",
"groups.5.linear.1.mlp.shared_expert.up.weight": "model.language_model.layers.21.mlp.shared_expert.up_proj.weight",
"groups.5.linear.1.mlp.shared_expert.down.weight": "model.language_model.layers.21.mlp.shared_expert.down_proj.weight",
"groups.5.linear.1.mlp.shared_expert_gate.weight": "model.language_model.layers.21.mlp.shared_expert_gate.weight",
"groups.5.linear.1.linear_attn.in_proj_qkv.weight": "model.language_model.layers.21.linear_attn.in_proj_qkv.weight",
"groups.5.linear.1.linear_attn.in_proj_z.weight": "model.language_model.layers.21.linear_attn.in_proj_z.weight",
"groups.5.linear.1.linear_attn.in_proj_b.weight": "model.language_model.layers.21.linear_attn.in_proj_b.weight",
"groups.5.linear.1.linear_attn.in_proj_a.weight": "model.language_model.layers.21.linear_attn.in_proj_a.weight",
"groups.5.linear.1.linear_attn.out_proj.weight": "model.language_model.layers.21.linear_attn.out_proj.weight",
"groups.5.linear.1.linear_attn.norm.weight": "model.language_model.layers.21.linear_attn.norm.weight",
"groups.5.linear.1.linear_attn.conv_weight": "model.language_model.layers.21.linear_attn.conv1d.weight",
"groups.5.linear.1.linear_attn.A_log": "model.language_model.layers.21.linear_attn.A_log",
"groups.5.linear.1.linear_attn.dt_bias": "model.language_model.layers.21.linear_attn.dt_bias",
"groups.5.linear.2.input_layernorm.weight": "model.language_model.layers.22.input_layernorm.weight",
"groups.5.linear.2.post_attention_layernorm.weight": "model.language_model.layers.22.post_attention_layernorm.weight",
"groups.5.linear.2.mlp.gate.weight": "model.language_model.layers.22.mlp.gate.weight",
"groups.5.linear.2.mlp.gate_up_proj": "model.language_model.layers.22.mlp.experts.gate_up_proj",
"groups.5.linear.2.mlp.down_proj": "model.language_model.layers.22.mlp.experts.down_proj",
"groups.5.linear.2.mlp.shared_expert.gate.weight": "model.language_model.layers.22.mlp.shared_expert.gate_proj.weight",
"groups.5.linear.2.mlp.shared_expert.up.weight": "model.language_model.layers.22.mlp.shared_expert.up_proj.weight",
"groups.5.linear.2.mlp.shared_expert.down.weight": "model.language_model.layers.22.mlp.shared_expert.down_proj.weight",
"groups.5.linear.2.mlp.shared_expert_gate.weight": "model.language_model.layers.22.mlp.shared_expert_gate.weight",
"groups.5.linear.2.linear_attn.in_proj_qkv.weight": "model.language_model.layers.22.linear_attn.in_proj_qkv.weight",
"groups.5.linear.2.linear_attn.in_proj_z.weight": "model.language_model.layers.22.linear_attn.in_proj_z.weight",
"groups.5.linear.2.linear_attn.in_proj_b.weight": "model.language_model.layers.22.linear_attn.in_proj_b.weight",
"groups.5.linear.2.linear_attn.in_proj_a.weight": "model.language_model.layers.22.linear_attn.in_proj_a.weight",
"groups.5.linear.2.linear_attn.out_proj.weight": "model.language_model.layers.22.linear_attn.out_proj.weight",
"groups.5.linear.2.linear_attn.norm.weight": "model.language_model.layers.22.linear_attn.norm.weight",
"groups.5.linear.2.linear_attn.conv_weight": "model.language_model.layers.22.linear_attn.conv1d.weight",
"groups.5.linear.2.linear_attn.A_log": "model.language_model.layers.22.linear_attn.A_log",
"groups.5.linear.2.linear_attn.dt_bias": "model.language_model.layers.22.linear_attn.dt_bias",
"groups.5.full.input_layernorm.weight": "model.language_model.layers.23.input_layernorm.weight",
"groups.5.full.post_attention_layernorm.weight": "model.language_model.layers.23.post_attention_layernorm.weight",
"groups.5.full.mlp.gate.weight": "model.language_model.layers.23.mlp.gate.weight",
"groups.5.full.mlp.gate_up_proj": "model.language_model.layers.23.mlp.experts.gate_up_proj",
"groups.5.full.mlp.down_proj": "model.language_model.layers.23.mlp.experts.down_proj",
"groups.5.full.mlp.shared_expert.gate.weight": "model.language_model.layers.23.mlp.shared_expert.gate_proj.weight",
"groups.5.full.mlp.shared_expert.up.weight": "model.language_model.layers.23.mlp.shared_expert.up_proj.weight",
"groups.5.full.mlp.shared_expert.down.weight": "model.language_model.layers.23.mlp.shared_expert.down_proj.weight",
"groups.5.full.mlp.shared_expert_gate.weight": "model.language_model.layers.23.mlp.shared_expert_gate.weight",
"groups.5.full.self_attn.q_proj.weight": "model.language_model.layers.23.self_attn.q_proj.weight",
"groups.5.full.self_attn.k_proj.weight": "model.language_model.layers.23.self_attn.k_proj.weight",
"groups.5.full.self_attn.v_proj.weight": "model.language_model.layers.23.self_attn.v_proj.weight",
"groups.5.full.self_attn.o_proj.weight": "model.language_model.layers.23.self_attn.o_proj.weight",
"groups.5.full.self_attn.q_norm.weight": "model.language_model.layers.23.self_attn.q_norm.weight",
"groups.5.full.self_attn.k_norm.weight": "model.language_model.layers.23.self_attn.k_norm.weight",
"groups.6.linear.0.input_layernorm.weight": "model.language_model.layers.24.input_layernorm.weight",
"groups.6.linear.0.post_attention_layernorm.weight": "model.language_model.layers.24.post_attention_layernorm.weight",
"groups.6.linear.0.mlp.gate.weight": "model.language_model.layers.24.mlp.gate.weight",
"groups.6.linear.0.mlp.gate_up_proj": "model.language_model.layers.24.mlp.experts.gate_up_proj",
"groups.6.linear.0.mlp.down_proj": "model.language_model.layers.24.mlp.experts.down_proj",
"groups.6.linear.0.mlp.shared_expert.gate.weight": "model.language_model.layers.24.mlp.shared_expert.gate_proj.weight",
"groups.6.linear.0.mlp.shared_expert.up.weight": "model.language_model.layers.24.mlp.shared_expert.up_proj.weight",
"groups.6.linear.0.mlp.shared_expert.down.weight": "model.language_model.layers.24.mlp.shared_expert.down_proj.weight",
"groups.6.linear.0.mlp.shared_expert_gate.weight": "model.language_model.layers.24.mlp.shared_expert_gate.weight",
"groups.6.linear.0.linear_attn.in_proj_qkv.weight": "model.language_model.layers.24.linear_attn.in_proj_qkv.weight",
"groups.6.linear.0.linear_attn.in_proj_z.weight": "model.language_model.layers.24.linear_attn.in_proj_z.weight",
"groups.6.linear.0.linear_attn.in_proj_b.weight": "model.language_model.layers.24.linear_attn.in_proj_b.weight",
"groups.6.linear.0.linear_attn.in_proj_a.weight": "model.language_model.layers.24.linear_attn.in_proj_a.weight",
"groups.6.linear.0.linear_attn.out_proj.weight": "model.language_model.layers.24.linear_attn.out_proj.weight",
"groups.6.linear.0.linear_attn.norm.weight": "model.language_model.layers.24.linear_attn.norm.weight",
"groups.6.linear.0.linear_attn.conv_weight": "model.language_model.layers.24.linear_attn.conv1d.weight",
"groups.6.linear.0.linear_attn.A_log": "model.language_model.layers.24.linear_attn.A_log",
"groups.6.linear.0.linear_attn.dt_bias": "model.language_model.layers.24.linear_attn.dt_bias",
"groups.6.linear.1.input_layernorm.weight": "model.language_model.layers.25.input_layernorm.weight",
"groups.6.linear.1.post_attention_layernorm.weight": "model.language_model.layers.25.post_attention_layernorm.weight",
"groups.6.linear.1.mlp.gate.weight": "model.language_model.layers.25.mlp.gate.weight",
"groups.6.linear.1.mlp.gate_up_proj": "model.language_model.layers.25.mlp.experts.gate_up_proj",
"groups.6.linear.1.mlp.down_proj": "model.language_model.layers.25.mlp.experts.down_proj",
"groups.6.linear.1.mlp.shared_expert.gate.weight": "model.language_model.layers.25.mlp.shared_expert.gate_proj.weight",
"groups.6.linear.1.mlp.shared_expert.up.weight": "model.language_model.layers.25.mlp.shared_expert.up_proj.weight",
"groups.6.linear.1.mlp.shared_expert.down.weight": "model.language_model.layers.25.mlp.shared_expert.down_proj.weight",
"groups.6.linear.1.mlp.shared_expert_gate.weight": "model.language_model.layers.25.mlp.shared_expert_gate.weight",
"groups.6.linear.1.linear_attn.in_proj_qkv.weight": "model.language_model.layers.25.linear_attn.in_proj_qkv.weight",
"groups.6.linear.1.linear_attn.in_proj_z.weight": "model.language_model.layers.25.linear_attn.in_proj_z.weight",
"groups.6.linear.1.linear_attn.in_proj_b.weight": "model.language_model.layers.25.linear_attn.in_proj_b.weight",
"groups.6.linear.1.linear_attn.in_proj_a.weight": "model.language_model.layers.25.linear_attn.in_proj_a.weight",
"groups.6.linear.1.linear_attn.out_proj.weight": "model.language_model.layers.25.linear_attn.out_proj.weight",
"groups.6.linear.1.linear_attn.norm.weight": "model.language_model.layers.25.linear_attn.norm.weight",
"groups.6.linear.1.linear_attn.conv_weight": "model.language_model.layers.25.linear_attn.conv1d.weight",
"groups.6.linear.1.linear_attn.A_log": "model.language_model.layers.25.linear_attn.A_log",
"groups.6.linear.1.linear_attn.dt_bias": "model.language_model.layers.25.linear_attn.dt_bias",
"groups.6.linear.2.input_layernorm.weight": "model.language_model.layers.26.input_layernorm.weight",
"groups.6.linear.2.post_attention_layernorm.weight": "model.language_model.layers.26.post_attention_layernorm.weight",
"groups.6.linear.2.mlp.gate.weight": "model.language_model.layers.26.mlp.gate.weight",
"groups.6.linear.2.mlp.gate_up_proj": "model.language_model.layers.26.mlp.experts.gate_up_proj",
"groups.6.linear.2.mlp.down_proj": "model.language_model.layers.26.mlp.experts.down_proj",
"groups.6.linear.2.mlp.shared_expert.gate.weight": "model.language_model.layers.26.mlp.shared_expert.gate_proj.weight",
"groups.6.linear.2.mlp.shared_expert.up.weight": "model.language_model.layers.26.mlp.shared_expert.up_proj.weight",
"groups.6.linear.2.mlp.shared_expert.down.weight": "model.language_model.layers.26.mlp.shared_expert.down_proj.weight",
"groups.6.linear.2.mlp.shared_expert_gate.weight": "model.language_model.layers.26.mlp.shared_expert_gate.weight",
"groups.6.linear.2.linear_attn.in_proj_qkv.weight": "model.language_model.layers.26.linear_attn.in_proj_qkv.weight",
"groups.6.linear.2.linear_attn.in_proj_z.weight": "model.language_model.layers.26.linear_attn.in_proj_z.weight",
"groups.6.linear.2.linear_attn.in_proj_b.weight": "model.language_model.layers.26.linear_attn.in_proj_b.weight",
"groups.6.linear.2.linear_attn.in_proj_a.weight": "model.language_model.layers.26.linear_attn.in_proj_a.weight",
"groups.6.linear.2.linear_attn.out_proj.weight": "model.language_model.layers.26.linear_attn.out_proj.weight",
"groups.6.linear.2.linear_attn.norm.weight": "model.language_model.layers.26.linear_attn.norm.weight",
"groups.6.linear.2.linear_attn.conv_weight": "model.language_model.layers.26.linear_attn.conv1d.weight",
"groups.6.linear.2.linear_attn.A_log": "model.language_model.layers.26.linear_attn.A_log",
"groups.6.linear.2.linear_attn.dt_bias": "model.language_model.layers.26.linear_attn.dt_bias",
"groups.6.full.input_layernorm.weight": "model.language_model.layers.27.input_layernorm.weight",
"groups.6.full.post_attention_layernorm.weight": "model.language_model.layers.27.post_attention_layernorm.weight",
"groups.6.full.mlp.gate.weight": "model.language_model.layers.27.mlp.gate.weight",
"groups.6.full.mlp.gate_up_proj": "model.language_model.layers.27.mlp.experts.gate_up_proj",
"groups.6.full.mlp.down_proj": "model.language_model.layers.27.mlp.experts.down_proj",
"groups.6.full.mlp.shared_expert.gate.weight": "model.language_model.layers.27.mlp.shared_expert.gate_proj.weight",
"groups.6.full.mlp.shared_expert.up.weight": "model.language_model.layers.27.mlp.shared_expert.up_proj.weight",
"groups.6.full.mlp.shared_expert.down.weight": "model.language_model.layers.27.mlp.shared_expert.down_proj.weight",
"groups.6.full.mlp.shared_expert_gate.weight": "model.language_model.layers.27.mlp.shared_expert_gate.weight",
"groups.6.full.self_attn.q_proj.weight": "model.language_model.layers.27.self_attn.q_proj.weight",
"groups.6.full.self_attn.k_proj.weight": "model.language_model.layers.27.self_attn.k_proj.weight",
"groups.6.full.self_attn.v_proj.weight": "model.language_model.layers.27.self_attn.v_proj.weight",
"groups.6.full.self_attn.o_proj.weight": "model.language_model.layers.27.self_attn.o_proj.weight",
"groups.6.full.self_attn.q_norm.weight": "model.language_model.layers.27.self_attn.q_norm.weight",
"groups.6.full.self_attn.k_norm.weight": "model.language_model.layers.27.self_attn.k_norm.weight",
"groups.7.linear.0.input_layernorm.weight": "model.language_model.layers.28.input_layernorm.weight",
"groups.7.linear.0.post_attention_layernorm.weight": "model.language_model.layers.28.post_attention_layernorm.weight",
"groups.7.linear.0.mlp.gate.weight": "model.language_model.layers.28.mlp.gate.weight",
"groups.7.linear.0.mlp.gate_up_proj": "model.language_model.layers.28.mlp.experts.gate_up_proj",
"groups.7.linear.0.mlp.down_proj": "model.language_model.layers.28.mlp.experts.down_proj",
"groups.7.linear.0.mlp.shared_expert.gate.weight": "model.language_model.layers.28.mlp.shared_expert.gate_proj.weight",
"groups.7.linear.0.mlp.shared_expert.up.weight": "model.language_model.layers.28.mlp.shared_expert.up_proj.weight",
"groups.7.linear.0.mlp.shared_expert.down.weight": "model.language_model.layers.28.mlp.shared_expert.down_proj.weight",
"groups.7.linear.0.mlp.shared_expert_gate.weight": "model.language_model.layers.28.mlp.shared_expert_gate.weight",
"groups.7.linear.0.linear_attn.in_proj_qkv.weight": "model.language_model.layers.28.linear_attn.in_proj_qkv.weight",
"groups.7.linear.0.linear_attn.in_proj_z.weight": "model.language_model.layers.28.linear_attn.in_proj_z.weight",
"groups.7.linear.0.linear_attn.in_proj_b.weight": "model.language_model.layers.28.linear_attn.in_proj_b.weight",
"groups.7.linear.0.linear_attn.in_proj_a.weight": "model.language_model.layers.28.linear_attn.in_proj_a.weight",
"groups.7.linear.0.linear_attn.out_proj.weight": "model.language_model.layers.28.linear_attn.out_proj.weight",
"groups.7.linear.0.linear_attn.norm.weight": "model.language_model.layers.28.linear_attn.norm.weight",
"groups.7.linear.0.linear_attn.conv_weight": "model.language_model.layers.28.linear_attn.conv1d.weight",
"groups.7.linear.0.linear_attn.A_log": "model.language_model.layers.28.linear_attn.A_log",
"groups.7.linear.0.linear_attn.dt_bias": "model.language_model.layers.28.linear_attn.dt_bias",
"groups.7.linear.1.input_layernorm.weight": "model.language_model.layers.29.input_layernorm.weight",
"groups.7.linear.1.post_attention_layernorm.weight": "model.language_model.layers.29.post_attention_layernorm.weight",
"groups.7.linear.1.mlp.gate.weight": "model.language_model.layers.29.mlp.gate.weight",
"groups.7.linear.1.mlp.gate_up_proj": "model.language_model.layers.29.mlp.experts.gate_up_proj",
"groups.7.linear.1.mlp.down_proj": "model.language_model.layers.29.mlp.experts.down_proj",
"groups.7.linear.1.mlp.shared_expert.gate.weight": "model.language_model.layers.29.mlp.shared_expert.gate_proj.weight",
"groups.7.linear.1.mlp.shared_expert.up.weight": "model.language_model.layers.29.mlp.shared_expert.up_proj.weight",
"groups.7.linear.1.mlp.shared_expert.down.weight": "model.language_model.layers.29.mlp.shared_expert.down_proj.weight",
"groups.7.linear.1.mlp.shared_expert_gate.weight": "model.language_model.layers.29.mlp.shared_expert_gate.weight",
"groups.7.linear.1.linear_attn.in_proj_qkv.weight": "model.language_model.layers.29.linear_attn.in_proj_qkv.weight",
"groups.7.linear.1.linear_attn.in_proj_z.weight": "model.language_model.layers.29.linear_attn.in_proj_z.weight",
"groups.7.linear.1.linear_attn.in_proj_b.weight": "model.language_model.layers.29.linear_attn.in_proj_b.weight",
"groups.7.linear.1.linear_attn.in_proj_a.weight": "model.language_model.layers.29.linear_attn.in_proj_a.weight",
"groups.7.linear.1.linear_attn.out_proj.weight": "model.language_model.layers.29.linear_attn.out_proj.weight",
"groups.7.linear.1.linear_attn.norm.weight": "model.language_model.layers.29.linear_attn.norm.weight",
"groups.7.linear.1.linear_attn.conv_weight": "model.language_model.layers.29.linear_attn.conv1d.weight",
"groups.7.linear.1.linear_attn.A_log": "model.language_model.layers.29.linear_attn.A_log",
"groups.7.linear.1.linear_attn.dt_bias": "model.language_model.layers.29.linear_attn.dt_bias",
"groups.7.linear.2.input_layernorm.weight": "model.language_model.layers.30.input_layernorm.weight",
"groups.7.linear.2.post_attention_layernorm.weight": "model.language_model.layers.30.post_attention_layernorm.weight",
"groups.7.linear.2.mlp.gate.weight": "model.language_model.layers.30.mlp.gate.weight",
"groups.7.linear.2.mlp.gate_up_proj": "model.language_model.layers.30.mlp.experts.gate_up_proj",
"groups.7.linear.2.mlp.down_proj": "model.language_model.layers.30.mlp.experts.down_proj",
"groups.7.linear.2.mlp.shared_expert.gate.weight": "model.language_model.layers.30.mlp.shared_expert.gate_proj.weight",
"groups.7.linear.2.mlp.shared_expert.up.weight": "model.language_model.layers.30.mlp.shared_expert.up_proj.weight",
"groups.7.linear.2.mlp.shared_expert.down.weight": "model.language_model.layers.30.mlp.shared_expert.down_proj.weight",
"groups.7.linear.2.mlp.shared_expert_gate.weight": "model.language_model.layers.30.mlp.shared_expert_gate.weight",
"groups.7.linear.2.linear_attn.in_proj_qkv.weight": "model.language_model.layers.30.linear_attn.in_proj_qkv.weight",
"groups.7.linear.2.linear_attn.in_proj_z.weight": "model.language_model.layers.30.linear_attn.in_proj_z.weight",
"groups.7.linear.2.linear_attn.in_proj_b.weight": "model.language_model.layers.30.linear_attn.in_proj_b.weight",
"groups.7.linear.2.linear_attn.in_proj_a.weight": "model.language_model.layers.30.linear_attn.in_proj_a.weight",
"groups.7.linear.2.linear_attn.out_proj.weight": "model.language_model.layers.30.linear_attn.out_proj.weight",
"groups.7.linear.2.linear_attn.norm.weight": "model.language_model.layers.30.linear_attn.norm.weight",
"groups.7.linear.2.linear_attn.conv_weight": "model.language_model.layers.30.linear_attn.conv1d.weight",
"groups.7.linear.2.linear_attn.A_log": "model.language_model.layers.30.linear_attn.A_log",
"groups.7.linear.2.linear_attn.dt_bias": "model.language_model.layers.30.linear_attn.dt_bias",
"groups.7.full.input_layernorm.weight": "model.language_model.layers.31.input_layernorm.weight",
"groups.7.full.post_attention_layernorm.weight": "model.language_model.layers.31.post_attention_layernorm.weight",
"groups.7.full.mlp.gate.weight": "model.language_model.layers.31.mlp.gate.weight",
"groups.7.full.mlp.gate_up_proj": "model.language_model.layers.31.mlp.experts.gate_up_proj",
"groups.7.full.mlp.down_proj": "model.language_model.layers.31.mlp.experts.down_proj",
"groups.7.full.mlp.shared_expert.gate.weight": "model.language_model.layers.31.mlp.shared_expert.gate_proj.weight",
"groups.7.full.mlp.shared_expert.up.weight": "model.language_model.layers.31.mlp.shared_expert.up_proj.weight",
"groups.7.full.mlp.shared_expert.down.weight": "model.language_model.layers.31.mlp.shared_expert.down_proj.weight",
"groups.7.full.mlp.shared_expert_gate.weight": "model.language_model.layers.31.mlp.shared_expert_gate.weight",
"groups.7.full.self_attn.q_proj.weight": "model.language_model.layers.31.self_attn.q_proj.weight",
"groups.7.full.self_attn.k_proj.weight": "model.language_model.layers.31.self_attn.k_proj.weight",
"groups.7.full.self_attn.v_proj.weight": "model.language_model.layers.31.self_attn.v_proj.weight",
"groups.7.full.self_attn.o_proj.weight": "model.language_model.layers.31.self_attn.o_proj.weight",
"groups.7.full.self_attn.q_norm.weight": "model.language_model.layers.31.self_attn.q_norm.weight",
"groups.7.full.self_attn.k_norm.weight": "model.language_model.layers.31.self_attn.k_norm.weight",
"groups.8.linear.0.input_layernorm.weight": "model.language_model.layers.32.input_layernorm.weight",
"groups.8.linear.0.post_attention_layernorm.weight": "model.language_model.layers.32.post_attention_layernorm.weight",
"groups.8.linear.0.mlp.gate.weight": "model.language_model.layers.32.mlp.gate.weight",
"groups.8.linear.0.mlp.gate_up_proj": "model.language_model.layers.32.mlp.experts.gate_up_proj",
"groups.8.linear.0.mlp.down_proj": "model.language_model.layers.32.mlp.experts.down_proj",
"groups.8.linear.0.mlp.shared_expert.gate.weight": "model.language_model.layers.32.mlp.shared_expert.gate_proj.weight",
"groups.8.linear.0.mlp.shared_expert.up.weight": "model.language_model.layers.32.mlp.shared_expert.up_proj.weight",
"groups.8.linear.0.mlp.shared_expert.down.weight": "model.language_model.layers.32.mlp.shared_expert.down_proj.weight",
"groups.8.linear.0.mlp.shared_expert_gate.weight": "model.language_model.layers.32.mlp.shared_expert_gate.weight",
"groups.8.linear.0.linear_attn.in_proj_qkv.weight": "model.language_model.layers.32.linear_attn.in_proj_qkv.weight",
"groups.8.linear.0.linear_attn.in_proj_z.weight": "model.language_model.layers.32.linear_attn.in_proj_z.weight",
"groups.8.linear.0.linear_attn.in_proj_b.weight": "model.language_model.layers.32.linear_attn.in_proj_b.weight",
"groups.8.linear.0.linear_attn.in_proj_a.weight": "model.language_model.layers.32.linear_attn.in_proj_a.weight",
"groups.8.linear.0.linear_attn.out_proj.weight": "model.language_model.layers.32.linear_attn.out_proj.weight",
"groups.8.linear.0.linear_attn.norm.weight": "model.language_model.layers.32.linear_attn.norm.weight",
"groups.8.linear.0.linear_attn.conv_weight": "model.language_model.layers.32.linear_attn.conv1d.weight",
"groups.8.linear.0.linear_attn.A_log": "model.language_model.layers.32.linear_attn.A_log",
"groups.8.linear.0.linear_attn.dt_bias": "model.language_model.layers.32.linear_attn.dt_bias",
"groups.8.linear.1.input_layernorm.weight": "model.language_model.layers.33.input_layernorm.weight",
"groups.8.linear.1.post_attention_layernorm.weight": "model.language_model.layers.33.post_attention_layernorm.weight",
"groups.8.linear.1.mlp.gate.weight": "model.language_model.layers.33.mlp.gate.weight",
"groups.8.linear.1.mlp.gate_up_proj": "model.language_model.layers.33.mlp.experts.gate_up_proj",
"groups.8.linear.1.mlp.down_proj": "model.language_model.layers.33.mlp.experts.down_proj",
"groups.8.linear.1.mlp.shared_expert.gate.weight": "model.language_model.layers.33.mlp.shared_expert.gate_proj.weight",
"groups.8.linear.1.mlp.shared_expert.up.weight": "model.language_model.layers.33.mlp.shared_expert.up_proj.weight",
"groups.8.linear.1.mlp.shared_expert.down.weight": "model.language_model.layers.33.mlp.shared_expert.down_proj.weight",
"groups.8.linear.1.mlp.shared_expert_gate.weight": "model.language_model.layers.33.mlp.shared_expert_gate.weight",
"groups.8.linear.1.linear_attn.in_proj_qkv.weight": "model.language_model.layers.33.linear_attn.in_proj_qkv.weight",
"groups.8.linear.1.linear_attn.in_proj_z.weight": "model.language_model.layers.33.linear_attn.in_proj_z.weight",
"groups.8.linear.1.linear_attn.in_proj_b.weight": "model.language_model.layers.33.linear_attn.in_proj_b.weight",
"groups.8.linear.1.linear_attn.in_proj_a.weight": "model.language_model.layers.33.linear_attn.in_proj_a.weight",
"groups.8.linear.1.linear_attn.out_proj.weight": "model.language_model.layers.33.linear_attn.out_proj.weight",
"groups.8.linear.1.linear_attn.norm.weight": "model.language_model.layers.33.linear_attn.norm.weight",
"groups.8.linear.1.linear_attn.conv_weight": "model.language_model.layers.33.linear_attn.conv1d.weight",
"groups.8.linear.1.linear_attn.A_log": "model.language_model.layers.33.linear_attn.A_log",
"groups.8.linear.1.linear_attn.dt_bias": "model.language_model.layers.33.linear_attn.dt_bias",
"groups.8.linear.2.input_layernorm.weight": "model.language_model.layers.34.input_layernorm.weight",
"groups.8.linear.2.post_attention_layernorm.weight": "model.language_model.layers.34.post_attention_layernorm.weight",
"groups.8.linear.2.mlp.gate.weight": "model.language_model.layers.34.mlp.gate.weight",
"groups.8.linear.2.mlp.gate_up_proj": "model.language_model.layers.34.mlp.experts.gate_up_proj",
"groups.8.linear.2.mlp.down_proj": "model.language_model.layers.34.mlp.experts.down_proj",
"groups.8.linear.2.mlp.shared_expert.gate.weight": "model.language_model.layers.34.mlp.shared_expert.gate_proj.weight",
"groups.8.linear.2.mlp.shared_expert.up.weight": "model.language_model.layers.34.mlp.shared_expert.up_proj.weight",
"groups.8.linear.2.mlp.shared_expert.down.weight": "model.language_model.layers.34.mlp.shared_expert.down_proj.weight",
"groups.8.linear.2.mlp.shared_expert_gate.weight": "model.language_model.layers.34.mlp.shared_expert_gate.weight",
"groups.8.linear.2.linear_attn.in_proj_qkv.weight": "model.language_model.layers.34.linear_attn.in_proj_qkv.weight",
"groups.8.linear.2.linear_attn.in_proj_z.weight": "model.language_model.layers.34.linear_attn.in_proj_z.weight",
"groups.8.linear.2.linear_attn.in_proj_b.weight": "model.language_model.layers.34.linear_attn.in_proj_b.weight",
"groups.8.linear.2.linear_attn.in_proj_a.weight": "model.language_model.layers.34.linear_attn.in_proj_a.weight",
"groups.8.linear.2.linear_attn.out_proj.weight": "model.language_model.layers.34.linear_attn.out_proj.weight",
"groups.8.linear.2.linear_attn.norm.weight": "model.language_model.layers.34.linear_attn.norm.weight",
"groups.8.linear.2.linear_attn.conv_weight": "model.language_model.layers.34.linear_attn.conv1d.weight",
"groups.8.linear.2.linear_attn.A_log": "model.language_model.layers.34.linear_attn.A_log",
"groups.8.linear.2.linear_attn.dt_bias": "model.language_model.layers.34.linear_attn.dt_bias",
"groups.8.full.input_layernorm.weight": "model.language_model.layers.35.input_layernorm.weight",
"groups.8.full.post_attention_layernorm.weight": "model.language_model.layers.35.post_attention_layernorm.weight",
"groups.8.full.mlp.gate.weight": "model.language_model.layers.35.mlp.gate.weight",
"groups.8.full.mlp.gate_up_proj": "model.language_model.layers.35.mlp.experts.gate_up_proj",
"groups.8.full.mlp.down_proj": "model.language_model.layers.35.mlp.experts.down_proj",
"groups.8.full.mlp.shared_expert.gate.weight": "model.language_model.layers.35.mlp.shared_expert.gate_proj.weight",
"groups.8.full.mlp.shared_expert.up.weight": "model.language_model.layers.35.mlp.shared_expert.up_proj.weight",
"groups.8.full.mlp.shared_expert.down.weight": "model.language_model.layers.35.mlp.shared_expert.down_proj.weight",
"groups.8.full.mlp.shared_expert_gate.weight": "model.language_model.layers.35.mlp.shared_expert_gate.weight",
"groups.8.full.self_attn.q_proj.weight": "model.language_model.layers.35.self_attn.q_proj.weight",
"groups.8.full.self_attn.k_proj.weight": "model.language_model.layers.35.self_attn.k_proj.weight",
"groups.8.full.self_attn.v_proj.weight": "model.language_model.layers.35.self_attn.v_proj.weight",
"groups.8.full.self_attn.o_proj.weight": "model.language_model.layers.35.self_attn.o_proj.weight",
"groups.8.full.self_attn.q_norm.weight": "model.language_model.layers.35.self_attn.q_norm.weight",
"groups.8.full.self_attn.k_norm.weight": "model.language_model.layers.35.self_attn.k_norm.weight",
"groups.9.linear.0.input_layernorm.weight": "model.language_model.layers.36.input_layernorm.weight",
"groups.9.linear.0.post_attention_layernorm.weight": "model.language_model.layers.36.post_attention_layernorm.weight",
"groups.9.linear.0.mlp.gate.weight": "model.language_model.layers.36.mlp.gate.weight",
"groups.9.linear.0.mlp.gate_up_proj": "model.language_model.layers.36.mlp.experts.gate_up_proj",
"groups.9.linear.0.mlp.down_proj": "model.language_model.layers.36.mlp.experts.down_proj",
"groups.9.linear.0.mlp.shared_expert.gate.weight": "model.language_model.layers.36.mlp.shared_expert.gate_proj.weight",
"groups.9.linear.0.mlp.shared_expert.up.weight": "model.language_model.layers.36.mlp.shared_expert.up_proj.weight",
"groups.9.linear.0.mlp.shared_expert.down.weight": "model.language_model.layers.36.mlp.shared_expert.down_proj.weight",
"groups.9.linear.0.mlp.shared_expert_gate.weight": "model.language_model.layers.36.mlp.shared_expert_gate.weight",
"groups.9.linear.0.linear_attn.in_proj_qkv.weight": "model.language_model.layers.36.linear_attn.in_proj_qkv.weight",
"groups.9.linear.0.linear_attn.in_proj_z.weight": "model.language_model.layers.36.linear_attn.in_proj_z.weight",
"groups.9.linear.0.linear_attn.in_proj_b.weight": "model.language_model.layers.36.linear_attn.in_proj_b.weight",
"groups.9.linear.0.linear_attn.in_proj_a.weight": "model.language_model.layers.36.linear_attn.in_proj_a.weight",
"groups.9.linear.0.linear_attn.out_proj.weight": "model.language_model.layers.36.linear_attn.out_proj.weight",
"groups.9.linear.0.linear_attn.norm.weight": "model.language_model.layers.36.linear_attn.norm.weight",
"groups.9.linear.0.linear_attn.conv_weight": "model.language_model.layers.36.linear_attn.conv1d.weight",
"groups.9.linear.0.linear_attn.A_log": "model.language_model.layers.36.linear_attn.A_log",
"groups.9.linear.0.linear_attn.dt_bias": "model.language_model.layers.36.linear_attn.dt_bias",
"groups.9.linear.1.input_layernorm.weight": "model.language_model.layers.37.input_layernorm.weight",
"groups.9.linear.1.post_attention_layernorm.weight": "model.language_model.layers.37.post_attention_layernorm.weight",
"groups.9.linear.1.mlp.gate.weight": "model.language_model.layers.37.mlp.gate.weight",
"groups.9.linear.1.mlp.gate_up_proj": "model.language_model.layers.37.mlp.experts.gate_up_proj",
"groups.9.linear.1.mlp.down_proj": "model.language_model.layers.37.mlp.experts.down_proj",
"groups.9.linear.1.mlp.shared_expert.gate.weight": "model.language_model.layers.37.mlp.shared_expert.gate_proj.weight",
"groups.9.linear.1.mlp.shared_expert.up.weight": "model.language_model.layers.37.mlp.shared_expert.up_proj.weight",
"groups.9.linear.1.mlp.shared_expert.down.weight": "model.language_model.layers.37.mlp.shared_expert.down_proj.weight",
"groups.9.linear.1.mlp.shared_expert_gate.weight": "model.language_model.layers.37.mlp.shared_expert_gate.weight",
"groups.9.linear.1.linear_attn.in_proj_qkv.weight": "model.language_model.layers.37.linear_attn.in_proj_qkv.weight",
"groups.9.linear.1.linear_attn.in_proj_z.weight": "model.language_model.layers.37.linear_attn.in_proj_z.weight",
"groups.9.linear.1.linear_attn.in_proj_b.weight": "model.language_model.layers.37.linear_attn.in_proj_b.weight",
"groups.9.linear.1.linear_attn.in_proj_a.weight": "model.language_model.layers.37.linear_attn.in_proj_a.weight",
"groups.9.linear.1.linear_attn.out_proj.weight": "model.language_model.layers.37.linear_attn.out_proj.weight",
"groups.9.linear.1.linear_attn.norm.weight": "model.language_model.layers.37.linear_attn.norm.weight",
"groups.9.linear.1.linear_attn.conv_weight": "model.language_model.layers.37.linear_attn.conv1d.weight",
"groups.9.linear.1.linear_attn.A_log": "model.language_model.layers.37.linear_attn.A_log",
"groups.9.linear.1.linear_attn.dt_bias": "model.language_model.layers.37.linear_attn.dt_bias",
"groups.9.linear.2.input_layernorm.weight": "model.language_model.layers.38.input_layernorm.weight",
"groups.9.linear.2.post_attention_layernorm.weight": "model.language_model.layers.38.post_attention_layernorm.weight",
"groups.9.linear.2.mlp.gate.weight": "model.language_model.layers.38.mlp.gate.weight",
"groups.9.linear.2.mlp.gate_up_proj": "model.language_model.layers.38.mlp.experts.gate_up_proj",
"groups.9.linear.2.mlp.down_proj": "model.language_model.layers.38.mlp.experts.down_proj",
"groups.9.linear.2.mlp.shared_expert.gate.weight": "model.language_model.layers.38.mlp.shared_expert.gate_proj.weight",
"groups.9.linear.2.mlp.shared_expert.up.weight": "model.language_model.layers.38.mlp.shared_expert.up_proj.weight",
"groups.9.linear.2.mlp.shared_expert.down.weight": "model.language_model.layers.38.mlp.shared_expert.down_proj.weight",
"groups.9.linear.2.mlp.shared_expert_gate.weight": "model.language_model.layers.38.mlp.shared_expert_gate.weight",
"groups.9.linear.2.linear_attn.in_proj_qkv.weight": "model.language_model.layers.38.linear_attn.in_proj_qkv.weight",
"groups.9.linear.2.linear_attn.in_proj_z.weight": "model.language_model.layers.38.linear_attn.in_proj_z.weight",
"groups.9.linear.2.linear_attn.in_proj_b.weight": "model.language_model.layers.38.linear_attn.in_proj_b.weight",
"groups.9.linear.2.linear_attn.in_proj_a.weight": "model.language_model.layers.38.linear_attn.in_proj_a.weight",
"groups.9.linear.2.linear_attn.out_proj.weight": "model.language_model.layers.38.linear_attn.out_proj.weight",
"groups.9.linear.2.linear_attn.norm.weight": "model.language_model.layers.38.linear_attn.norm.weight",
"groups.9.linear.2.linear_attn.conv_weight": "model.language_model.layers.38.linear_attn.conv1d.weight",
"groups.9.linear.2.linear_attn.A_log": "model.language_model.layers.38.linear_attn.A_log",
"groups.9.linear.2.linear_attn.dt_bias": "model.language_model.layers.38.linear_attn.dt_bias",
"groups.9.full.input_layernorm.weight": "model.language_model.layers.39.input_layernorm.weight",
"groups.9.full.post_attention_layernorm.weight": "model.language_model.layers.39.post_attention_layernorm.weight",
"groups.9.full.mlp.gate.weight": "model.language_model.layers.39.mlp.gate.weight",
"groups.9.full.mlp.gate_up_proj": "model.language_model.layers.39.mlp.experts.gate_up_proj",
"groups.9.full.mlp.down_proj": "model.language_model.layers.39.mlp.experts.down_proj",
"groups.9.full.mlp.shared_expert.gate.weight": "model.language_model.layers.39.mlp.shared_expert.gate_proj.weight",
"groups.9.full.mlp.shared_expert.up.weight": "model.language_model.layers.39.mlp.shared_expert.up_proj.weight",
"groups.9.full.mlp.shared_expert.down.weight": "model.language_model.layers.39.mlp.shared_expert.down_proj.weight",
"groups.9.full.mlp.shared_expert_gate.weight": "model.language_model.layers.39.mlp.shared_expert_gate.weight",
"groups.9.full.self_attn.q_proj.weight": "model.language_model.layers.39.self_attn.q_proj.weight",
"groups.9.full.self_attn.k_proj.weight": "model.language_model.layers.39.self_attn.k_proj.weight",
"groups.9.full.self_attn.v_proj.weight": "model.language_model.layers.39.self_attn.v_proj.weight",
"groups.9.full.self_attn.o_proj.weight": "model.language_model.layers.39.self_attn.o_proj.weight",
"groups.9.full.self_attn.q_norm.weight": "model.language_model.layers.39.self_attn.q_norm.weight",
"groups.9.full.self_attn.k_norm.weight": "model.language_model.layers.39.self_attn.k_norm.weight"
}linnet.toml66 B
toml
[package]
name = "qwen3_5_moe"
version = "0.1.0"
language = "0.1"nest.toml2.1 kB
toml
[model]
name = "qwen3.6-35b-a3b"
title = "Qwen3.6 35B-A3B"
summary = "A 35B-parameter hybrid mixture of experts (3B active): three Gated DeltaNet (linear attention) layers for every gated full-attention layer, 256 experts with 8 chosen per token plus a gated shared expert, and an untied output head; the text model of Qwen3.6-35B-A3B, without its vision tower."
license = "Apache-2.0"
family = "qwen3_5_moe"
tags = ["text-generation", "decoder-only", "mixture-of-experts", "linear-attention", "gated-deltanet", "hybrid"]
[links]
huggingface = "https://huggingface.co/Qwen/Qwen3.6-35B-A3B"
github = "https://github.com/QwenLM/Qwen3.5"
[source]
path = "src/lib.linnet"
root = "Model"
entry = "forward"
[generics]
Vocab = 248320
H = 2048
Heads = 16
KvHeads = 2
HeadDim = 256
KeyHeads = 16
ValueHeads = 32
KeyDim = 128
ValueDim = 128
Conv = 8192
Width = 4
Experts = 256
TopK = 8
ExpertInner = 512
SharedInner = 512
Groups = 10
Batch = 1
MaxSeq = 262144
T = "bf16"
[check]
B = 1
S = 8
[weights]
repo = "Qwen/Qwen3.6-35B-A3B"
revision = "995ad96eacd98c81ed38be0c5b274b04031597b0"
files = [
"model-00001-of-00026.safetensors",
"model-00002-of-00026.safetensors",
"model-00003-of-00026.safetensors",
"model-00004-of-00026.safetensors",
"model-00005-of-00026.safetensors",
"model-00006-of-00026.safetensors",
"model-00007-of-00026.safetensors",
"model-00008-of-00026.safetensors",
"model-00009-of-00026.safetensors",
"model-00010-of-00026.safetensors",
"model-00011-of-00026.safetensors",
"model-00012-of-00026.safetensors",
"model-00013-of-00026.safetensors",
"model-00014-of-00026.safetensors",
"model-00015-of-00026.safetensors",
"model-00016-of-00026.safetensors",
"model-00017-of-00026.safetensors",
"model-00018-of-00026.safetensors",
"model-00019-of-00026.safetensors",
"model-00020-of-00026.safetensors",
"model-00021-of-00026.safetensors",
"model-00022-of-00026.safetensors",
"model-00023-of-00026.safetensors",
"model-00024-of-00026.safetensors",
"model-00025-of-00026.safetensors",
"model-00026-of-00026.safetensors",
]
bindings = "bindings.json"src
attention.linnet8.1 kB
linnet
// Qwen3.5's full-attention layers (every fourth): grouped-query attention
// with a per-head RMS norm on the queries and keys, rotary positions on the
// first quarter of each head, and an output gate. `q_proj` gives each head
// its query and its gate side by side (`[Heads, 2 * HeadDim]`); the heads'
// outputs are scaled by `sigmoid(gate)` before `o_proj`.
//
// The caches are the usual `[Batch, KvHeads, MaxSeq, HeadDim]` pair:
// `decode`/`prefill` for one batch at one position, `decode_rows` and
// `prefill_slots` for a server's rows each at its own length.
module qwen3_5_moe.attention
use crate.norm::{RmsNorm}
use crate.rope::{THETA, tables}
use std.nn.activations::{sigmoid}
use std.nn.attention::{causal_mask, grouped_attention, grouped_attention_rows}
use std.nn.cache::{write_at, write_rows, write_slots, write_span}
use std.nn.linear::{Linear}
use std.nn.rope::{rope, rope_rows}
pub block GatedAttention<
H: Dim,
Heads: Dim,
KvHeads: Dim,
HeadDim: Dim,
Batch: Dim,
MaxSeq: Dim,
T: Float,
>
where
Heads > 0,
KvHeads > 0,
Heads % KvHeads == 0,
(HeadDim / 4) % 2 == 0,
MaxSeq > 0,
Batch > 0
{
sub q_proj: Linear<H, Heads * HeadDim * 2, T>
sub k_proj: Linear<H, KvHeads * HeadDim, T>
sub v_proj: Linear<H, KvHeads * HeadDim, T>
sub o_proj: Linear<Heads * HeadDim, H, T>
sub q_norm: RmsNorm<HeadDim, T>
sub k_norm: RmsNorm<HeadDim, T>
state cache_k: Tensor[Batch, KvHeads, MaxSeq, HeadDim; T]
state cache_v: Tensor[Batch, KvHeads, MaxSeq, HeadDim; T]
// A whole sequence, causal, no cache.
pub fn forward<B: Dim, S: Dim>(x: Tensor[B, S, H; T]) -> Tensor[B, S, H; T] {
let (cos_table, sin_table) = tables<S, HeadDim / 4, T>(THETA)
let (q, gate, k, v) = project<B, S>(x)
let mixed = grouped_attention(
rotate<B, Heads, S>(q, cos_table, sin_table),
rotate<B, KvHeads, S>(k, cos_table, sin_table),
v,
rsqrt(cast<f32>(HeadDim)),
some(causal_mask<S, S>()),
)
return finish<B, S>(mixed, gate)
}
// One position `pos` for every row.
pub fn decode(x: Tensor[Batch, 1, H; T], pos: i32) -> Tensor[Batch, 1, H; T] {
let (cos_table, sin_table) = tables<MaxSeq, HeadDim / 4, T>(THETA)
let cos_at[d] = cos_table[cast<i64>(pos), d]
let sin_at[d] = sin_table[cast<i64>(pos), d]
let cos_row = reshape(cos_at, [1, HeadDim / 4])
let sin_row = reshape(sin_at, [1, HeadDim / 4])
let (q, gate, k, v) = project<Batch, 1>(x)
cache_k = write_at(cache_k, rotate<Batch, KvHeads, 1>(k, cos_row, sin_row), pos)
cache_v = write_at(cache_v, v, pos)
let positions = iota<i32>(MaxSeq)
let seen[s] = positions[s] <= pos
let mixed = grouped_attention(
rotate<Batch, Heads, 1>(q, cos_row, sin_row),
cache_k,
cache_v,
rsqrt(cast<f32>(HeadDim)),
some(reshape(seen, [1, MaxSeq])),
)
return finish<Batch, 1>(mixed, gate)
}
// `S` positions from `pos` for every row, as a prompt arrives.
pub fn prefill<S: Dim>(x: Tensor[Batch, S, H; T], pos: i32) -> Tensor[Batch, S, H; T]
where S > 0 {
let (cos_table, sin_table) = tables<MaxSeq, HeadDim / 4, T>(THETA)
let rows[i] = cast<i64>(pos) + iota<i64>(S)[i]
let cos_rows[i, d] = cos_table[rows[i], d]
let sin_rows[i, d] = sin_table[rows[i], d]
let (q, gate, k, v) = project<Batch, S>(x)
cache_k = write_span(cache_k, rotate<Batch, KvHeads, S>(k, cos_rows, sin_rows), pos)
cache_v = write_span(cache_v, v, pos)
let positions = iota<i32>(MaxSeq)
let queries = iota<i32>(S)
let seen[i, s] = positions[s] <= pos + queries[i]
let mixed = grouped_attention(
rotate<Batch, Heads, S>(q, cos_rows, sin_rows),
cache_k,
cache_v,
rsqrt(cast<f32>(HeadDim)),
some(seen),
)
return finish<Batch, S>(mixed, gate)
}
// One position per row, row `b` at `positions[b]`.
pub fn decode_rows(
x: Tensor[Batch, 1, H; T],
positions: Tensor[Batch; i32],
) -> Tensor[Batch, 1, H; T] {
let (cos_table, sin_table) = tables<MaxSeq, HeadDim / 4, T>(THETA)
let cos_at[b, d] = cos_table[cast<i64>(positions[b]), d]
let sin_at[b, d] = sin_table[cast<i64>(positions[b]), d]
let cos_rows = reshape(cos_at, [Batch, 1, HeadDim / 4])
let sin_rows = reshape(sin_at, [Batch, 1, HeadDim / 4])
let (q, gate, k, v) = project<Batch, 1>(x)
cache_k = write_rows(cache_k, rotate_rows<Batch, KvHeads>(k, cos_rows, sin_rows), positions)
cache_v = write_rows(cache_v, v, positions)
let slots = iota<i32>(MaxSeq)
let seen[b, s] = slots[s] <= positions[b]
let mixed = grouped_attention_rows(
rotate_rows<Batch, Heads>(q, cos_rows, sin_rows),
cache_k,
cache_v,
rsqrt(cast<f32>(HeadDim)),
reshape(seen, [Batch, 1, MaxSeq]),
)
return finish<Batch, 1>(mixed, gate)
}
// `M` prompts of `S` tokens from their first positions into rows
// `slots`: each prompt's keys and values fill its row, its queries
// attend causally among themselves.
pub fn prefill_slots<M: Dim, S: Dim>(
x: Tensor[M, S, H; T],
slots: Tensor[M; i32],
) -> Tensor[M, S, H; T]
where
M > 0,
S > 0
{
let (cos_table, sin_table) = tables<S, HeadDim / 4, T>(THETA)
let (q, gate, k, v) = project<M, S>(x)
let rotated = rotate<M, KvHeads, S>(k, cos_table, sin_table)
cache_k = write_slots(cache_k, rotated, slots, 0)
cache_v = write_slots(cache_v, v, slots, 0)
let mixed = grouped_attention(
rotate<M, Heads, S>(q, cos_table, sin_table),
rotated,
v,
rsqrt(cast<f32>(HeadDim)),
some(causal_mask<S, S>()),
)
return finish<M, S>(mixed, gate)
}
// Queries and their gates, keys, values: `[B, Heads, S, HeadDim]`,
// `[B, S, Heads * HeadDim]`, and `[B, KvHeads, S, HeadDim]` twice, the
// queries and keys normalized.
fn project<B: Dim, S: Dim>(
x: Tensor[B, S, H; T],
) -> (Tensor[B, Heads, S, HeadDim; T], Tensor[B, S, Heads *
HeadDim; T], Tensor[B, KvHeads, S, HeadDim; T], Tensor[B, KvHeads, S, HeadDim; T]) {
let both = reshape(q_proj.forward(x), [B, S, Heads, 2 * HeadDim])
let q = q_norm.forward(both[:, :, :, 0:HeadDim])
let gate = reshape(both[:, :, :, HeadDim:2 * HeadDim], [B, S, Heads * HeadDim])
let k = k_norm.forward(reshape(k_proj.forward(x), [B, S, KvHeads, HeadDim]))
let v = reshape(v_proj.forward(x), [B, S, KvHeads, HeadDim])
return (
permute(q, [0, 2, 1, 3]),
gate,
permute(k, [0, 2, 1, 3]),
permute(v, [0, 2, 1, 3]),
)
}
// The heads merged, gated, projected back.
fn finish<B: Dim, S: Dim>(
mixed: Tensor[B, Heads, S, HeadDim; T],
gate: Tensor[B, S, Heads * HeadDim; T],
) -> Tensor[B, S, H; T] {
let merged = reshape(permute(mixed, [0, 2, 1, 3]), [B, S, Heads * HeadDim])
return o_proj.forward(merged * sigmoid(gate))
}
// Rotary positions on each head's first quarter, one table for the batch.
fn rotate<B: Dim, N: Dim, S: Dim>(
x: Tensor[B, N, S, HeadDim; T],
cos_table: Tensor[S, HeadDim / 4; T],
sin_table: Tensor[S, HeadDim / 4; T],
) -> Tensor[B, N, S, HeadDim; T] {
let turned = rope(x[:, :, :, 0:HeadDim / 4], cos_table, sin_table)
return concat(turned, x[:, :, :, HeadDim / 4:HeadDim], axis = -1)
}
// `rotate` with a table row per sequence, one position each.
fn rotate_rows<B: Dim, N: Dim>(
x: Tensor[B, N, 1, HeadDim; T],
cos_rows: Tensor[B, 1, HeadDim / 4; T],
sin_rows: Tensor[B, 1, HeadDim / 4; T],
) -> Tensor[B, N, 1, HeadDim; T] {
let turned = rope_rows(x[:, :, :, 0:HeadDim / 4], cos_rows, sin_rows)
return concat(turned, x[:, :, :, HeadDim / 4:HeadDim], axis = -1)
}
}deltanet.linnet8.8 kB
linnet
// Qwen3.5's linear-attention layers (three of every four): Gated DeltaNet.
//
// One projection gives the queries, keys and values side by side; a
// depthwise causal convolution of width `Width` runs over them, then
// `silu`. The queries and keys are L2-normalized per head, and each value
// head keeps a state of `[KeyDim, ValueDim]` that the gated delta rule
// (`std.nn.recurrent`) forgets by `exp(decay)` and writes at rate `beta` each
// token:
//
// beta = sigmoid(b), decay = -exp(A_log) * softplus(a + dt_bias)
//
// The heads' outputs are normalized and gated by `silu(z)`, then projected
// back. `KeyHeads` query/key heads each serve `ValueHeads / KeyHeads` value
// heads.
//
// Two states per layer, by row: `history`, the convolution's last
// `Width - 1` inputs, and `recurrent`, the delta rule's state in f32. A
// prompt starts both from zero; a decoding step reads and replaces them.
module qwen3_5_moe.deltanet
use crate.norm::{GatedRmsNorm}
use std.nn.activations::{sigmoid, silu}
use std.nn.cache::{write_slots}
use std.nn.linear::{Linear}
use std.nn.recurrent::{causal_conv, gated_delta_rule, gated_delta_step}
pub block GatedDeltaNet<
H: Dim,
KeyHeads: Dim,
ValueHeads: Dim,
KeyDim: Dim,
ValueDim: Dim,
Conv: Dim,
Width: Dim,
Batch: Dim,
T: Float,
>
where
KeyHeads > 0,
KeyDim > 0,
ValueHeads % KeyHeads == 0,
Conv == 2 * KeyHeads * KeyDim + ValueHeads * ValueDim,
Width > 1,
Batch > 0
{
sub in_proj_qkv: Linear<H, Conv, T>
sub in_proj_z: Linear<H, ValueHeads * ValueDim, T>
sub in_proj_b: Linear<H, ValueHeads, T>
sub in_proj_a: Linear<H, ValueHeads, T>
sub norm: GatedRmsNorm<ValueDim, T>
sub out_proj: Linear<ValueHeads * ValueDim, H, T>
param conv_weight: Tensor[Conv, 1, Width; T]
param A_log: Tensor[ValueHeads; T]
param dt_bias: Tensor[ValueHeads; T]
state history: Tensor[Batch, 1, Width - 1, Conv; T]
state recurrent: Tensor[Batch, ValueHeads, KeyDim, ValueDim; f32]
// A whole sequence from zero, no state kept.
pub fn forward<B: Dim, S: Dim>(x: Tensor[B, S, H; T]) -> Tensor[B, S, H; T]
where S > 0 {
let lengths = fill<i32>([B], cast<i32>(S))
let (out, _, _) = span<B, S>(x, zero_history<B>(), zero_state<B>(), lengths)
return out
}
// One token for every row, each continuing its state.
pub fn decode(x: Tensor[Batch, 1, H; T]) -> Tensor[Batch, 1, H; T] {
let mixed = in_proj_qkv.forward(x)
let window = concat(reshape(history, [Batch, Width - 1, Conv]), mixed, axis = 1)
let convolved = causal_conv(mixed, reshape(history, [Batch, Width - 1, Conv]), conv_taps())
history = reshape(window[:, 1:Width, :], [Batch, 1, Width - 1, Conv])
let (q, k, v) = heads<Batch, 1>(cast<T>(silu(convolved)))
let (decay, beta) = gates<Batch, 1>(x)
let stepped = gated_delta_step(
reshape(q, [Batch, KeyHeads, KeyDim]),
reshape(k, [Batch, KeyHeads, KeyDim]),
reshape(v, [Batch, ValueHeads, ValueDim]),
reshape(decay, [Batch, ValueHeads]),
reshape(beta, [Batch, ValueHeads]),
recurrent,
rsqrt(cast<f32>(KeyDim)),
)
recurrent = permute(stepped[:, 1:1 + KeyDim, :, :], [0, 2, 1, 3])
return finish<Batch, 1>(
reshape(cast<T>(stepped[:, 0:1, :, :]), [Batch, 1, ValueHeads, ValueDim]),
x,
)
}
// `S` tokens for every row: from zero at position 0, otherwise
// continuing the rows' states.
pub fn prefill<S: Dim>(x: Tensor[Batch, S, H; T], pos: i32) -> Tensor[Batch, S, H; T]
where S > 0 {
let fresh = pos == 0
let start_history =
if fresh { zero_history<Batch>() } else { reshape(history, [Batch, Width - 1, Conv]) }
let start_state = if fresh { zero_state<Batch>() } else { recurrent }
let lengths = fill<i32>([Batch], cast<i32>(S))
let (out, ends, final) = span<Batch, S>(x, start_history, start_state, lengths)
history = reshape(ends, [Batch, 1, Width - 1, Conv])
recurrent = final
return out
}
// `M` prompts from their first positions into rows `slots`, prompt `m`
// the first `lengths[m]` of its `S` tokens: the padding after it neither
// decays nor writes the state, and the history is its last inputs.
pub fn prefill_slots<M: Dim, S: Dim>(
x: Tensor[M, S, H; T],
slots: Tensor[M; i32],
lengths: Tensor[M; i32],
) -> Tensor[M, S, H; T]
where
M > 0,
S > 0
{
let (out, ends, final) = span<M, S>(x, zero_history<M>(), zero_state<M>(), lengths)
history = write_slots(history, reshape(ends, [M, 1, Width - 1, Conv]), slots, 0)
recurrent = write_slots(recurrent, final, slots, 0)
return out
}
// `S` tokens of `B` sequences from `start_history` and `start_state`,
// sequence `b` the first `lengths[b]`: the outputs, the history after
// each sequence's last token, and its final state.
fn span<B: Dim, S: Dim>(
x: Tensor[B, S, H; T],
start_history: Tensor[B, Width - 1, Conv; T],
start_state: Tensor[B, ValueHeads, KeyDim, ValueDim; f32],
lengths: Tensor[B; i32],
) -> (Tensor[B, S, H; T], Tensor[B, Width -
1, Conv; T], Tensor[B, ValueHeads, KeyDim, ValueDim; f32])
where S > 0 {
let mixed = in_proj_qkv.forward(x)
let convolved = causal_conv(mixed, start_history, conv_taps())
let (q, k, v) = heads<B, S>(cast<T>(silu(convolved)))
let (decay, beta) = gates<B, S>(x)
let steps = iota<i32>(S)
let inside[b, s] = steps[s] < lengths[b]
let kept_decay[b, s, h] = select(inside[b, s], decay[b, s, h], 0.0)
let kept_beta[b, s, h] = select(inside[b, s], beta[b, s, h], 0.0)
let ruled = gated_delta_rule(
q,
k,
v,
kept_decay,
kept_beta,
start_state,
rsqrt(cast<f32>(KeyDim)),
)
let final = permute(ruled[:, S:S + KeyDim, :, :], [0, 2, 1, 3])
// The last `Width - 1` inputs before each sequence's end, the
// history's own where the sequence is shorter.
let padded = concat(start_history, mixed, axis = 1)
let taps = iota<i32>(Width - 1)
let at[b, j] = cast<i64>(lengths[b] + taps[j])
let ends[b, j, c] = padded[b, at[b, j], c]
return (finish<B, S>(cast<T>(ruled[:, 0:S, :, :]), x), ends, final)
}
// The convolved projection split into L2-normalized query and key heads
// and the value heads.
fn heads<B: Dim, S: Dim>(
convolved: Tensor[B, S, Conv; T],
) -> (Tensor[B, S, KeyHeads, KeyDim; T], Tensor[B, S, KeyHeads, KeyDim; T], Tensor[B, S, ValueHeads, ValueDim; T]) {
let q = reshape(convolved[:, :, 0:KeyHeads * KeyDim], [B, S, KeyHeads, KeyDim])
let k = reshape(
convolved[:, :, KeyHeads * KeyDim:2 * KeyHeads * KeyDim],
[B, S, KeyHeads, KeyDim],
)
let v = reshape(convolved[:, :, 2 * KeyHeads * KeyDim:Conv], [B, S, ValueHeads, ValueDim])
return (unit<B, S>(q), unit<B, S>(k), v)
}
// Each head scaled to unit length, in f32.
fn unit<B: Dim, S: Dim>(
x: Tensor[B, S, KeyHeads, KeyDim; T],
) -> Tensor[B, S, KeyHeads, KeyDim; T] {
let xf = cast<f32>(x)
let squares[b, s, h] = sum[d] xf[b, s, h, d] * xf[b, s, h, d]
let scaled[b, s, h, d] = xf[b, s, h, d] * rsqrt(squares[b, s, h] + 1e-6)
return cast<T>(scaled)
}
// Each token's decay (log space) and write rate per value head, in f32.
fn gates<B: Dim, S: Dim>(
x: Tensor[B, S, H; T],
) -> (Tensor[B, S, ValueHeads; f32], Tensor[B, S, ValueHeads; f32]) {
let a = cast<f32>(in_proj_a.forward(x))
let rate = sigmoid(cast<f32>(in_proj_b.forward(x)))
let shifted[b, s, h] = a[b, s, h] + cast<f32>(dt_bias[h])
// softplus, stable for large inputs
let soft[b, s, h] = max(shifted[b, s, h], 0.0) + log(1.0 + exp(-abs(shifted[b, s, h])))
let decay[b, s, h] = -exp(cast<f32>(A_log[h])) * soft[b, s, h]
return (decay, rate)
}
// The heads' outputs normalized, gated by `silu(z)`, projected back.
fn finish<B: Dim, S: Dim>(
out: Tensor[B, S, ValueHeads, ValueDim; T],
x: Tensor[B, S, H; T],
) -> Tensor[B, S, H; T] {
let z = reshape(in_proj_z.forward(x), [B, S, ValueHeads, ValueDim])
return out_proj.forward(reshape(norm.forward(out, z), [B, S, ValueHeads * ValueDim]))
}
fn conv_taps() -> Tensor[Conv, Width; T] {
return reshape(conv_weight, [Conv, Width])
}
fn zero_history<B: Dim>() -> Tensor[B, Width - 1, Conv; T] {
return fill<T>([B, Width - 1, Conv], 0.0)
}
fn zero_state<B: Dim>() -> Tensor[B, ValueHeads, KeyDim, ValueDim; f32] {
return fill<f32>([B, ValueHeads, KeyDim, ValueDim], 0.0)
}
}lib.linnet10.4 kB
linnet
// Qwen3.5-MoE, the text model (Qwen3.6-35B-A3B is this architecture): a hybrid
// decoder whose layers come in fours, three of linear attention (Gated
// DeltaNet, `crate.deltanet`) and then one of gated full attention
// (`crate.attention`), each followed by a mixture of experts (`crate.moe`), with zero-centered
// RMS norms throughout and an untied output head. The checkpoint's vision
// tower and its multi-token-prediction layer are not part of it.
//
// The entries a decoder serves from: `forward` and `next_token` over whole
// sequences, `prefill`/`decode` for a batch at one position, and
// `prefill_slots`/`decode_rows` for a server's rows each at its own length.
// The full-attention layers keep key/value caches; the linear-attention
// layers keep a convolution history and a recurrent state per row.
module qwen3_5_moe
use crate.attention::{GatedAttention}
use crate.deltanet::{GatedDeltaNet}
use crate.norm::{RmsNorm}
use std.nn.embedding::{Embedding}
use std.nn.linear::{Linear}
use crate.moe::{SparseMoe}
pub block DeltaLayer<
H: Dim,
KeyHeads: Dim,
ValueHeads: Dim,
KeyDim: Dim,
ValueDim: Dim,
Conv: Dim,
Width: Dim,
Experts: Dim,
TopK: Dim,
ExpertInner: Dim,
SharedInner: Dim,
Batch: Dim,
T: Float,
>
where
KeyHeads > 0,
KeyDim > 0,
ValueHeads % KeyHeads == 0,
Conv == 2 * KeyHeads * KeyDim + ValueHeads * ValueDim,
Width > 1,
Experts > 0,
TopK > 0,
Batch > 0
{
sub input_layernorm: RmsNorm<H, T>
sub linear_attn: GatedDeltaNet<H, KeyHeads, ValueHeads, KeyDim, ValueDim, Conv, Width, Batch, T>
sub post_attention_layernorm: RmsNorm<H, T>
sub mlp: SparseMoe<H, Experts, TopK, ExpertInner, SharedInner, T>
pub fn forward<B: Dim, S: Dim>(x: Tensor[B, S, H; T]) -> Tensor[B, S, H; T]
where S > 0 {
return feed(x + linear_attn.forward(input_layernorm.forward(x)))
}
pub fn decode(x: Tensor[Batch, 1, H; T]) -> Tensor[Batch, 1, H; T] {
return feed(x + linear_attn.decode(input_layernorm.forward(x)))
}
pub fn prefill<S: Dim>(x: Tensor[Batch, S, H; T], pos: i32) -> Tensor[Batch, S, H; T]
where S > 0 {
return feed(x + linear_attn.prefill(input_layernorm.forward(x), pos))
}
pub fn prefill_slots<M: Dim, S: Dim>(
x: Tensor[M, S, H; T],
slots: Tensor[M; i32],
lengths: Tensor[M; i32],
) -> Tensor[M, S, H; T]
where
M > 0,
S > 0
{
return feed(x + linear_attn.prefill_slots(input_layernorm.forward(x), slots, lengths))
}
fn feed<B: Dim, S: Dim>(x: Tensor[B, S, H; T]) -> Tensor[B, S, H; T] {
return x + mlp.forward(post_attention_layernorm.forward(x))
}
}
pub block AttentionLayer<
H: Dim,
Heads: Dim,
KvHeads: Dim,
HeadDim: Dim,
Experts: Dim,
TopK: Dim,
ExpertInner: Dim,
SharedInner: Dim,
Batch: Dim,
MaxSeq: Dim,
T: Float,
>
where
Heads > 0,
Experts > 0,
TopK > 0,
KvHeads > 0,
Heads % KvHeads == 0,
(HeadDim / 4) % 2 == 0,
MaxSeq > 0,
Batch > 0
{
sub input_layernorm: RmsNorm<H, T>
sub self_attn: GatedAttention<H, Heads, KvHeads, HeadDim, Batch, MaxSeq, T>
sub post_attention_layernorm: RmsNorm<H, T>
sub mlp: SparseMoe<H, Experts, TopK, ExpertInner, SharedInner, T>
pub fn forward<B: Dim, S: Dim>(x: Tensor[B, S, H; T]) -> Tensor[B, S, H; T] {
return feed(x + self_attn.forward(input_layernorm.forward(x)))
}
pub fn decode(x: Tensor[Batch, 1, H; T], pos: i32) -> Tensor[Batch, 1, H; T] {
return feed(x + self_attn.decode(input_layernorm.forward(x), pos))
}
pub fn prefill<S: Dim>(x: Tensor[Batch, S, H; T], pos: i32) -> Tensor[Batch, S, H; T]
where S > 0 {
return feed(x + self_attn.prefill(input_layernorm.forward(x), pos))
}
pub fn decode_rows(
x: Tensor[Batch, 1, H; T],
positions: Tensor[Batch; i32],
) -> Tensor[Batch, 1, H; T] {
return feed(x + self_attn.decode_rows(input_layernorm.forward(x), positions))
}
pub fn prefill_slots<M: Dim, S: Dim>(
x: Tensor[M, S, H; T],
slots: Tensor[M; i32],
) -> Tensor[M, S, H; T]
where
M > 0,
S > 0
{
return feed(x + self_attn.prefill_slots(input_layernorm.forward(x), slots))
}
fn feed<B: Dim, S: Dim>(x: Tensor[B, S, H; T]) -> Tensor[B, S, H; T] {
return x + mlp.forward(post_attention_layernorm.forward(x))
}
}
// Four layers: three of linear attention, then one of full attention
// (`full_attention_interval` 4). A `sub` array holds one block type, so the
// four are the repeating unit.
pub block LayerGroup<
H: Dim,
Heads: Dim,
KvHeads: Dim,
HeadDim: Dim,
KeyHeads: Dim,
ValueHeads: Dim,
KeyDim: Dim,
ValueDim: Dim,
Conv: Dim,
Width: Dim,
Experts: Dim,
TopK: Dim,
ExpertInner: Dim,
SharedInner: Dim,
Batch: Dim,
MaxSeq: Dim,
T: Float,
>
where
Heads > 0,
Experts > 0,
TopK > 0,
KvHeads > 0,
Heads % KvHeads == 0,
(HeadDim / 4) % 2 == 0,
KeyHeads > 0,
KeyDim > 0,
ValueHeads % KeyHeads == 0,
Conv == 2 * KeyHeads * KeyDim + ValueHeads * ValueDim,
Width > 1,
MaxSeq > 0,
Batch > 0
{
sub linear: [DeltaLayer<H, KeyHeads, ValueHeads, KeyDim, ValueDim, Conv, Width, Experts, TopK, ExpertInner, SharedInner, Batch, T>; 3]
sub full: AttentionLayer<H, Heads, KvHeads, HeadDim, Experts, TopK, ExpertInner, SharedInner, Batch, MaxSeq, T>
pub fn forward<B: Dim, S: Dim>(x: Tensor[B, S, H; T]) -> Tensor[B, S, H; T]
where S > 0 {
var y = x
static for layer in linear {
y = layer.forward(y)
}
return full.forward(y)
}
pub fn decode(x: Tensor[Batch, 1, H; T], pos: i32) -> Tensor[Batch, 1, H; T] {
var y = x
static for layer in linear {
y = layer.decode(y)
}
return full.decode(y, pos)
}
pub fn prefill<S: Dim>(x: Tensor[Batch, S, H; T], pos: i32) -> Tensor[Batch, S, H; T]
where S > 0 {
var y = x
static for layer in linear {
y = layer.prefill(y, pos)
}
return full.prefill(y, pos)
}
pub fn decode_rows(
x: Tensor[Batch, 1, H; T],
positions: Tensor[Batch; i32],
) -> Tensor[Batch, 1, H; T] {
var y = x
static for layer in linear {
y = layer.decode(y)
}
return full.decode_rows(y, positions)
}
pub fn prefill_slots<M: Dim, S: Dim>(
x: Tensor[M, S, H; T],
slots: Tensor[M; i32],
lengths: Tensor[M; i32],
) -> Tensor[M, S, H; T]
where
M > 0,
S > 0
{
var y = x
static for layer in linear {
y = layer.prefill_slots(y, slots, lengths)
}
return full.prefill_slots(y, slots)
}
}
pub block Model<
Vocab: Dim,
H: Dim,
Heads: Dim,
KvHeads: Dim,
HeadDim: Dim,
KeyHeads: Dim,
ValueHeads: Dim,
KeyDim: Dim,
ValueDim: Dim,
Conv: Dim,
Width: Dim,
Experts: Dim,
TopK: Dim,
ExpertInner: Dim,
SharedInner: Dim,
Groups: Dim,
Batch: Dim,
MaxSeq: Dim,
T: Float = bf16,
>
where
Heads > 0,
Experts > 0,
TopK > 0,
KvHeads > 0,
Heads % KvHeads == 0,
(HeadDim / 4) % 2 == 0,
KeyHeads > 0,
KeyDim > 0,
ValueHeads % KeyHeads == 0,
Conv == 2 * KeyHeads * KeyDim + ValueHeads * ValueDim,
Width > 1,
MaxSeq > 0,
Batch > 0
{
sub embed_tokens: Embedding<Vocab, H, T>
sub groups: [LayerGroup<H, Heads, KvHeads, HeadDim, KeyHeads, ValueHeads, KeyDim, ValueDim, Conv, Width, Experts, TopK, ExpertInner, SharedInner, Batch, MaxSeq, T>; Groups]
sub norm: RmsNorm<H, T>
sub lm_head: Linear<H, Vocab, T>
// Logits for every position.
pub entry forward<B: Dim, S: Dim>(tokens: Tensor[B, S; i32]) -> Tensor[B, S, Vocab; T]
where S > 0 {
let rows = reshape(norm.forward(hidden_states(tokens)), [B * S, H])
return reshape(lm_head.forward(rows), [B, S, Vocab])
}
// Logits for the last position only.
pub entry next_token<B: Dim, S: Dim>(tokens: Tensor[B, S; i32]) -> Tensor[B, Vocab; T]
where S > 0 {
return logits(hidden_states(tokens)[:, S - 1, :])
}
// One new token per sequence at `pos`, continuing the caches and states.
pub entry decode(token: Tensor[Batch, 1; i32], pos: i32) -> Tensor[Batch, Vocab; T] {
var x = embed_tokens.forward(token)
static for group in groups {
x = group.decode(x, pos)
}
return logits(x[:, 0, :])
}
// A prompt of `S` tokens at `pos` in one pass (from zero at position 0);
// the logits after its last token.
pub entry prefill<S: Dim>(tokens: Tensor[Batch, S; i32], pos: i32) -> Tensor[Batch, Vocab; T]
where S > 0 {
var x = embed_tokens.forward(tokens)
static for group in groups {
x = group.prefill(x, pos)
}
return logits(x[:, S - 1, :])
}
// One new token per row, row `b` at `positions[b]` (continuous
// batching). A row with no request computes along and is ignored.
pub entry decode_rows(
tokens: Tensor[Batch, 1; i32],
positions: Tensor[Batch; i32],
) -> Tensor[Batch, Vocab; T] {
var x = embed_tokens.forward(tokens)
static for group in groups {
x = group.decode_rows(x, positions)
}
return logits(x[:, 0, :])
}
// `M` requests' prompts into rows `slots`, prompt `m` the first
// `lengths[m]` of its `S` tokens; the logits after each prompt's last
// token. `decode_rows` continues row `slots[m]` at `lengths[m]`.
pub entry prefill_slots<M: Dim, S: Dim>(
tokens: Tensor[M, S; i32],
slots: Tensor[M; i32],
lengths: Tensor[M; i32],
) -> Tensor[M, Vocab; T]
where
M > 0,
S > 0
{
var x = embed_tokens.forward(tokens)
static for group in groups {
x = group.prefill_slots(x, slots, lengths)
}
let last[b, h] = x[b, cast<i64>(lengths[b]) - 1, h]
return logits(last)
}
fn logits<R: Dim>(x: Tensor[R, H; T]) -> Tensor[R, Vocab; T] {
return lm_head.forward(norm.forward(x))
}
fn hidden_states<B: Dim, S: Dim>(tokens: Tensor[B, S; i32]) -> Tensor[B, S, H; T]
where S > 0 {
var x = embed_tokens.forward(tokens)
static for group in groups {
x = group.forward(x)
}
return x
}
}moe.linnet2.8 kB
linnet
// Qwen3.5-MoE's feed-forward block: a router picks `TopK` of `Experts`
// experts per token, by its softmax, renormalized over the chosen (which is
// the softmax of the chosen logits alone); each chosen expert is a SwiGLU
// of width `ExpertInner`, its gate and up projections stored side by side.
// Beside them a shared expert runs on every token, scaled by its own
// sigmoid gate.
//
// The chosen experts go through `std.nn.moe`: on a GPU PyTorch multiplies
// each expert by the rows that chose it (a grouped product), elsewhere the
// bodies' dense form.
module qwen3_5_moe.moe
use std.nn.activations::{sigmoid, silu}
use std.nn.linear::{Linear}
use std.nn.mlp::{SwiGlu}
use std.nn.moe::{combine_experts, linear_experts_shared}
use std.nn.softmax::{softmax}
pub block SparseMoe<H: Dim, Experts: Dim, TopK: Dim, ExpertInner: Dim, SharedInner: Dim, T: Float>
where
Experts > 0,
TopK > 0
{
sub gate: Linear<H, Experts, T>
param gate_up_proj: Tensor[Experts, 2 * ExpertInner, H; T]
param down_proj: Tensor[Experts, H, ExpertInner; T]
sub shared_expert: SwiGlu<H, SharedInner, T>
sub shared_expert_gate: Linear<H, 1, T>
pub fn forward<B: Dim, S: Dim>(x: Tensor[B, S, H; T]) -> Tensor[B, S, H; T] {
let rows = reshape(x, [B * S, H])
let logits = gate.forward(rows)
let ranked = ranks<B * S>(logits)
let slots = iota<i32>(Experts)
let places = iota<i32>(TopK)
let chosen[r, k] = sum<i32>[e] select(ranked[r, e] == places[k], slots[e], 0)
let expert[r, k] = cast<i64>(chosen[r, k])
// The softmax over the chosen logits alone, in f32: the others are
// pushed below its floor and come out as exactly zero.
let kept[r, e] = select(ranked[r, e] < cast<i32>(TopK), cast<f32>(logits[r, e]), -1e30)
let probabilities = softmax(kept)
let weight[r, k] = sum[e] select(ranked[r, e] == places[k], probabilities[r, e], 0.0)
let gate_up = linear_experts_shared(rows, gate_up_proj, expert)
let activated =
silu(gate_up[:, :, 0:ExpertInner]) * gate_up[:, :, ExpertInner:2 * ExpertInner]
let routed = combine_experts(activated, down_proj, expert, cast<T>(weight))
let shared = shared_expert.forward(rows) * sigmoid(shared_expert_gate.forward(rows))
return reshape(routed + shared, [B, S, H])
}
// Each expert's rank among a row's router logits: 0 for the largest,
// ties to the lower index as `torch.topk` orders them.
fn ranks<R: Dim>(logits: Tensor[R, Experts; T]) -> Tensor[R, Experts; i32] {
let slots = iota<i32>(Experts)
let outranks[r, e, j] =
logits[r, j] > logits[r, e] || (logits[r, j] == logits[r, e] && slots[j] < slots[e])
let beaten[r, e] = sum<i32>[j] cast<i32>(outranks[r, e, j])
return beaten
}
}norm.linnet1.3 kB
linnet
// Qwen3.5's two normalizations.
module qwen3_5_moe.norm
use std.nn.activations::{silu}
// RMS normalization around one: the checkpoint stores `w` and the layer
// scales by `1 + w`, in f32, rounding once at the end (Qwen3-Next's
// zero-centered norm).
pub block RmsNorm<H: Dim, T: Float> {
param weight: Tensor[H; T]
pub fn forward<*S: Shape>(x: Tensor[*S, H; T]) -> Tensor[*S, H; T] {
let xf = cast<f32>(x)
let mean_square[*s] = sum[h] xf[*s, h] * xf[*s, h] / cast<f32>(H)
let scaled[*s, h] = xf[*s, h] * rsqrt(mean_square[*s] + 1e-6) * (1.0 + cast<f32>(weight[h]))
return cast<T>(scaled)
}
}
// The linear attention's output norm, gated: each head's output normalized,
// rounded, scaled by `weight` (around zero, not one), then times
// `silu(gate)` in f32.
pub block GatedRmsNorm<D: Dim, T: Float> {
param weight: Tensor[D; T]
pub fn forward<*S: Shape>(x: Tensor[*S, D; T], gate: Tensor[*S, D; T]) -> Tensor[*S, D; T] {
let xf = cast<f32>(x)
let mean_square[*s] = sum[d] xf[*s, d] * xf[*s, d] / cast<f32>(D)
let normalized[*s, d] = xf[*s, d] * rsqrt(mean_square[*s] + 1e-6)
let weighted = cast<f32>(cast<T>(normalized) * weight)
return cast<T>(weighted * silu(cast<f32>(gate)))
}
}rope.linnet860 B
linnet
// Rotary tables for Qwen3.5's full-attention layers: only the first quarter
// of each head (`partial_rotary_factor` 0.25, 64 of 256) rotates, at a base
// of 10,000,000. The checkpoint's multimodal rope (`mrope_section`) gives a
// text token the same position on all three of its axes, which is this.
module qwen3_5_moe.rope
pub const THETA: f32 = 10000000.0
// `(cos, sin)` of shape `[S, D]` for `D` rotating dimensions, each frequency
// repeated for both halves as `rope` expects.
pub fn tables<S: Dim, D: Dim, T: Float>(theta: f32) -> (Tensor[S, D; T], Tensor[S, D; T])
where D % 2 == 0 {
let inv_freq[i] = exp(-(cast<f32>(iota<i64>(D / 2)[i]) * 2.0 / cast<f32>(D)) * log(theta))
let angles[s, i] = cast<f32>(iota<i64>(S)[s]) * inv_freq[i]
let full = concat(angles, angles, axis = -1)
return (cast<T>(cos(full)), cast<T>(sin(full)))
}model-00001-of-00026.safetensorsHugging Face ↗
model-00002-of-00026.safetensorsHugging Face ↗
model-00003-of-00026.safetensorsHugging Face ↗
model-00004-of-00026.safetensorsHugging Face ↗
model-00005-of-00026.safetensorsHugging Face ↗
model-00006-of-00026.safetensorsHugging Face ↗
model-00007-of-00026.safetensorsHugging Face ↗
model-00008-of-00026.safetensorsHugging Face ↗
model-00009-of-00026.safetensorsHugging Face ↗
model-00010-of-00026.safetensorsHugging Face ↗
model-00011-of-00026.safetensorsHugging Face ↗
model-00012-of-00026.safetensorsHugging Face ↗
model-00013-of-00026.safetensorsHugging Face ↗
model-00014-of-00026.safetensorsHugging Face ↗
model-00015-of-00026.safetensorsHugging Face ↗
model-00016-of-00026.safetensorsHugging Face ↗
model-00017-of-00026.safetensorsHugging Face ↗
model-00018-of-00026.safetensorsHugging Face ↗
model-00019-of-00026.safetensorsHugging Face ↗
model-00020-of-00026.safetensorsHugging Face ↗
model-00021-of-00026.safetensorsHugging Face ↗
model-00022-of-00026.safetensorsHugging Face ↗
model-00023-of-00026.safetensorsHugging Face ↗
model-00024-of-00026.safetensorsHugging Face ↗