The DSpark draft layers' Gemma RMSNorms fused with the MXFP8 quantize of
the linear after them.
Each draft layer all-reduces two row-parallel outputs (o_proj,
down_proj), normalizes the sum right after (with the residual add), and
the next linear (gate_up_proj, the next layer's qkv_proj) quantizes
it to MXFP8: the row norm and the quantize are two launches, ten times per
draft step.
Here (FusedGemmaNormQuant) the stock all-reduce stays; the residual add,
the norm and the quantize run in one launch, and the result is handed to the
linear as a QuantizedActivation (also for the context states' norm before
the context K/V projection). The norm is exactly that of
drafter.GemmaRMSNormRows (the residual add rounded to the residual's
dtype and stored, the sum of squares over the same 8192-wide block,
x * rstd * (1 + w)), and the quantize is FlashInfer's MXFP8 quantize bit
for bit (UE8M0 round-up, RN e4m3, the F8_128x4 scale layout).
Fusing the all-reduce as well (a peer-memory Lamport exchange inside the
norm) measured slower at TP4 than FlashInfer's MNNVL all-reduce, which uses
NVLS multicast, so it is not done.
Classes:
Functions:
FusedGemmaNormQuant
The forward of a draft Gemma RMSNorm after the stock all-reduce, with
the next linear's MXFP8 quantize in the same launch: residual add, norm
and quantize, bit for bit as the row norm kernel followed by FlashInfer's
quantize.
Source code in vllm/models/minimax_m3/nvidia/specialized_decode/draft_norm.py
| class FusedGemmaNormQuant:
"""The forward of a draft Gemma RMSNorm after the stock all-reduce, with
the next linear's MXFP8 quantize in the same launch: residual add, norm
and quantize, bit for bit as the row norm kernel followed by FlashInfer's
quantize."""
def __init__(self, norm: nn.Module, consumer: nn.Module | None, stock, guard=None):
self._norm = norm
self._consumer = consumer
self._quant: bool | None = None
self._guard = guard # () -> bool: the consumer is the norm's only reader
self.stock = stock
def __call__(self, x: torch.Tensor, residual: torch.Tensor | None = None):
if self._quant is None:
self._quant = _takes_mxfp8(self._consumer) and (
self._guard is None or self._guard()
)
if (
not self._quant
or x.dim() != 2
or not 0 < x.shape[0] <= MAX_ROWS
or x.dtype != torch.bfloat16
or x.stride(-1) != 1
or x.shape[1] % 32
or x.shape[1] > _BLOCK
or (
residual is not None
and (residual.shape != x.shape or residual.stride(-1) != 1)
)
):
return self.stock(x, residual)
scratch = _SCRATCH.get(x.device)
if scratch is None or scratch.shape[1] != x.shape[1]:
scratch = _SCRATCH[x.device] = torch.empty(
MAX_ROWS, x.shape[1], dtype=torch.bfloat16, device=x.device
)
quant = _quant_buffers(x)
norm = self._norm
res = residual if residual is not None else x # x is rewritten unchanged
_norm_quant_kernel[(x.shape[0],)](
x,
x.stride(0),
res,
res.stride(0),
norm.weight,
scratch,
x.shape[1],
norm.variance_epsilon,
quant[0],
quant[1],
HAS_RESIDUAL=residual is not None,
BLOCK=_BLOCK,
num_warps=8,
)
out = _quantized(quant, x)
if residual is None:
return out
return out, residual
|
attach_fused_norms(model, stock_forward)
Fuse each draft norm with the MXFP8 quantize of the linear it feeds.
stock_forward(norm): the (already patched) forward of a norm. Returns
the number of fused norms.
Source code in vllm/models/minimax_m3/nvidia/specialized_decode/draft_norm.py
| def attach_fused_norms(model: nn.Module, stock_forward) -> int:
"""Fuse each draft norm with the MXFP8 quantize of the linear it feeds.
`stock_forward(norm)`: the (already patched) forward of a norm. Returns
the number of fused norms."""
reason = fused_norms_unsupported(model)
if reason:
logger.info("DSpark draft norms stay separate: %s.", reason)
return 0
inner = model.model
count = 0
for layer in inner.layers:
attn = layer.self_attn
pairs = [
# The draft attention reads its input's dtype for the attention output
# unless it states one (FLASHINFER_TS with an FP8 query): only then may the
# input norm hand it a QuantizedActivation.
(
layer.input_layernorm,
attn.qkv_proj,
lambda attn=attn: getattr(attn, "_attn_output_dtype", None) is not None,
),
(layer.post_attention_layernorm, layer.mlp.gate_up_proj, None),
]
for norm, consumer, guard in pairs:
norm.forward = FusedGemmaNormQuant(
norm, consumer, stock_forward(norm), guard=guard
)
count += 1
# The context states' norm feeds the fused context K/V projection.
hidden_norm = getattr(inner, "hidden_norm", None)
ctx_proj = getattr(inner, "_context_kv_proj", None)
if hidden_norm is not None and ctx_proj is not None:
hidden_norm.forward = FusedGemmaNormQuant(
hidden_norm,
ctx_proj,
stock_forward(hidden_norm),
# A BF16 fused K/V weight takes precedence over the projection.
guard=lambda: getattr(inner, "_fused_kv_weight", None) is None,
)
count += 1
logger.info("DSpark draft norms: %d with the MXFP8 quantize fused.", count)
return count
|
fused_norms_unsupported(model)
Why the draft norms keep their stock forward ("" when they fuse).
Source code in vllm/models/minimax_m3/nvidia/specialized_decode/draft_norm.py
| def fused_norms_unsupported(model: nn.Module) -> str:
"""Why the draft norms keep their stock forward ("" when they fuse)."""
layers = getattr(getattr(model, "model", None), "layers", None)
if not layers:
return "no draft layers"
for layer in layers:
for norm in (layer.input_layernorm, layer.post_attention_layernorm):
if type(norm).__name__ != "MiniMAXGemmaRMSNorm":
return f"a {type(norm).__name__} draft norm"
if type(getattr(model.model, "norm", None)).__name__ != "MiniMAXGemmaRMSNorm":
return "a final norm that is not a Gemma RMSNorm"
if model.model.norm.weight.dtype != torch.bfloat16:
return f"a {model.model.norm.weight.dtype} draft model"
return ""
|