Skip to content

vllm.model_executor.kernels.mhc.warmup

Classes:

HCPrenormGemmKernel

Bases: VllmJitKernel['HCPrenormGemmKernel.CompileKey']

DeepGEMM's tf32_hc_prenorm_gemm, which JITs once per (n, k, splits).

Source code in vllm/model_executor/kernels/mhc/warmup.py
class HCPrenormGemmKernel(VllmJitKernel["HCPrenormGemmKernel.CompileKey"]):
    """DeepGEMM's `tf32_hc_prenorm_gemm`, which JITs once per (n, k, splits)."""

    @dataclass(frozen=True, slots=True)
    class CompileKey:
        n: int
        k: int
        n_splits: int

    def get_warmup_keys(
        self,
        *,
        max_tokens: int,
        n: int,
        k: int,
        num_splits: Callable[[int, int], int],
    ) -> list[CompileKey]:
        # num_splits is the caller's runtime heuristic; it moves every 64 tokens.
        splits = {
            num_splits(k, num_tokens) for num_tokens in range(1, max_tokens + 1, 64)
        }
        return [self.CompileKey(n=n, k=k, n_splits=s) for s in sorted(splits)]

    def compile(self, compile_key: CompileKey) -> None:
        if compile_key in self._compiled_cache:
            return
        from vllm.utils.deep_gemm import tf32_hc_prenorm_gemm

        n, k, splits = compile_key.n, compile_key.k, compile_key.n_splits
        # M is a launch argument, not a template parameter, so any M works.
        num_tokens = 64
        device = current_platform.device_type
        x = torch.zeros((num_tokens, k), device=device, dtype=torch.bfloat16)
        fn = torch.zeros((n, k), device=device, dtype=torch.float32)
        out = torch.empty((splits, num_tokens, n), device=device, dtype=torch.float32)
        sqr_sum = torch.empty((splits, num_tokens), device=device, dtype=torch.float32)
        tf32_hc_prenorm_gemm(x, fn, out, sqr_sum, splits)
        self._compiled_cache[compile_key] = True