Skip to content

vllm.model_executor.kernels.linear.mxfp8.Mxfp8LinearKernel

Classes:

Mxfp8LinearKernel

Bases: ABC

Base class for MXFP8 quantized linear kernels.

Each subclass implements a specific GEMM backend (FlashInfer CUTLASS, Marlin, emulation).

Methods:

Attributes:

Source code in vllm/model_executor/kernels/linear/mxfp8/Mxfp8LinearKernel.py
class Mxfp8LinearKernel(ABC):
    """Base class for MXFP8 quantized linear kernels.

    Each subclass implements a specific GEMM backend (FlashInfer CUTLASS,
    Marlin, emulation).
    """

    supports_pre_processed_weights: bool = False
    """True if `process_weights_after_loading` only rewrites parameters, so
    weights exported by the weight cache daemon can be used as-is."""

    def __init__(self, c: Mxfp8LinearLayerConfig) -> None:
        assert self.can_implement(c)[0]
        assert self.is_supported()[0]
        self.config = c

    def input_quant_key(self) -> QuantKey | None:
        """Return the input quantization key supported by this kernel. If the kernel
        does not support input quantization outside of the kernel, return None.
        """
        return None

    @classmethod
    @abstractmethod
    def is_supported(
        cls, compute_capability: int | None = None
    ) -> tuple[bool, str | None]:
        raise NotImplementedError

    @classmethod
    def executed_activation_key(
        cls, activation_quant_key: QuantKey | None
    ) -> QuantKey | None:
        """The activation quantization the kernel runs for a layer requesting
        `activation_quant_key`. Weight-only kernels return None; kernel
        selection then warns about, or under VLLM_STRICT_QUANT_SCHEME forbids,
        the fallback."""
        return activation_quant_key

    @classmethod
    @abstractmethod
    def can_implement(cls, c: Mxfp8LinearLayerConfig) -> tuple[bool, str | None]:
        raise NotImplementedError

    @classmethod
    def supports_batch_invariance(cls) -> bool:
        """Whether a row's output is independent of the other rows in the batch."""
        return False

    @abstractmethod
    def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
        raise NotImplementedError

    @abstractmethod
    def apply_weights(
        self,
        layer: torch.nn.Module,
        x: torch.Tensor,
        bias: torch.Tensor | None = None,
    ) -> torch.Tensor:
        raise NotImplementedError

supports_pre_processed_weights = False class-attribute instance-attribute

True if process_weights_after_loading only rewrites parameters, so weights exported by the weight cache daemon can be used as-is.

executed_activation_key(activation_quant_key) classmethod

The activation quantization the kernel runs for a layer requesting activation_quant_key. Weight-only kernels return None; kernel selection then warns about, or under VLLM_STRICT_QUANT_SCHEME forbids, the fallback.

Source code in vllm/model_executor/kernels/linear/mxfp8/Mxfp8LinearKernel.py
@classmethod
def executed_activation_key(
    cls, activation_quant_key: QuantKey | None
) -> QuantKey | None:
    """The activation quantization the kernel runs for a layer requesting
    `activation_quant_key`. Weight-only kernels return None; kernel
    selection then warns about, or under VLLM_STRICT_QUANT_SCHEME forbids,
    the fallback."""
    return activation_quant_key

input_quant_key()

Return the input quantization key supported by this kernel. If the kernel does not support input quantization outside of the kernel, return None.

Source code in vllm/model_executor/kernels/linear/mxfp8/Mxfp8LinearKernel.py
def input_quant_key(self) -> QuantKey | None:
    """Return the input quantization key supported by this kernel. If the kernel
    does not support input quantization outside of the kernel, return None.
    """
    return None

supports_batch_invariance() classmethod

Whether a row's output is independent of the other rows in the batch.

Source code in vllm/model_executor/kernels/linear/mxfp8/Mxfp8LinearKernel.py
@classmethod
def supports_batch_invariance(cls) -> bool:
    """Whether a row's output is independent of the other rows in the batch."""
    return False

Mxfp8LinearLayerConfig dataclass

Configuration for an MXFP8 linear layer.

All MXFP8 layers share the same structure: FP8-E4M3 weights with uint8 (E8M0) per-block scales at block size 32.

Attributes:

  • weight_shape (tuple[int, int]) –

    The layer's (out_features, in_features), i.e. (N, K).

Source code in vllm/model_executor/kernels/linear/mxfp8/Mxfp8LinearKernel.py
@dataclass(slots=True)
class Mxfp8LinearLayerConfig:
    """Configuration for an MXFP8 linear layer.

    All MXFP8 layers share the same structure: FP8-E4M3 weights with
    uint8 (E8M0) per-block scales at block size 32.

    Attributes:
        weight_shape: The layer's `(out_features, in_features)`, i.e. `(N, K)`.

    """

    weight_shape: tuple[int, int]
    bmm_batch_size: int | None = None