class XPUExperts(LoRAExpertsMixin, mk.FusedMoEExpertsModular):
def __init__(
self,
moe_config: FusedMoEConfig,
quant_config: FusedMoEQuantConfig,
max_num_tokens: int | None = None,
num_dispatchers: int | None = None,
):
super().__init__(
moe_config,
quant_config,
max_num_tokens,
num_dispatchers,
)
self.gemm1_clamp_limit = quant_config.gemm1_clamp_limit
self.fused_moe_impl: XpuFusedMoe | None = None
self._lora_ones = torch.empty(0)
is_xe2_or_xe3 = torch.ops._xpu_C.is_xe2_arch() or torch.ops._xpu_C.is_xe3_arch()
self._expects_unquantized_inputs = is_xe2_or_xe3
@property
def expects_unquantized_inputs(self) -> bool:
return self._expects_unquantized_inputs
@staticmethod
def activation_format() -> mk.FusedMoEActivationFormat:
return mk.FusedMoEActivationFormat.Standard
@staticmethod
def _supports_current_device() -> bool:
return current_platform.is_xpu()
@staticmethod
def _supports_no_act_and_mul() -> bool:
return True
@staticmethod
def _supports_activation(activation: MoEActivation) -> bool:
return activation in [
MoEActivation.SILU,
MoEActivation.GELU,
MoEActivation.GELU_TANH,
MoEActivation.SWIGLUOAI,
MoEActivation.RELU2_NO_MUL,
]
@staticmethod
def _supports_parallel_config(moe_parallel_config: FusedMoEParallelConfig) -> bool:
return True
@staticmethod
def _supports_quant_scheme(
weight_key: QuantKey | None,
activation_key: QuantKey | None,
) -> bool:
SUPPORTED_W_A = [
(None, None),
(kFp8StaticTensorSym, None),
(kFp8StaticTensorSym, kFp8DynamicTensorSym),
]
return (weight_key, activation_key) in SUPPORTED_W_A
def finalize_weight_and_reduce_impl(self) -> mk.TopKWeightAndReduce:
return TopKWeightAndReduceNoOP()
def workspace_shapes(
self,
M: int,
N: int,
K: int,
topk: int,
global_num_experts: int,
local_num_experts: int,
expert_tokens_meta: mk.ExpertTokensMetadata | None,
activation: MoEActivation,
) -> tuple[tuple[int, ...], tuple[int, ...], tuple[int, ...]]:
workspace1 = (0,)
workspace2 = (0,)
output = (M, K)
return (workspace1, workspace2, output)
def apply(
self,
output: torch.Tensor,
hidden_states: torch.Tensor,
w1: torch.Tensor,
w2: torch.Tensor,
topk_weights: torch.Tensor,
topk_ids: torch.Tensor,
activation: MoEActivation,
global_num_experts: int,
expert_map: torch.Tensor | None,
a1q_scale: torch.Tensor | None,
a2_scale: torch.Tensor | None,
workspace13: torch.Tensor,
workspace2: torch.Tensor,
expert_tokens_meta: mk.ExpertTokensMetadata | None,
apply_router_weight_on_input: bool,
):
if self.moe_config.is_lora_enabled and apply_router_weight_on_input:
raise NotImplementedError(
"XPU MoE LoRA requires routing weights on the output."
)
if self.fused_moe_impl is None:
topk = topk_ids.size(-1)
if (
self.quant_config is not None
and self.quant_config.weight_quant_dtype == "mxfp4"
):
w1 = w1.view(torch.float4_e2m1fn_x2)
w2 = w2.view(torch.float4_e2m1fn_x2)
# XpuFusedMoe relayouts the loaded [E, N, K] weights and scales
# in place for this device's grouped GEMM.
self.fused_moe_impl = XpuFusedMoe(
w13=w1,
w13_scales=self.w1_scale,
w13_bias=self.w1_bias,
w2=w2,
w2_scales=self.w2_scale,
w2_bias=self.w2_bias,
n_experts_per_token=topk,
activation=activation.value,
num_experts=self.moe_config.num_local_experts,
ep_rank=self.moe_config.ep_rank,
ep_size=self.moe_config.ep_size,
expert_map=expert_map,
gemm1_clamp_limit=self.gemm1_clamp_limit,
)
assert self.fused_moe_impl is not None
if self.moe_config.ep_size > 1:
# Avoid topk_ids = -1, remap them to an
# expert not owned by this rank so they are skipped.
sentinel = global_num_experts - 1 if self.moe_config.ep_rank == 0 else 0
topk_ids = torch.where(topk_ids < 0, sentinel, topk_ids)
self._apply_experts(
output,
hidden_states,
w1,
w2,
topk_weights,
topk_ids,
expert_map,
a1q_scale,
)
def _apply_experts(
self,
output: torch.Tensor,
hidden_states: torch.Tensor,
w1: torch.Tensor,
w2: torch.Tensor,
topk_weights: torch.Tensor,
topk_ids: torch.Tensor,
expert_map: torch.Tensor | None,
a1q_scale: torch.Tensor | None,
) -> None:
assert self.fused_moe_impl is not None
hooks = {}
if self._lora_context is not None:
hooks = self._lora_hooks(
self._lora_context,
hidden_states,
w1,
w2,
topk_weights,
topk_ids,
expert_map,
)
self.fused_moe_impl.apply(
output=output,
hidden_states=hidden_states,
topk_weights=topk_weights,
topk_ids=topk_ids,
a1q_scale=a1q_scale,
**hooks,
)
def _lora_hooks(
self,
ctx: MoELoRAContext,
hidden_states: torch.Tensor,
w1: torch.Tensor,
w2: torch.Tensor,
topk_weights: torch.Tensor,
topk_ids: torch.Tensor,
expert_map: torch.Tensor | None,
) -> dict[str, Callable[..., None]]:
"""Builds XpuFusedMoe hooks that add the LoRA deltas in place.
gemm1_output / act_output / gemm2_output hold one row per
(token, topk) pair grouped by expert; `row_map` gives each pair's
row (-1 for skipped pairs), so the LoRA kernels read and accumulate
into those rows directly.
"""
num_tokens, top_k = topk_ids.shape
# XpuFusedMoe stores bf16 weights as [E, K, N]; the LoRA config
# lookup expects the loaded [E, N, K] shapes.
w1 = w1.transpose(-1, -2)
w2 = w2.transpose(-1, -2)
# moe_gather applies the routing weights to base and LoRA outputs.
if self._lora_ones.shape[0] < num_tokens:
self._lora_ones = torch.ones_like(topk_weights)
unweighted = self._lora_ones[:num_tokens]
w13_meta: tuple = ()
def activation_hook(gemm1_output: torch.Tensor, row_map: torch.Tensor):
nonlocal w13_meta
w13_meta = self.apply_w13_lora(
ctx,
y=gemm1_output,
x=hidden_states,
topk_ids=topk_ids,
topk_weights=unweighted,
expert_map=expert_map,
w1=w1,
w2=w2,
num_tokens=num_tokens,
top_k_num=top_k,
row_map=row_map.view(-1),
)
def output_hook(
gemm2_output: torch.Tensor,
act_output: torch.Tensor,
row_map: torch.Tensor,
):
sorted_token_ids, expert_ids, num_tokens_post_padded, token_lora_map = (
w13_meta
)
self.apply_w2_lora(
ctx,
y=gemm2_output.view(num_tokens, top_k, -1),
x=act_output,
topk_weights=unweighted,
sorted_token_ids_lora=sorted_token_ids,
expert_ids_lora=expert_ids,
num_tokens_post_padded_lora=num_tokens_post_padded,
token_lora_mapping=token_lora_map,
num_tokens=num_tokens,
w1=w1,
w2=w2,
top_k_num=top_k,
row_map=row_map.view(-1),
)
return dict(activation_hook=activation_hook, output_hook=output_hook)