Skip to content

vllm.model_executor.models.qwen3_dspine

Classes:

AdjacentCausalInjection

Bases: Module

BF16 adjacent injection and FP32 last-write refinement.

Source code in vllm/model_executor/models/qwen3_dspine.py
class AdjacentCausalInjection(nn.Module):
    """BF16 adjacent injection and FP32 last-write refinement."""

    def __init__(
        self,
        *,
        hidden_size: int,
        transfer_dim: int,
        message_dim: int,
        num_layers: int,
        block_size: int,
        params_dtype: torch.dtype,
        quant_config: QuantizationConfig | None,
        eps: float,
        prefix: str,
    ) -> None:
        super().__init__()
        self.hidden_size = hidden_size
        self.transfer_dim = transfer_dim
        self.num_layers = num_layers
        self.block_size = block_size
        self.eps = eps

        def linear(input_size, output_size, name):
            return ReplicatedLinear(
                input_size,
                output_size,
                bias=False,
                params_dtype=params_dtype,
                quant_config=quant_config,
                prefix=maybe_prefix(prefix, name),
                return_bias=False,
            )

        self.readouts = nn.ModuleList(
            linear(hidden_size, transfer_dim, f"readouts.{layer_idx}")
            for layer_idx in range(num_layers)
        )
        self.writes = nn.ModuleList(
            linear(message_dim, hidden_size, f"writes.{layer_idx}")
            for layer_idx in range(num_layers)
        )
        self.message = linear(transfer_dim, message_dim, "message")
        self.receiver_gate = linear(transfer_dim, message_dim, "receiver_gate")
        self.predecessor_gate = linear(transfer_dim, message_dim, "predecessor_gate")
        self.gate_bias = nn.Parameter(
            torch.empty(message_dim, dtype=params_dtype), requires_grad=False
        )

    @staticmethod
    def rms(states: torch.Tensor) -> torch.Tensor:
        return (
            states.float()
            .square()
            .mean(-1, keepdim=True)
            .clamp_min(torch.finfo(torch.float32).tiny)
            .sqrt()
        )

    def project(self, layer_idx: int, hidden_states: torch.Tensor) -> torch.Tensor:
        normalized = hidden_states.float() / (self.rms(hidden_states) + self.eps)
        projected = self.readouts[layer_idx](normalized.to(hidden_states.dtype)).float()
        features = (
            math.sqrt(self.transfer_dim)
            * projected
            / (projected.norm(dim=-1, keepdim=True) + self.eps)
        )
        return features.to(hidden_states.dtype)

    def token_features(
        self, codebook: torch.Tensor, token_ids: torch.Tensor
    ) -> torch.Tensor:
        return (codebook[token_ids.long()].float() * math.sqrt(self.transfer_dim)).to(
            codebook.dtype
        )

    def write(
        self,
        layer_idx,
        receiver_states,
        receiver_features,
        predecessor_features,
        *,
        refine: bool = False,
    ):
        def linear(layer, features):
            if refine:
                return F.linear(features, layer.weight.float())
            return layer(features).float()

        gate = torch.sigmoid(
            self.receiver_gate(receiver_features).float()
            + linear(self.predecessor_gate, predecessor_features)
            + self.gate_bias.float()
        )
        message = linear(self.message, predecessor_features)
        gated_message = gate * message
        if not refine:
            gated_message = gated_message.to(receiver_states.dtype)
        update = linear(self.writes[layer_idx], gated_message)
        result = receiver_states.float() + self.rms(receiver_states) * update
        return result if refine else result.to(receiver_states.dtype)

    def forward(
        self,
        layer_idx: int,
        pre_injection_states: torch.Tensor,
        *,
        anchor_features: torch.Tensor,
    ) -> torch.Tensor:
        blocks = pre_injection_states.reshape(-1, self.block_size, self.hidden_size)
        features = self.project(layer_idx, blocks)
        predecessor = torch.cat((anchor_features[:, None, :], features[:, 1:-1]), dim=1)
        successors = self.write(layer_idx, blocks[:, 1:], features[:, 1:], predecessor)
        return torch.cat((blocks[:, :1], successors), dim=1).reshape_as(
            pre_injection_states
        )