Skip to content

vllm.models.muse_spark

Muse Spark HF model entry point.

The model classes load on first access, so importing a submodule such as common.vision doesn't import the NVIDIA model. Its kernels import FlashInfer, which queries the GPU at import and so initializes CUDA in the importing process; processes forked from it afterwards can't use CUDA.

Modules:

  • amd –

    AMD ROCm implementation of Muse Spark.

  • common –

    Platform-independent MuseSpark preprocessing.

  • nvidia –

    NVIDIA-specific MuseSpark model implementations.

Classes:

MuseSparkDSparkForCausalLM

Bases: Qwen3DSparkForCausalLM

Muse DSpark head with the target's output-distribution transform.

Methods:

Source code in vllm/models/muse_spark/nvidia/dspark.py
class MuseSparkDSparkForCausalLM(Qwen3DSparkForCausalLM):
    """Muse DSpark head with the target's output-distribution transform."""

    has_own_lm_head = True

    hf_to_vllm_mapper = WeightsMapper(
        orig_to_new_stacked={
            ".self_attn.q_proj.": (".self_attn.qkv_proj.", "q"),
            ".self_attn.k_proj.": (".self_attn.qkv_proj.", "k"),
            ".self_attn.v_proj.": (".self_attn.qkv_proj.", "v"),
            ".self_attn.o_gate_proj.": (".self_attn.qkv_proj.", "o_gate"),
            ".feed_forward.gate_proj.": (".feed_forward.gate_up_proj.", 0),
            ".feed_forward.up_proj.": (".feed_forward.gate_up_proj.", 1),
        }
    )

    def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
        nn.Module.__init__(self)
        assert vllm_config.speculative_config is not None
        self.draft_model_config = vllm_config.speculative_config.draft_model_config
        self.config = self.draft_model_config.hf_config
        target_layer_num = vllm_config.model_config.get_total_num_hidden_layers()
        self.model = MuseSparkDSparkModel(
            vllm_config=vllm_config,
            prefix=maybe_prefix(prefix, "model"),
            start_layer_id=target_layer_num,
        )
        self.lm_head = ParallelLMHead(
            self.config.draft_vocab_size,
            self.config.hidden_size,
            prefix=maybe_prefix(prefix, "lm_head"),
        )
        self.logits_processor = LogitsProcessor(self.config.draft_vocab_size)
        self.target_vocab_size = vllm_config.model_config.get_vocab_size()
        self.draft_id_to_target_id = nn.Parameter(
            torch.zeros(self.config.draft_vocab_size, dtype=torch.long),
            requires_grad=False,
        )
        metap = self.config.metap
        if metap.use_metap and metap.router_has_one_inf_dim:
            width_mult = self.config.hidden_size / metap.base_width
            if metap.metap_mode == "mup":
                self.output_multiplier = width_mult**-1
            elif metap.metap_mode == "sp":
                self.output_multiplier = width_mult**-0.5
            else:
                raise ValueError(f"Unknown Metap mode {metap.metap_mode!r}")
        else:
            self.output_multiplier = 1.0
        self.output_soft_cap_temp = self.config.output_soft_cap_temp
        topk = getattr(self.config, "dspark_draft_topk", None)
        if self.model.specialized_decode and topk is not None:
            # The speculator's top-k loop never calls sample_sequential.
            raise ValueError(
                "The Muse Spark specialized decode runs the fused DSpark Markov "
                "loop, which does not run here: dspark_draft_topk is set"
            )
        # The fused Markov loop under the specialized decode (sample_sequential).
        self._markov_sampler: MarkovSampler | None = None
        self._ordinary_markov_chain: MarkovChain | None = None
        self._ordinary_markov_chain_initialized = False
        if self.model.specialized_decode:
            # The hook exists only here: without it the speculator may
            # batch-shard the draft logits and the sequential sampling.
            self.sample_sequential = self._sample_sequential_fused

    def apply_output_logit_transform(self, logits: torch.Tensor) -> torch.Tensor:
        logits = logits.float() * self.output_multiplier
        if self.output_soft_cap_temp is not None:
            logits = self.output_soft_cap_temp * torch.tanh(
                logits / self.output_soft_cap_temp
            )
        return logits

    def sample_markov_chain(
        self,
        speculator,
        base_logits,
        prev,
        idx_map,
        sample_pos,
        out_tokens,
        *,
        prev_indices=None,
    ) -> torch.Tensor | None:
        """Fuse sampling tails after the ordinary base-logits projection.
        Returns each step's Markov embedding, `[rows, steps, rank]`, or None
        when the speculator must run its own loop."""
        if (
            self.model.specialized_decode
            or self.model._compilation_mode != CompilationMode.NONE
            or not current_platform.is_cuda()
            or not current_platform.is_device_capability_family(100)
        ):
            return None
        if not self._ordinary_markov_chain_initialized:
            from .specialized_decode.runtime.markov_chain import build_markov_chain

            self._ordinary_markov_chain = build_markov_chain(self, speculator)
            self._ordinary_markov_chain_initialized = True
            if self._ordinary_markov_chain is not None:
                logger.info_once(
                    "Using the fused DSpark Markov chain (confidence head: %s).",
                    speculator.use_confidence_head,
                    scope="process",
                )
        if self._ordinary_markov_chain is None:
            return None
        return self._ordinary_markov_chain.run(
            speculator,
            base_logits,
            prev,
            idx_map,
            sample_pos,
            out_tokens,
            prev_indices=prev_indices,
        )

    def output_logit_transform_params(self) -> tuple[float, float | None]:
        """`(multiplier, soft_cap)` of `apply_output_logit_transform`.

        The fused DSpark Markov loop computes the transform from these values
        (`logits * multiplier`, then `soft_cap * tanh(x / soft_cap)` unless
        `soft_cap` is None) instead of calling it, so a subclass that changes
        the transform must override this too.
        """
        return self.output_multiplier, self.output_soft_cap_temp

    def _sample_sequential_fused(
        self,
        speculator: "DSparkSpeculator",
        num_reqs: int,
        head_hidden: torch.Tensor,
    ) -> bool:
        """The `sample_sequential` hook under the specialized decode, which
        DSparkSpeculator._sample_sequential asks first: the fused Markov
        sampling loop samples the drafts, and this returns True."""
        if self._markov_sampler is None:
            # Built on the first call (the memory-profiling run), from the
            # speculator's sampling state.
            self._markov_sampler = self._build_markov_sampler(speculator)
        draft_logits = _draft_logits_values(speculator)
        assert draft_logits is not None
        self._markov_sampler.sample(
            num_reqs,
            head_hidden,
            sample_indices=speculator.sample_indices,
            sample_idx_mapping=speculator.sample_idx_mapping,
            sample_pos=speculator.sample_pos,
            input_ids=speculator.input_buffers.input_ids,
            anchor_idx=speculator._anchor_idx,
            seeds=speculator.seeds,
            temperature=speculator.temperature,
            draft_tokens=speculator.draft_tokens,
            draft_logits=draft_logits,
        )
        return True

    def _build_markov_sampler(self, speculator: "DSparkSpeculator") -> "MarkovSampler":
        if speculator.use_confidence_head or speculator.use_acceptance_estimator:
            raise ValueError(
                "The Muse Spark specialized decode's fused DSpark Markov loop "
                "computes neither the confidence head's outputs nor the "
                "acceptance estimates: it does not run with adaptive verification."
            )
        from vllm.models.muse_spark.nvidia.specialized_decode.runtime.markov import (
            build_markov_sampler,
        )

        return build_markov_sampler(
            self,
            draft_logits=_draft_logits_values(speculator),
            draft_topk=speculator._draft_topk,
            watermarked=speculator.draft_watermarker is not None,
            use_fp64_gumbel=speculator.use_fp64_gumbel,
            sample_from_anchor=speculator.sample_from_anchor,
            max_num_reqs=speculator.max_num_reqs,
            num_steps=speculator.num_speculative_steps,
        )

    def compute_draft_logits(self, hidden_states: torch.Tensor) -> torch.Tensor:
        # Identical full-vocabulary heads may be shared with an FP32 target head.
        return self.logits_processor(
            self.lm_head, hidden_states.to(self.lm_head.weight.dtype)
        )

    def combine_hidden_states(self, hidden_states: torch.Tensor) -> torch.Tensor:
        # Target residuals are FP32; the training collator casts features before FC.
        hidden_states = hidden_states.to(self.model.fc.weight.dtype)
        out = super().combine_hidden_states(hidden_states)
        if isinstance(self.model.fc, ColumnParallelLinear):
            # Each rank holds its block of output columns for every token.
            out = gather_columns(out.view(-1, out.shape[-1])).view(*out.shape[:-1], -1)
        return out

    def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
        model_weights = {}
        for name, loaded_weight in weights:
            if name == "t2d":
                continue
            if name == "d2t":
                name = "draft_id_to_target_id"
            elif "lm_head" not in name:
                name = "model." + name
            model_weights[name] = loaded_weight
        mapper = WeightsMapper(
            orig_to_new_substr={
                "model.embed_tokens": None,
                "model.mask_embedding": None,
            },
            orig_to_new_stacked=self.hf_to_vllm_mapper.orig_to_new_stacked,
        )
        return AutoWeightsLoader(self).load_weights(
            model_weights.items(), mapper=mapper
        )

    def process_weights_after_loading(self) -> None:
        if self.model.specialized_draft is not None or self._markov_sampler is not None:
            # Runs again after a weight reload; the captured draft steps read
            # the kernels' constants and buffers built from the loaded weights.
            raise RuntimeError(
                "The Muse Spark specialized decode does not support reloading "
                "the DSpark draft's weights"
            )
        # Builds the stacked K/V weights that the fused context chain reads.
        super().process_weights_after_loading()
        materialize_residual_path_constants(self.model)
        if self.model.specialized_decode:
            from vllm.models.muse_spark.nvidia.specialized_decode.runtime.draft import (
                build_draft_decode,
            )

            self.model.specialized_draft = build_draft_decode(
                self.model, self.model._compilation_mode
            )
        elif (
            current_platform.is_cuda()
            and current_platform.is_device_capability_family(100)
            and self.model._compilation_mode == CompilationMode.NONE
        ):
            from vllm.models.muse_spark.nvidia.specialized_decode.runtime.draft import (
                build_draft_norms,
            )

            self.model._fused_draft_norms = build_draft_norms(self.model)

_sample_sequential_fused(speculator, num_reqs, head_hidden)

The sample_sequential hook under the specialized decode, which DSparkSpeculator._sample_sequential asks first: the fused Markov sampling loop samples the drafts, and this returns True.

Source code in vllm/models/muse_spark/nvidia/dspark.py
def _sample_sequential_fused(
    self,
    speculator: "DSparkSpeculator",
    num_reqs: int,
    head_hidden: torch.Tensor,
) -> bool:
    """The `sample_sequential` hook under the specialized decode, which
    DSparkSpeculator._sample_sequential asks first: the fused Markov
    sampling loop samples the drafts, and this returns True."""
    if self._markov_sampler is None:
        # Built on the first call (the memory-profiling run), from the
        # speculator's sampling state.
        self._markov_sampler = self._build_markov_sampler(speculator)
    draft_logits = _draft_logits_values(speculator)
    assert draft_logits is not None
    self._markov_sampler.sample(
        num_reqs,
        head_hidden,
        sample_indices=speculator.sample_indices,
        sample_idx_mapping=speculator.sample_idx_mapping,
        sample_pos=speculator.sample_pos,
        input_ids=speculator.input_buffers.input_ids,
        anchor_idx=speculator._anchor_idx,
        seeds=speculator.seeds,
        temperature=speculator.temperature,
        draft_tokens=speculator.draft_tokens,
        draft_logits=draft_logits,
    )
    return True

output_logit_transform_params()

(multiplier, soft_cap) of apply_output_logit_transform.

The fused DSpark Markov loop computes the transform from these values (logits * multiplier, then soft_cap * tanh(x / soft_cap) unless soft_cap is None) instead of calling it, so a subclass that changes the transform must override this too.

Source code in vllm/models/muse_spark/nvidia/dspark.py
def output_logit_transform_params(self) -> tuple[float, float | None]:
    """`(multiplier, soft_cap)` of `apply_output_logit_transform`.

    The fused DSpark Markov loop computes the transform from these values
    (`logits * multiplier`, then `soft_cap * tanh(x / soft_cap)` unless
    `soft_cap` is None) instead of calling it, so a subclass that changes
    the transform must override this too.
    """
    return self.output_multiplier, self.output_soft_cap_temp

sample_markov_chain(speculator, base_logits, prev, idx_map, sample_pos, out_tokens, *, prev_indices=None)

Fuse sampling tails after the ordinary base-logits projection. Returns each step's Markov embedding, [rows, steps, rank], or None when the speculator must run its own loop.

Source code in vllm/models/muse_spark/nvidia/dspark.py
def sample_markov_chain(
    self,
    speculator,
    base_logits,
    prev,
    idx_map,
    sample_pos,
    out_tokens,
    *,
    prev_indices=None,
) -> torch.Tensor | None:
    """Fuse sampling tails after the ordinary base-logits projection.
    Returns each step's Markov embedding, `[rows, steps, rank]`, or None
    when the speculator must run its own loop."""
    if (
        self.model.specialized_decode
        or self.model._compilation_mode != CompilationMode.NONE
        or not current_platform.is_cuda()
        or not current_platform.is_device_capability_family(100)
    ):
        return None
    if not self._ordinary_markov_chain_initialized:
        from .specialized_decode.runtime.markov_chain import build_markov_chain

        self._ordinary_markov_chain = build_markov_chain(self, speculator)
        self._ordinary_markov_chain_initialized = True
        if self._ordinary_markov_chain is not None:
            logger.info_once(
                "Using the fused DSpark Markov chain (confidence head: %s).",
                speculator.use_confidence_head,
                scope="process",
            )
    if self._ordinary_markov_chain is None:
        return None
    return self._ordinary_markov_chain.run(
        speculator,
        base_logits,
        prev,
        idx_map,
        sample_pos,
        out_tokens,
        prev_indices=prev_indices,
    )

MuseSparkForConditionalGeneration

Bases: MuseSparkOmniMixin, MuseSparkForCausalLM, SupportsMultiModal

Muse Spark Omni: text stack plus NaViT vision and waveform audio.

Source code in vllm/models/muse_spark/nvidia/multimodal.py
@register_omni_processor
class MuseSparkForConditionalGeneration(
    MuseSparkOmniMixin, MuseSparkForCausalLM, SupportsMultiModal
):
    """Muse Spark Omni: text stack plus NaViT vision and waveform audio."""

    hf_to_vllm_mapper = MuseSparkForCausalLM.hf_to_vllm_mapper | OMNI_WEIGHTS_MAPPER