Diff Coverage

Diff: origin/master...HEAD, staged and unstaged changes

Source File Diff Coverage (%) Missing Lines
hyper_parallel/codegen/emit/modeling.py 100%  
hyper_parallel/codegen/emit/replacement.py 100%  
hyper_parallel/codegen/inline/framework_spec.py 81.0% 263,362,365-366
hyper_parallel/codegen/inline/ir.py 100%  
hyper_parallel/codegen/inline/observations.py 88.9% 65
hyper_parallel/codegen/inline/patch_engine.py 100%  
hyper_parallel/codegen/inline/replacement_pass.py 100%  
hyper_parallel/codegen/model_build_overlay.py 100%  
hyper_parallel/components/functional/npu_fusion_attention.py 100%  
hyper_parallel/components/losses/model_output.py 50.0% 58-59,61
hyper_parallel/components/modules/attention/gated_grouped_query_attention/__init__.py 100%  
hyper_parallel/components/modules/attention/gated_grouped_query_attention/cp_gated_grouped_query_attention.py 100%  
hyper_parallel/components/modules/attention/gated_grouped_query_attention/gated_grouped_query_attention.py 88.9% 38,53,60,64,67,69,93,104
hyper_parallel/components/modules/attention/gated_grouped_query_attention/gated_grouped_query_attention_family.py 100%  
hyper_parallel/components/modules/attention/gated_multi_latent_attention/__init__.py 100%  
hyper_parallel/components/modules/attention/gated_multi_latent_attention/cp_gated_mla_backward_overlap.py 65.2% 44,58-64
hyper_parallel/components/modules/attention/gated_multi_latent_attention/cp_gated_multi_latent_attention.py 41.8% 52,58-62,65-66,79-81,93-95,110-112,115-117,119-122,125-127,129-130,141-142,144
hyper_parallel/components/modules/attention/gated_multi_latent_attention/gated_multi_latent_attention.py 70.7% 53,55,57,75-76,102,110,115-116,130-132
hyper_parallel/components/modules/attention/gated_multi_latent_attention/gated_multi_latent_attention_family.py 100%  
hyper_parallel/components/modules/attention/grouped_query_attention/__init__.py 100%  
hyper_parallel/components/modules/attention/grouped_query_attention/cp_grouped_query_attention.py 52.9% 76-84,95-100,151,163-173,180-183,194-200,206-213,216-221,223-227,229-232,235-238,249-262,288,295,297,346-347,351-354
hyper_parallel/components/modules/attention/grouped_query_attention/grouped_query_attention.py 86.1% 53,56,59,65,71-73,108,110,115,136,147,152,161-166,171-173,176,179,186,188,196,198,226-228,242,245,259,269,289,315,451
hyper_parallel/components/modules/attention/grouped_query_attention/grouped_query_attention_family.py 93.0% 39,47,57,62,68
hyper_parallel/components/modules/attention/multi_latent_attention/__init__.py 100%  
hyper_parallel/components/modules/attention/multi_latent_attention/cp_mla_backward_overlap.py 73.5% 45,57-59,69,89,93-95
hyper_parallel/components/modules/attention/multi_latent_attention/cp_multi_latent_attention.py 68.2% 440,442,475,507,577,579-580,618,623,626,632-633,635-636,640-642,646,648-649,653,659,661,663,665-667,670-671,674,676,681,683,688-689,691,693,709,712,715,718,731-733,738-739,750,752,756
hyper_parallel/components/modules/attention/multi_latent_attention/multi_latent_attention.py 88.9% 81,94,154,158,174,186,191,199,210,215,249,276,278,291,297,317-319,328-331,422-423,426,527,530
hyper_parallel/components/modules/attention/multi_latent_attention/multi_latent_attention_family.py 92.2% 39,54,64,66,68,111
hyper_parallel/components/modules/family.py 66.7% 157
hyper_parallel/components/modules/rms_norm.py 50.0% 131
hyper_parallel/data/batching/runtime_input.py 80.8% 203,208,211,215,222
hyper_parallel/distributed/_builder/applier.py 91.7% 364
hyper_parallel/distributed/_builder/family_layout.py 83.3% 126,134,139,145
hyper_parallel/distributed/context_parallel/cp_context.py 61.5% 99,110,119,122-123
hyper_parallel/distributed/context_parallel/cp_types.py 100%  
hyper_parallel/distributed/module_binding.py 83.3% 140,145
hyper_parallel/trainer/base.py 100%  
hyper_parallel/trainer/config/resolver.py 100%  
hyper_parallel/codegen/inline/framework_spec.py
259
260
261
262
263
264
265
266
267
        return None
    if get_module_family(component) is not None:
        # A thin source-model adapter may delegate to a registered family.
        # Preserve that request so later TP/CP promotion does not freeze the local class.
        return ReplacementSpec(
            old_ctor=old_ctor, new_ctor=target.component,
            imports=(ImportPatch(target.module, (target.component,)),),
            mode="wrap_source", keyword_args=keyword_args, remove_class=False, defer_to_fqns=True,
        )
358
359
360
361
362
363
364
365
366
367
368
369
370
def _module_family_replacement_spec(target: str, module_type: str | None) -> ReplacementSpec | None:
    """Retain registered module-family construction instead of freezing its concrete class."""
    module_name, _, name = target.replace(":", ".", 1).rpartition(".")
    if not module_name or not name:
        return None
    try:
        factory = getattr(importlib.import_module(module_name), name, None)
    except ImportError:
        return None
    if get_module_family(factory) is None or not class_name_of(module_type):
        return None
    return ReplacementSpec(
        old_ctor=class_name_of(module_type), new_ctor=name,
hyper_parallel/codegen/inline/observations.py
61
62
63
64
65
66
67
68
    arguments = []
    for name, value in sorted((record.get("target_config") or {}).items()):
        text = repr(value)
        if not name.isidentifier() or ast.literal_eval(text) != value:
            raise ValueError(f"codegen cannot preserve construction argument {name!r}")
        arguments.append(f"{name}={text}")
    return tuple(arguments)

hyper_parallel/components/losses/model_output.py
54
55
56
57
58
59
60
61
62
63
64
        # again can erase the only supervised token on a CP shard. Trainer owns
        # target alignment, so use its count whenever it is available.
        if valid_token_count is not None:
            has_valid_labels = valid_token_count > 0
        elif labels is not None:
            has_valid_labels = labels[..., 1:].ne(IGNORE_INDEX).any()
        else:
            return local_loss
        local_loss = torch.where(has_valid_labels, local_loss, torch.zeros_like(local_loss))

        return local_loss
hyper_parallel/components/modules/attention/gated_grouped_query_attention/gated_grouped_query_attention.py
34
35
36
37
38
39
40
41
42

    def _initialize_dimensions(self, module: nn.Module | None) -> None:
        """Extend ordinary GQA dimensions by one gate value per query channel."""
        if not self.fuse_qkv:
            raise ValueError("Gated GQA currently requires fused Q/K/V/gate projections")
        super()._initialize_dimensions(module)
        if self.qk_head_dim != self.v_head_dim:
            raise ValueError("GatedGroupedQueryAttention requires the gate width to match the attention output width")
        self.gated_qkv_group_width = self.qkv_group_width + self.num_key_value_groups * self.qk_head_dim
49
50
51
52
53
54
55
56
57
    def _validate_source_projections(self, module: nn.Module) -> None:
        """Validate separate gated-query, key, value, and output projections."""
        projections = (module.q_proj, module.k_proj, module.v_proj)
        if any(not isinstance(projection, nn.Linear) for projection in projections):
            raise TypeError("q_proj, k_proj, and v_proj must be torch.nn.Linear")
        expected_sizes = (
            2 * self.num_heads * self.qk_head_dim,
            self.num_key_value_heads * self.qk_head_dim,
            self.num_key_value_heads * self.v_head_dim,
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
            self.num_key_value_heads * self.qk_head_dim,
            self.num_key_value_heads * self.v_head_dim,
        )
        if tuple(projection.out_features for projection in projections) != expected_sizes:
            raise ValueError(
                "Q/gate, K, and V projection sizes are incompatible with the configured gated GQA dimensions"
            )
        if not _projections_can_fuse(projections):
            raise ValueError("Q/gate, K, and V projections cannot be represented by one fused projection")
        attention_output_size = self.num_heads * self.v_head_dim
        if not isinstance(module.o_proj, nn.Linear):
            raise TypeError("o_proj must be torch.nn.Linear")
        if module.o_proj.in_features != attention_output_size:
            raise ValueError("o_proj input size is incompatible with the configured gated GQA dimensions")

    def _initialize_projection(self, module: nn.Module) -> None:
        """Create the fused gated-QKV projection and attach source output layers."""
        self.linear_qkv = nn.Linear(
89
90
91
92
93
94
95
96
97
        config = self.config
        self.scaling = getattr(module, "scaling", self.qk_head_dim**-0.5)
        self.attention_dropout = getattr(module, "attention_dropout", 0.0)
        if isinstance(self.attention_dropout, nn.Dropout):
            self.attention_dropout = self.attention_dropout.p
        self.is_causal = getattr(module, "is_causal", True)
        self.sliding_window = getattr(module, "sliding_window", getattr(config, "sliding_window", None))
        self.rotary_interleaved = getattr(
            module, "rotary_interleaved",
100
101
102
103
104
105
106
107

    def _source_qkv(self, module: nn.Module) -> tuple:
        """Validate the per-query-head source gate layout before fusion."""
        if any(not hasattr(module, name) for name in ("q_proj", "k_proj", "v_proj", "o_proj")):
            raise TypeError("Gated GQA requires separate Q/gate, K, V and output projections")
        self._validate_source_projections(module)
        self._source_is_fused = False
        return module.q_proj, ("q_proj", "k_proj", "v_proj")
hyper_parallel/components/modules/attention/gated_multi_latent_attention/cp_gated_mla_backward_overlap.py
40
41
42
43
44
45
46
47
    def __init__(self, source: nn.Linear) -> None:
        """Preserve the source parameter tree and train/freeze state."""
        nn.Module.__init__(self)
        if type(source) is not nn.Linear and not isinstance(source, CPGatedMLAOutputLinear):  # pylint: disable=unidiomatic-typecheck
            raise ValueError("Gated MLA scheduled output requires a plain Linear")
        self.in_features, self.out_features = source.in_features, source.out_features
        self.weight, self.bias = source.weight, source.bias
        self.train(source.training)
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
            attention_output: Local or head-partitioned attention output.
            mla_output_state: Invocation-local layout and optional gate-gradient handoff.
            mla_output_gate: Sigmoid gate paired with a scheduled CP output.
        """
        if mla_output_state is None:
            if mla_output_gate is not None:
                raise ValueError("Scheduled gated output requires a state")
            return F.linear(attention_output, self.weight, self.bias)
        if mla_output_gate is None:
            raise ValueError("Scheduled gated output requires its sigmoid gate")
        return _GatedOutputProjection.apply(attention_output, self.weight, self.bias, mla_output_gate, mla_output_state)


class GatedMLAInputState:
    """Invocation-local gate values and the optional deferred native derivative.
hyper_parallel/components/modules/attention/gated_multi_latent_attention/cp_gated_multi_latent_attention.py
48
49
50
51
52
53
54
55
56

    def prepare_output_projection(self) -> None:
        """Install the existing gated output derivative inside its managed Linear."""
        if self.cp_plan.tp_degree != 1:
            raise ValueError("Gated MLA CP currently supports TP1 only")
        if self.cp_plan.overlap and self.cp_plan.ulysses_degree > 1:
            self.o_proj = CPGatedMLAOutputLinear(self.o_proj)

    def _prepare_input_gate(self, hidden_states: torch.Tensor) -> GatedMLAInputState | None:
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
            self.o_proj = CPGatedMLAOutputLinear(self.o_proj)

    def _prepare_input_gate(self, hidden_states: torch.Tensor) -> GatedMLAInputState | None:
        """Reserve a gate in the expanded pure-Ulysses input window, when enabled."""
        plan = self.cp_plan
        if not plan.overlap or plan.ulysses_degree <= 1:
            return None
        if plan.strategy != "expanded_ulysses" or plan.all_gather_degree != 1:
            return None
        # Frozen inputs keep native gate backward; the paired consumer otherwise
        # releases it after the input gradient collectives have been submitted.
        defer_gate = self._overlap_output_backward() and hidden_states.requires_grad
        return GatedMLAInputState(hidden_states, self.project_gate, defer_gate)

    def prepare_attention_inputs(
        self, hidden_states: torch.Tensor,
    ) -> tuple[MLALatentInputs, GatedMLAInputState | None]:
75
76
77
78
79
80
81
82
83
84
85

        Returns:
            Normalized latents and an optional gate state for expanded pure Ulysses.
        """
        input_gate = self._prepare_input_gate(hidden_states)
        state, _ = super().prepare_attention_inputs(hidden_states)
        return state, input_gate

    def finish_input_exchange(self, query: torch.Tensor, transfers: Any, overlap_backward: bool,
                              input_gate: Any) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
        """Place algorithm-owned gate computation between Q submission and input waits.
89
90
91
92
93
94
95
96
97
98
99
            transfers: Pending input collective state shared with paired backward nodes.
            overlap_backward: Whether to retain the paired input adjoints.
            input_gate: This invocation's deferred gate, or None for ordinary scheduling.
        """
        if input_gate is None:
            return super().finish_input_exchange(query, transfers, overlap_backward, None)
        return finish_gated_expanded_inputs(query, transfers, input_gate)

    def restore_attention(self, output: torch.Tensor, hidden_states: torch.Tensor,
                          output_state: GatedMLAInputState | None) -> tuple[torch.Tensor, CPGatedMLAOutputState]:
        """Schedule token restoration and gate readiness before the managed output projection.
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
        Returns:
            Restored output, or raw FA output when joint backward requires deferred
            restoration; the typed state carries the gate and the deferred operation.
        """
        input_gate = output_state
        output_a2a_handle = None
        if input_gate is None and self.cp_plan.overlap and self.cp_plan.ulysses_degree > 1:
            # The gate remains a native child-module call, with its original
            # FSDP hooks/autograd, while O communication is in flight.
            output_a2a_handle = async_ulysses_seq_to_head_launch(output, 2, 1, self.ulysses_mesh)
        if input_gate is None:
            gate = self.project_gate(hidden_states)
        else:
            gate, input_gate.value = input_gate.value, None
        if self._overlap_output_backward():
            deferred_gate = input_gate if input_gate is not None and input_gate.defer_gate_gradient else None
            state = GatedMLAOutputProjectionState(
                self.ulysses_mesh, self.v_head_dim, input_gate=deferred_gate, forward_a2a_handle=output_a2a_handle,
            )
            return output, CPGatedMLAOutputState(gate, state)
        if output_a2a_handle is not None:
            output = output_a2a_handle.wait()
        else:
            output, _ = super().restore_attention(output, hidden_states, None)
        return output, CPGatedMLAOutputState(gate)

    def project_output(self, attention_output: torch.Tensor, hidden_states: torch.Tensor,
                       output_state: CPGatedMLAOutputState) -> torch.Tensor:
        """Apply the ready gate through the algorithm or the existing joint output Function.
137
138
139
140
141
142
143
144
            attention_output: Restored token-local output, or raw FA output for the joint Function.
            hidden_states: Original local tokens defining the output shape.
            output_state: Gate and optional O-exchange/Wo state produced by restore_attention.
        """
        if output_state.projection is not None:
            return self.o_proj(attention_output, mla_output_state=output_state.projection,
                               mla_output_gate=output_state.gate)
        return super().project_output(attention_output, hidden_states, output_state.gate)
hyper_parallel/components/modules/attention/gated_multi_latent_attention/gated_multi_latent_attention.py
49
50
51
52
53
54
55
56
57
58
59
60
                bias=getattr(config, "attention_bias", False), dtype=getattr(config, "dtype", None),
            )
            return
        if not getattr(module, "use_nope", False):
            raise ValueError("GatedMultiLatentAttention requires a NoPE source module")
        if not isinstance(module, GatedMultiLatentAttentionModule) and not getattr(module, "use_output_gate", False):
            raise ValueError("GatedMultiLatentAttention requires an enabled output gate")
        if type(module.g_proj) is not nn.Linear:  # pylint: disable=unidiomatic-typecheck
            raise ValueError("Gated MLA requires a plain Linear output gate")
        super().__init__(module=module, config=config)
        self.use_nope = True
        self.g_proj = module.g_proj
71
72
73
74
75
76
77
78
79
80

        Returns:
            The original tensor, without a copy or a positional transform.
        """
        del position_embeddings, interleaved
        return tensor

    def apply_position(
        self, tensor: torch.Tensor, position_embeddings: tuple[torch.Tensor, torch.Tensor] | None,
        *, interleaved: bool | None = None,
 98
 99
100
101
102
103
104
105

        Args:
            position_embeddings: Ignored positional input from the model caller.
        """
        del position_embeddings

    def project_gate(self, hidden_states: torch.Tensor) -> torch.Tensor:
        """Compute the mandatory token-local output gate through its managed Linear.
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120

        Args:
            hidden_states: Original local input in BSH layout.
        """
        return self.g_proj(hidden_states).sigmoid()

    def reset_parameters(self) -> None:
        """Initialize direct-config projections and gate, preserving source initialization."""
        if self._config_initialization:
            super().reset_parameters()
            self.g_proj.reset_parameters()

    def project_output(self, attention_output: torch.Tensor, hidden_states: torch.Tensor,
                       output_state: torch.Tensor | None) -> torch.Tensor:
        """Apply the source gate before Wo, without an additional output Norm.
126
127
128
129
130
131
132

        Returns:
            Gated and projected hidden states.
        """
        gate = self.project_gate(hidden_states) if output_state is None else output_state
        attention_output = attention_output.reshape(*hidden_states.shape[:2], -1)
        return self.o_proj(attention_output * gate)
hyper_parallel/components/modules/attention/grouped_query_attention/cp_grouped_query_attention.py
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88

def _local_mask(mask: torch.Tensor, global_length: int, query_length: int,
                query_begin: int, key_end: int) -> torch.Tensor:
    """Slice global Q/K masks; two-dimensional non-square masks are batch padding."""
    if mask.shape[-1] != global_length:
        raise ValueError("GQA CP mask must cover the global key sequence")
    if mask.ndim == 2 and mask.shape == (global_length, global_length):
        mask = mask[None, None]
    if mask.ndim >= 3 and mask.shape[-2] not in (1, query_length):
        if mask.shape[-2] != global_length:
            raise ValueError("GQA CP query mask must cover the local or global query interval")
        mask = mask[..., query_begin:query_begin + query_length, :]
    return mask[..., :key_end]


def _valid_query_rows(mask: torch.Tensor | None, batch: int, global_length: int,
                      query_length: int, query_rank: int) -> torch.Tensor | None:
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
    The common FA can infer padded queries from a self-attention mask only
    when Q/K cover the same tokens. Here the CP rank owns a query interval
    inside global KV; slice its rows explicitly after FA.
    """
    if mask is None or mask.ndim != 2 or mask.shape == (global_length, global_length):
        return None
    if mask.shape != (batch, global_length):
        raise ValueError("GQA CP padding mask must have shape [batch, global_sequence]")
    begin = query_rank * query_length
    return mask[:, begin:begin + query_length].to(torch.bool)


@dataclass(frozen=True)
class CPGQAPreparedInputs:
147
148
149
150
151
152
153
154
155
        self.ulysses_degree = ulysses_degree

    def parallel_module_spec(self) -> ParallelModuleSpec:
        """Request borrowed CP resources in addition to the shared TP boundary."""
        return ParallelModuleSpec("attention.gqa.cp", "cp",
                                  {"prepare_attention_inputs": ("hidden_states", "positions"),
                                   "exchange_attention": ("prepared",),
                                   "restore_attention": ("output", "output_state")},
                                  validate=validate_gqa_layout)
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177

        Args:
            binding: Final module layout and the existing job's TP/CP resources.
        """
        if binding.cp_context is None:
            raise ValueError("GQA CP requires a bound CPContext")
        self.cp_context = binding.cp_context
        degree = self.cp_context.size
        self.ulysses_degree = degree if self.ulysses_degree is None else self.ulysses_degree
        if self.ulysses_degree < 1 or degree % self.ulysses_degree:
            raise ValueError("GQA Ulysses degree must divide CP degree")
        if self.num_heads % self.ulysses_degree or self.num_key_value_heads % self.ulysses_degree:
            raise ValueError("GQA Ulysses degree must divide TP-local Q and KV heads; use KV AG or a smaller U")
        self.ulysses_context, self.query_context = self.cp_context.split(self.ulysses_degree)
        self.all_gather_degree = degree // self.ulysses_degree

    def _transfer_input(self, tensor: torch.Tensor, *, is_query: bool) -> tuple[torch.Tensor, Any]:
        """Keep Q local for KV AG; otherwise exchange this ready operand along U."""
        if self.ulysses_degree > 1:
176
177
178
179
180
181
182
183
184
185
186
187
        """Keep Q local for KV AG; otherwise exchange this ready operand along U."""
        if self.ulysses_degree > 1:
            if self.overlap:
                return tensor, self.ulysses_context.launch_sequence_to_head(tensor)
            return self.ulysses_context.sequence_to_head(tensor), None
        if self.overlap and self.all_gather_degree > 1 and not is_query:
            return tensor, self.query_context.launch_gather_sequence(tensor).collective
        return tensor, None

    def _finish_input_transfers(self, query: torch.Tensor, key: torch.Tensor, value: torch.Tensor,
                                query_pending: Any, key_pending: Any, value_pending: Any) -> tuple:
        """Preserve hybrid K/V early submission before waiting for Q."""
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
                if self.all_gather_degree > 1:
                    key_gather = self.query_context.launch_gather_sequence(key_pending.wait())
                    value_gather = self.query_context.launch_gather_sequence(value_pending.wait())
                    return query_pending.wait(), key_gather.wait()[0], value_gather.wait()[0]
                return query_pending.wait(), key_pending.wait(), value_pending.wait()
            if self.all_gather_degree > 1:
                return query, key_pending.wait(), value_pending.wait()
        elif self.all_gather_degree > 1:
            key = self.query_context.gather_sequence(key)[0]
            value = self.query_context.gather_sequence(value)[0]
        return query, key, value

    def _attention_operands(self, query: torch.Tensor, key: torch.Tensor, value: torch.Tensor,
                            attention_mask: torch.Tensor | None, lengths: Any, options: dict,
                            output_state: Any) -> GQAInputs:
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
    def _attention_operands(self, query: torch.Tensor, key: torch.Tensor, value: torch.Tensor,
                            attention_mask: torch.Tensor | None, lengths: Any, options: dict,
                            output_state: Any) -> GQAInputs:
        """Convert global packed metadata into the actual FA query/KV document fragments."""
        batch, query_length = query.shape[:2]
        global_length = key.shape[1]
        query_rank = self.query_context.rank if self.all_gather_degree > 1 else 0
        causal = options.pop("is_causal", self.is_causal)
        window = options.pop("sliding_window", self.sliding_window)
        packed_options = {}
        valid_queries = _valid_query_rows(attention_mask, batch, global_length, query_length, query_rank)
        if lengths is not None and self.all_gather_degree == 1:
            # Pure Ulysses already owns the complete sequence. Do not repack KV
            # by document: FA can consume the original batch-major TND directly.
            packed_options = {"actual_seq_len": lengths}
        elif lengths is not None or (self.all_gather_degree > 1 and attention_mask is None):
            if attention_mask is not None:
                raise ValueError("GQA CP packed attention requires document lengths without an explicit mask")
            documents = tuple(lengths or range(global_length, (batch + 1) * global_length, global_length))
            ranges, q_lengths, kv_lengths = _query_documents(
                documents, batch, global_length, query_length, query_rank, causal)
            query = query.flatten(0, 1).unsqueeze(0)
            key, value = (_select_document_keys(tensor, ranges) for tensor in (key, value))
            packed_options = {"actual_seq_qlen": q_lengths, "actual_seq_kvlen": kv_lengths}
        elif attention_mask is not None:
            query_begin = query_rank * query_length
            # Prefix truncation lets the common FA use its ordinary right-aligned causal contract.
            key_end = query_begin + query_length if causal else global_length
            key, value = key[:, :key_end], value[:, :key_end]
            attention_mask = _local_mask(attention_mask, global_length, query_length, query_begin, key_end)
            if query_length != key_end:
                # The explicit mask already includes the global query offset.
                # Kernel-local next_tokens=0 can incorrectly clip unequal-Q/K backward.
                options.setdefault("next_tokens", 2147483647)
        options.update(is_causal=causal, sliding_window=window, **packed_options)
        output_state = CPGQAOutputState((batch, query_length), valid_queries, output_state)
        return GQAInputs(
            query.transpose(1, 2), key.transpose(1, 2), value.transpose(1, 2),
            attention_mask, options, output_state)

    def prepare_positions(self, query_shape: tuple[int, int], positions: Any) -> Any:
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
        Args:
            query_shape: Batch and local sequence dimensions before exchange.
            positions: Local or global cosine/sine tables, or None.
        """
        if not hasattr(self, "cp_context"):
            raise RuntimeError("Bind GQA CP execution before forward")
        if positions is None:
            return None
        local_length = query_shape[1]
        global_length = local_length * self.cp_context.size
        parts = []
        for part in positions:
            if part.shape[-2] == global_length:
                part = part.narrow(-2, self.cp_context.rank * local_length, local_length)
            elif part.shape[-2] != local_length:
                raise ValueError("GQA CP positions must cover the local or global token sequence")
            parts.append(part)
        return tuple(parts)

    def prepare_attention_inputs(self, hidden_states: torch.Tensor, positions: Any,
                                 attention_mask: torch.Tensor | None = None, past_key_values: Any = None,
                                 actual_seq_len: Any = None, **kwargs: Any) -> CPGQAPreparedInputs:
284
285
286
287
288
289
290
291
292
        """
        # Cache updates change KV ownership between calls. The training exchange
        # assumes a fixed global sequence, so reject caches before any collective.
        if past_key_values is not None:
            raise ValueError("GQA CP training does not support a KV cache")
        batch, local_length = hidden_states.shape[:2]
        global_length = local_length * self.cp_context.size
        options = dict(kwargs, actual_seq_len=actual_seq_len)
        q_lengths, k_lengths = resolve_packed_sequence_lengths(
291
292
293
294
295
296
297
298
299
300
301
        options = dict(kwargs, actual_seq_len=actual_seq_len)
        q_lengths, k_lengths = resolve_packed_sequence_lengths(
            options, query_tokens=batch * global_length, key_tokens=batch * global_length)
        if q_lengths != k_lengths:
            raise ValueError("GQA CP self-attention requires matching global query/KV document lengths")
        if q_lengths is not None and attention_mask is not None:
            raise ValueError("GQA CP packed lengths cannot be combined with an explicit mask")
        options = {name: value for name, value in kwargs.items() if name not in PACKED_SEQUENCE_ARGUMENTS}
        if self.fuse_qkv:
            query, key, value, output_state = self.split_projection(hidden_states)
            query = self.normalize_position(query, self.q_norm, positions)
342
343
344
345
346
347
348
349
350
351
352
353
354

        Returns:
            Token-local output and only the algorithm's optional gate, without CP metadata.
        """
        output = output.reshape(*output_state.query_shape, output.shape[-2], output.shape[-1])
        if output_state.valid_queries is not None:
            # Batched training may pad shorter samples, e.g. lengths [5, 8] in B2/S8.
            # Prefix KV selection makes Q/K lengths unequal, so zero the original
            # padded query rows explicitly; this is not communication/head padding.
            output = output.masked_fill(~output_state.valid_queries[:, :, None, None], 0)
        if self.ulysses_degree > 1:
            output = self.ulysses_context.head_to_sequence(output)
        return output, output_state.projection_state
hyper_parallel/components/modules/attention/grouped_query_attention/grouped_query_attention.py
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63

def apply_gqa_position(tensor: torch.Tensor, positions: Any, interleaved: bool) -> torch.Tensor:
    """Rotate one BSND branch, retaining the source half-band/interleaved convention."""
    if positions is None:
        return tensor
    cos, sin = positions
    if cos.ndim == 2:
        cos, sin = cos.unsqueeze(0), sin.unsqueeze(0)
    rotary_dim = cos.shape[-1]
    if rotary_dim > tensor.shape[-1] or rotary_dim % 2:
        raise ValueError("GQA rotary dimension must be even and fit the head dimension")
    # Keep partial-width half-layout rotation on its established eager path.
    if not interleaved and rotary_dim < tensor.shape[-1]:
        rotary, tail = tensor.split((rotary_dim, tensor.shape[-1] - rotary_dim), dim=-1)
        return torch.cat((rotary * cos.unsqueeze(2) + rotate_half(rotary) * sin.unsqueeze(2), tail), dim=-1)
61
62
63
64
65
66
67
68
69
    if not interleaved and rotary_dim < tensor.shape[-1]:
        rotary, tail = tensor.split((rotary_dim, tensor.shape[-1] - rotary_dim), dim=-1)
        return torch.cat((rotary * cos.unsqueeze(2) + rotate_half(rotary) * sin.unsqueeze(2), tail), dim=-1)
    if interleaved:
        cos, sin = (part[..., :rotary_dim // 2].repeat_interleave(2, dim=-1) for part in (cos, sin))
    cos, sin = (part.expand(tensor.shape[0], -1, -1).transpose(0, 1) for part in (cos, sin))
    result = apply_rotary_pos_emb_single(tensor.transpose(0, 1), cos, sin,
                                       rotary_interleaved=interleaved, use_fused_rotary_pos_emb=True)
    result = result.transpose(0, 1)
67
68
69
70
71
72
73
74
75
76
77
    result = apply_rotary_pos_emb_single(tensor.transpose(0, 1), cos, sin,
                                       rotary_interleaved=interleaved, use_fused_rotary_pos_emb=True)
    result = result.transpose(0, 1)
    if interleaved:
        rotary, tail = result.split((rotary_dim, result.shape[-1] - rotary_dim), dim=-1)
        rotary = rotary.unflatten(-1, (-1, 2)).transpose(-1, -2).flatten(-2)
        result = torch.cat((rotary, tail), dim=-1) if tail.shape[-1] else rotary
    return result


@dataclass(frozen=True)
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
        self.num_heads = config.num_attention_heads
        self.num_key_value_heads = config.num_key_value_heads
        dimensions = (config.hidden_size, self.num_heads, self.num_key_value_heads)
        if any(not isinstance(value, int) or isinstance(value, bool) or value <= 0 for value in dimensions):
            raise ValueError("GQA hidden size and head counts must be positive integers")
        if self.num_heads % self.num_key_value_heads:
            raise ValueError("num_attention_heads must be divisible by num_key_value_heads")
        self.num_key_value_groups = self.num_heads // self.num_key_value_heads
        self.qk_head_dim = getattr(module, "head_dim", getattr(config, "head_dim", None))
        if self.qk_head_dim is None:
            if config.hidden_size % self.num_heads:
                raise ValueError("GQA requires an explicit head_dim when hidden_size is not divisible by query heads")
            self.qk_head_dim = config.hidden_size // self.num_heads
        self.v_head_dim = getattr(module, "v_head_dim", getattr(config, "v_head_dim", None))
        if self.v_head_dim is None:
            self.v_head_dim = self.qk_head_dim
132
133
134
135
136
137
138
139
140
        config = self.config
        self.scaling = getattr(module, "scaling", self.qk_head_dim**-0.5)
        self.attention_dropout = getattr(module, "attention_dropout", 0.0)
        if isinstance(self.attention_dropout, nn.Dropout):
            self.attention_dropout = self.attention_dropout.p
        self.is_causal = getattr(module, "is_causal", True)
        self.sliding_window = getattr(module, "sliding_window", getattr(config, "sliding_window", None))
        self.rotary_interleaved = getattr(
            module,
143
144
145
146
147
148
149
150
151
152
153
154
155
        )
        # Reject unsupported source mathematics rather than dropping its sink,
        # normalization or gate while converting the checkpoint.
        if getattr(module, "param_sink_number", 0) > 0:
            raise ValueError(
                "GroupedQueryAttention only implements standard GQA; parameter-sink attention "
                "must use the sink_attention high-performance interface."
            )
        if getattr(module, "attn_groupnorm", False) or getattr(module, "attn_elementwise_gate", False):
            raise ValueError(
                "GroupedQueryAttention only implements standard GQA; group normalization and "
                "elementwise attention gates require a dedicated replacement."
            )
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
    def _source_qkv(self, module: nn.Module) -> tuple[nn.Linear, tuple[str, ...]]:
        """Validate and return the source QKV projection layout."""
        self._source_is_fused = hasattr(module, "qkv_proj")
        if self._source_is_fused:
            if not self.fuse_qkv:
                raise ValueError("fuse_qkv=False requires separate Q/K/V source projections")
            source_qkv = module.qkv_proj
            if not isinstance(source_qkv, nn.Linear):
                raise TypeError("qkv_proj must be torch.nn.Linear")
            expected_size = (
                self.num_heads * self.qk_head_dim
                + self.num_key_value_heads * self.qk_head_dim
                + self.num_key_value_heads * self.v_head_dim
            )
            if source_qkv.out_features != expected_size:
                raise ValueError("qkv_proj output size is incompatible with the configured GQA dimensions")
            return source_qkv, ("qkv_proj",)
        required = ("q_proj", "k_proj", "v_proj")
        if any(not hasattr(module, name) for name in required):
            raise TypeError("GroupedQueryAttention requires qkv_proj or q_proj/k_proj/v_proj on the source module")
        projections = tuple(getattr(module, name) for name in required)
        if any(not isinstance(projection, nn.Linear) for projection in projections):
            raise TypeError("q_proj, k_proj, and v_proj must be torch.nn.Linear")
        expected_sizes = (
            self.num_heads * self.qk_head_dim,
            self.num_key_value_heads * self.qk_head_dim,
            self.num_key_value_heads * self.v_head_dim,
182
183
184
185
186
187
188
189
190
191
            self.num_key_value_heads * self.qk_head_dim,
            self.num_key_value_heads * self.v_head_dim,
        )
        if tuple(projection.out_features for projection in projections) != expected_sizes:
            raise ValueError("Q/K/V projection sizes are incompatible with the configured GQA dimensions")
        if len({projection.in_features for projection in projections}) != 1:
            raise ValueError("Q/K/V projections must accept the same hidden width")
        if self.fuse_qkv and not _projections_can_fuse(projections):
            raise ValueError("Q, K, and V projections cannot be represented by one fused projection")
        return module.q_proj, required
192
193
194
195
196
197
198
199
200
201
202

    def _initialize_projections(self, module: nn.Module, source_qkv: nn.Linear) -> None:
        """Create fused rows or retain the independent source projection boundaries."""
        if not isinstance(getattr(module, "o_proj", None), nn.Linear):
            raise TypeError("GroupedQueryAttention requires o_proj on the source module")
        if module.o_proj.in_features != self.num_heads * self.v_head_dim:
            raise ValueError("o_proj input size is incompatible with the configured GQA dimensions")
        if not self.fuse_qkv:
            self._initialize_separate_projections(module)
            return
        projection_size = self.num_key_value_heads * self.qkv_group_width
222
223
224
225
226
227
228
229
230
231
232
            norm = getattr(module, name, None)
            # The former three-projection Qwen path used fused direct-scale RMSNorm.
            # Retain that kernel as a real child Module so FSDP hooks are not bypassed.
            if norm is not None and hasattr(norm, "variance_epsilon"):
                if set(dict(norm.named_parameters())) != {"weight"}:
                    raise ValueError("GQA direct-scale RMSNorm requires only a weight parameter")
                norm = RMSNorm(module=norm)
            setattr(self, name, norm)

    def __init__(self, *, config: Any = None, module: nn.Module | None = None,
                 fuse_qkv: bool = True) -> None:
238
239
240
241
242
243
244
245
246
247
248
249
            fuse_qkv: Use grouped linear_qkv rows or independent Q/K/V children.
        """
        super().__init__()
        if not isinstance(fuse_qkv, bool):
            raise TypeError("GQA fuse_qkv must be boolean")
        self.fuse_qkv = fuse_qkv
        if (config is None) == (module is None):
            raise ValueError("GroupedQueryAttention requires exactly one of config or module")
        self._config_initialization = module is None or getattr(module, "_config_initialization", False)
        self.attention_interface = npu_fusion_attention_forward
        self.config = config if module is None else module.config
        source = config if module is None else module
255
256
257
258
259
260
261
262
263
        if isinstance(module, GroupedQueryAttentionModule):
            if self.fuse_qkv != module.fuse_qkv:
                raise ValueError("GQA promotion cannot change the fused/split parameter representation")
            if hasattr(self, "gated_qkv_group_width") != hasattr(module, "gated_qkv_group_width"):
                raise ValueError("GQA promotion cannot add or remove query gate parameters")
            # Promotion happens before sharding: retain weights, FQNs and conversion provenance.
            for name in ("_source_is_fused", "_source_projection_names"):
                setattr(self, name, getattr(module, name, None))
            # Preserve registration order and aliases as well as parameter names.
265
266
267
268
269
270
271
272
273
                if name and "." not in name:
                    self.add_module(name, child)
            for name in ("q_norm", "k_norm"):
                if not hasattr(self, name):
                    setattr(self, name, getattr(module, name, None))
            self._initialize_attention_options(module)
            self.train(module.training)
            return
        self._initialize_attention_options(module)
285
286
287
288
289
290
291
292
293
        if any(not isinstance(value, bool) for value in (bias, output_bias, qk_norm)):
            raise ValueError("GQA attention_bias, o_proj_bias and qk_norm must be boolean")
        eps = getattr(config, "rms_norm_eps", 1e-6)
        if qk_norm and (not isinstance(eps, (int, float)) or isinstance(eps, bool) or not eps > 0):
            raise ValueError("GQA rms_norm_eps must be positive when qk_norm is enabled")
        dtype = getattr(config, "dtype", None)
        self._source_is_fused = False
        self._source_projection_names = ()
        self._initialize_config_projections(config, bias, dtype)
311
312
313
314
315
316
317
318
319

    def reset_parameters(self) -> None:
        """Initialize native/meta parameters without resetting converted source weights."""
        if not self._config_initialization:
            return
        for name in ("linear_qkv", "q_proj", "k_proj", "v_proj", "o_proj"):
            projection = getattr(self, name, None)
            if projection is not None:
                projection.reset_parameters()
447
448
449
450
451
452
453
454
455
        """
        query, key, value, output_state = self.project_qkv(hidden_states, positions)
        query, key, value = (tensor.transpose(1, 2) for tensor in (query, key, value))
        if past_key_values is not None:
            key, value = past_key_values.update(key, value, self.layer_idx)
        if attention_mask is not None and attention_mask.ndim == 4:
            # HF cache masks may reserve more KV slots than the cache currently exposes.
            attention_mask = attention_mask[..., :key.shape[2]]
        return GQAInputs(query, key, value, attention_mask,
hyper_parallel/components/modules/attention/grouped_query_attention/grouped_query_attention_family.py
35
36
37
38
39
40
41
42
43
    """Reject TP degrees that split KV groups before constructing parameters."""
    del context
    config = config if module is None else module.config
    if config is None:
        raise ValueError("GQA requires configuration")
    heads = (config.num_attention_heads, config.num_key_value_heads)
    if any(not isinstance(value, int) or isinstance(value, bool) or value <= 0 for value in heads):
        raise ValueError("GQA head counts must be positive integers")
    if not isinstance(member.arguments.get("fuse_qkv", True), bool):
43
44
45
46
47
48
49
50
51
    if not isinstance(member.arguments.get("fuse_qkv", True), bool):
        raise TypeError("GQA fuse_qkv must be boolean")
    # Declarative replacement requests omit defaults, unlike a direct Python factory call.
    if not isinstance(member.arguments.get("overlap", False), bool):
        raise TypeError("GQA overlap must be boolean")
    _validate_gqa_degrees(config, member)


def _validate_gqa_degrees(config, member):
53
54
55
56
57
58
59
60
61
62
63
64
65
    degree = member.arguments.get("ulysses_degree")
    cp = member.axis_sizes["cp"]
    degree = cp if degree is None else degree
    if (not isinstance(degree, int) or isinstance(degree, bool)) or degree < 1 or (cp > 1 and cp % degree):
        raise ValueError("GQA Ulysses degree must be a positive integer dividing CP degree")
    tp = member.axis_sizes["tp"]
    if config.num_attention_heads % tp:
        raise ValueError("GQA query heads must be divisible by TP degree")
    if config.num_key_value_heads % tp:
        raise ValueError(
            "GQA KV heads must be divisible by TP degree for the current whole-KV-group layout; "
            "TP KV replication is not implemented"
        )
64
65
66
67
68
69
70
71
72
            "TP KV replication is not implemented"
        )

    if cp > 1 and (config.num_attention_heads // tp % degree or config.num_key_value_heads // tp % degree):
        raise ValueError("GQA Ulysses degree must divide TP-local Q and KV heads")

def gqa_sharding(*, module: nn.Module, module_fqn: str, member: Any, context: Any) -> ModuleShardingSpec:
    """Own all GQA parameters and declare the actual external token layout."""
    del module_fqn, member
hyper_parallel/components/modules/attention/multi_latent_attention/cp_mla_backward_overlap.py
41
42
43
44
45
46
47
48
    def __init__(self, source: nn.Linear) -> None:
        """Retain source Parameters without registering an extra child prefix."""
        nn.Module.__init__(self)
        if type(source) is not nn.Linear and not isinstance(source, CPMLAOutputLinear):  # pylint: disable=unidiomatic-typecheck
            raise ValueError("MLA scheduled output requires a plain Linear")
        self.in_features, self.out_features = source.in_features, source.out_features
        self.weight, self.bias = source.weight, source.bias
        self.train(source.training)
53
54
55
56
57
58
59
60
61
62
63
        Args:
            attention_output: Local attention output or head-partitioned CP output.
            mla_output_state: Invocation-local CP layout; absent for an ordinary projection.
        """
        if mla_output_state is None:
            return F.linear(attention_output, self.weight, self.bias)
        return _OutputProjection.apply(attention_output, self.weight, self.bias, mla_output_state)


class CPMLAKVLinear(nn.Linear):
    """Select computation heads inside the managed KV up-projection boundary."""
65
66
67
68
69
70
71
72
73
    def __init__(self, source: nn.Linear, head_dim: int) -> None:
        """Reuse full TP-local parameter storage; CP changes only computation ownership."""
        nn.Module.__init__(self)
        if type(source) is not nn.Linear and not isinstance(source, CPMLAKVLinear):  # pylint: disable=unidiomatic-typecheck
            raise ValueError("MLA KV projection requires a plain Linear")
        self.in_features, self.out_features = source.in_features, source.out_features
        self.weight, self.bias = source.weight, source.bias
        self.head_dim = head_dim
        self.train(source.training)
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
        if mla_head_range is None:
            return F.linear(latent, self.weight, self.bias)
        begin, end = mla_head_range
        if not 0 <= begin < end <= self.weight.shape[0] // self.head_dim:
            raise ValueError("MLA projection head range is outside the managed parameter")
        rows = slice(begin * self.head_dim, end * self.head_dim)
        bias = None if self.bias is None else self.bias[rows]
        if mla_gathered is not None:
            if mla_cp_mesh is None:
                raise ValueError("Scheduled latent projection requires its CP mesh")
            return _LatentKVLinear.apply(latent, self.weight[rows], bias, mla_gathered, mla_cp_mesh)
        return F.linear(latent, self.weight[rows], bias)


@dataclass
hyper_parallel/components/modules/attention/multi_latent_attention/cp_multi_latent_attention.py
436
437
438
439
440
441
442
443
444
445
446
            binding: Framework-owned CP context, including existing process groups.
        """
        context = binding.cp_context
        if context is None:
            raise ValueError("MLA CP requires a bound CPContext")
        if context.size != self.cp_plan.cp_degree:
            raise ValueError("MLACPPlan degree must match the CP mesh")
        self.cp_context, self.cp_mesh = context, context.mesh
        self.query_rank, self.head_rank = divmod(context.rank, self.cp_plan.ulysses_degree)
        self.ulysses_context, self.query_context = context.split(self.cp_plan.ulysses_degree)
        self.ulysses_mesh, self.all_gather_mesh = self.ulysses_context.mesh, self.query_context.mesh
471
472
473
474
475
476
477
478
479
        """
        if past_key_values is not None:
            raise ValueError("MLA CP training does not support a KV cache")
        if self.cp_context is None:
            raise RuntimeError("MLA CP resources must be bound before forward")

    def exchange_attention(self, state: MLALatentInputs,
                           position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None,
                           attention_mask: torch.Tensor | None = None, past_key_values: Any = None,
503
504
505
506
507
508
509
510
        rotated_key = rotary_fn(state.key_rope.unsqueeze(2), positions,
                                interleaved=self.rotary_interleaved).squeeze(2)
        overlap_input = self._overlap_output_backward() and self.cp_plan.all_gather_degree == 1
        if output_state is not None:
            overlap_input = output_state.defer_gate_gradient
        query, key, value = self._prepare_distributed_inputs(
            state, positions, rotated_key, partition, overlap_input, output_state, rotary_fn)
        return MLAInputs(query, key, value, mask, options, output_state)
573
574
575
576
577
578
579
580
581
582
583
584
            )
        if overlap_input_backward:
            # Expanded has returned above and AG/hybrid were rejected earlier:
            # this is latent pure Ulysses, pairing Q A2A with KV projection/RS.
            return self._prepare_latent_ulysses_inputs(state, positions, key_rope, rotary_fn)
        # AG/hybrid inputs and forward-only latent Ulysses use native adjoints.
        prepare = self._expanded_overlap_inputs if expanded else self._latent_overlap_inputs
        return prepare(state, positions, key_rope, partition, rotary_fn=rotary_fn)

    def restore_attention(self, output: torch.Tensor, hidden_states: torch.Tensor,
                          output_state: Any) -> tuple[torch.Tensor, Any]:
        """Restore query ownership, or defer O exchange to the joint output Function.
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
        """Exchange completed expanded operands without performing projection math."""
        if self.cp_plan.ulysses_degree > 1:
            widths = (query.shape[-1], key.shape[-1], value.shape[-1])
            payload = torch.cat(tuple(tensor.transpose(1, 2) for tensor in (query, key, value)), dim=-1)
            payload = self.ulysses_context.sequence_to_head(payload)
            query, key, value = (tensor.transpose(1, 2) for tensor in payload.split(widths, dim=-1))
        if self.cp_plan.all_gather_degree > 1:
            widths = (key.shape[-1], value.shape[-1])
            payload = torch.cat((key.transpose(1, 2), value.transpose(1, 2)), dim=-1)
            if self.cp_plan.kv_exchange == "all_to_all_v":
                payload = _selective_kv_launch(payload, partition, self.all_gather_mesh, 1, False).wait()
            else:
                payload = partition.select(self.query_context.gather_sequence(payload)[0], 1)
            key, value = (tensor.transpose(1, 2) for tensor in payload.split(widths, dim=-1))
        return query, key, value

    def _latent_inputs(self, state, query, key_rope, partition=None):
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
        return query, key, value

    def _latent_inputs(self, state, query, key_rope, partition=None):
        """Exchange Q heads, gather shared KV latents, then expand only owned heads."""
        if self.cp_plan.ulysses_degree > 1:
            query = self.ulysses_context.sequence_to_head(query, seq_dim=2, head_dim=1)
        payload = torch.cat((state.kv_latent, key_rope), dim=-1)
        payload = self._launch_latent_kv(payload, partition, overlap=False).wait()
        return query, *self._expand_latent(payload, partition)

    def _launch_latent_kv(self, payload, partition=None, *, overlap=True):
        """Exchange compressed KV before the algorithm's head-selective expansion."""
        if self.cp_plan.kv_exchange == "all_to_all_v":
            return _selective_kv_launch(payload, partition, self.cp_mesh, self.cp_plan.ulysses_degree, overlap)
        return self.cp_context.launch_gather_sequence(payload).collective

    def _expand_latent(self, payload, partition=None):
        """Keep the same managed head-selective projection for both communication schedules."""
        if partition is not None and self.cp_plan.kv_exchange == "all_gather":
            payload = partition.select(payload, 1)
        dimensions = self.cp_plan.dimensions
        heads = self.cp_plan.compute_heads
        start = self.head_rank * heads
        kv_latent, key_rope = payload.split((dimensions.kv_rank, dimensions.rope_dim), dim=-1)
        # A contiguous latent preserves the native Linear bias-fusion path after payload splitting.
        return self.project_key_value(
            kv_latent.contiguous(), key_rope, heads=heads, head_range=(start, start + heads),
        )

    def _expanded_overlap_inputs(self, state, positions, key_rope, partition=None, *, rotary_fn):
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
        )

    def _expanded_overlap_inputs(self, state, positions, key_rope, partition=None, *, rotary_fn):
        """Overlap KV transfer with Q preparation, respecting the two-stage hybrid dependency."""
        key, value = self.project_key_value(state.kv_latent, key_rope, heads=self.cp_plan.local_heads)
        payload = torch.cat((key.transpose(1, 2), value.transpose(1, 2)), dim=-1)
        use_ulysses = self.cp_plan.ulysses_degree > 1
        if use_ulysses:
            kv_transfer_handle = self.ulysses_context.launch_sequence_to_head(payload)
        else:
            kv_transfer_handle = self._launch_expanded_kv(payload, partition)
        query = self.project_query(state.query_latent, positions, heads=self.cp_plan.local_heads, rotary_fn=rotary_fn)
        query_a2a_handle = (self.ulysses_context.launch_sequence_to_head(query.transpose(1, 2))
                            if use_ulysses else None)
        payload = kv_transfer_handle.wait()
        if use_ulysses and self.cp_plan.all_gather_degree > 1:
            payload = self._launch_expanded_kv(payload, partition).wait()
        if query_a2a_handle is not None:
            query = query_a2a_handle.wait().transpose(1, 2)
        if partition is not None and self.cp_plan.kv_exchange == "all_gather":
            payload = partition.select(payload, 1)
        key, value = payload.split((self.qk_head_dim, self.v_head_dim), dim=-1)
        return query, key.transpose(1, 2), value.transpose(1, 2)

    def _launch_expanded_kv(self, payload, partition):
        """Complete the sequence dimension after Ulysses has selected this head shard."""
        if self.cp_plan.kv_exchange == "all_to_all_v":
            return _selective_kv_launch(payload, partition, self.all_gather_mesh, 1, True)
        return self.query_context.launch_gather_sequence(payload).collective

    def _latent_overlap_inputs(self, state, positions, key_rope, partition=None, *, rotary_fn):
        """Cover latent transfer with Q preparation, then Q A2A with KV expansion."""
        payload = torch.cat((state.kv_latent, key_rope), dim=-1)
        kv_transfer_handle = self._launch_latent_kv(payload, partition)
        query = self.project_query(state.query_latent, positions, heads=self.cp_plan.local_heads, rotary_fn=rotary_fn)
        payload = kv_transfer_handle.wait()
        query_a2a_handle = (self.ulysses_context.launch_sequence_to_head(query.transpose(1, 2))
                            if self.cp_plan.ulysses_degree > 1 else None)
        key, value = self._expand_latent(payload, partition)
        if query_a2a_handle is not None:
            query = query_a2a_handle.wait().transpose(1, 2)
        return query, key, value
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
            value=MLAInputA2AState(self.ulysses_mesh),
        )
        # V needs no positional transform: it is ready as soon as KV up-projection
        # is split. Shared-K positional math was already applied by the caller.
        key_nope, value = self.project_key_value_parts(state.kv_latent, heads=self.cp_plan.local_heads)
        # Keep K assembly inside V's communication window.
        key_nope = launch_input(value, key_nope, input_a2a.value)
        key = self.assemble_key(key_nope, key_rope)
        # K is complete; Q projection and its model-defined rotation can cover it.
        query_latent = launch_input(key, state.query_latent, input_a2a.key)
        query = self.project_query(
            query_latent, positions, heads=self.cp_plan.local_heads, rotary_fn=rotary_fn,
        ).transpose(1, 2)
        query, key, value = self.finish_input_exchange(query, input_a2a, overlap_input_backward, input_gate)
        return query.transpose(1, 2), key.transpose(1, 2), value.transpose(1, 2)

    def finish_input_exchange(self, query: torch.Tensor, transfers: Any, overlap_backward: bool,
                              input_gate: Any) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
            transfers: Pending key/value exchanges and their backward state.
            overlap_backward: Pair backward communication with independent projections.
            input_gate: Must be None for ordinary MLA.
        """
        if input_gate is not None:
            raise ValueError("Ordinary MLA cannot consume a gated input dependency")
        return finish_expanded_inputs(query, transfers, overlap_backward)

    def _prepare_latent_ulysses_inputs(self, state, positions, key_rope, rotary_fn):
        """Prepare latent Ulysses inputs and pair their Q/KV backward dependencies."""
        payload = torch.cat((state.kv_latent, key_rope), dim=-1)
        kv_all_gather_handle = self._launch_latent_kv(payload)
        query = self.project_query(
            state.query_latent, positions, heads=self.cp_plan.local_heads, rotary_fn=rotary_fn,
        ).transpose(1, 2)
        # wait() normally adds the default AllGather autograd node. Instead,
        # _LatentKVLinear launches SUM RS before KV wgrad and returns dPayload
746
747
748
749
750
751
752
753
754
755
756
757
758
            gathered = kv_all_gather_handle.wait()
        query_a2a = MLAInputA2AState(self.ulysses_mesh)
        # Same payload value, with Q's inverse A2A bracketing its KV backward.
        kv_input = _InputA2ALaunch.apply(query, payload, query_a2a)
        heads = self.cp_plan.compute_heads
        start = self.head_rank * heads
        projected, shared_key = self.kv_b_proj(
            kv_input, mla_head_range=(start, start + heads),
            mla_gathered=gathered, mla_cp_mesh=self.cp_mesh,
        )
        key, value = self.assemble_key_value(projected, shared_key, heads=heads)
        query, key, value = _LatentInputA2AWait.apply(key, value, query_a2a)
        return query.transpose(1, 2), key, value
hyper_parallel/components/modules/attention/multi_latent_attention/multi_latent_attention.py
77
78
79
80
81
82
83
84
85
    if position_embeddings is None:
        return tensor
    cos, sin = (part.to(tensor.dtype) for part in position_embeddings)
    if cos.ndim == 2:
        cos, sin = cos.unsqueeze(0), sin.unsqueeze(0)
    if interleaved:
        cos, sin = (part[..., :tensor.shape[-1] // 2].repeat_interleave(2, dim=-1)
                    for part in (cos, sin))
    cos, sin = (part.expand(tensor.shape[0], -1, -1).transpose(0, 1) for part in (cos, sin))
90
91
92
93
94
95
96
97
98
    if interleaved:
        # V3 consumes real/imaginary half-bands; the kernel returns adjacent pairs.
        rotated = rotated.permute(1, 2, 0, 3)
        return rotated.unflatten(-1, (-1, 2)).transpose(-1, -2).flatten(-2).transpose(1, 2)
    return rotated.transpose(0, 1)


@dataclass(frozen=True)
class MLALatentInputs:
150
151
152
153
154
155
156
157
158
159
160
161
162
        self.attention_interface = attention_interface
        self._config_initialization = module is None or getattr(module, "_config_initialization", False)
        if module is None:
            if config is None:
                raise ValueError("MLA construction requires config or a source module")
            self._initialize_config(config)
            return
        if config is not None:
            raise ValueError("MLA construction accepts config or module, not both")
        self._initialize_source(module)

    def _initialize_source(self, module: nn.Module) -> None:
        """Read source math options and reuse its managed projection children."""
170
171
172
173
174
175
176
177
178
        )
        required += ("linear_qkv",) if self._source_is_fused else ("q_a_proj", "kv_a_proj_with_mqa")
        missing = [name for name in required if not hasattr(module, name)]
        if missing:
            raise TypeError(f"MultiLatentAttention source module is missing: {missing}")

        config = module.config
        self.config = config
        self.layer_idx = getattr(module, "layer_idx", getattr(module, "layer_number", None))
182
183
184
185
186
187
188
189
190
191
192
193
194
195
            "num_key_value_heads",
            getattr(config, "num_key_value_heads", self.num_heads),
        )
        if self.num_key_value_heads is None:
            self.num_key_value_heads = self.num_heads
        self.num_key_value_groups = self.num_heads // self.num_key_value_heads
        self.q_lora_rank = getattr(module, "q_lora_rank", config.q_lora_rank)
        self.kv_lora_rank = getattr(module, "kv_lora_rank", config.kv_lora_rank)
        if self.q_lora_rank is None:
            raise ValueError("MultiLatentAttention requires the Q-LoRA projection path")
        self.qk_rope_head_dim = getattr(module, "qk_rope_head_dim", config.qk_rope_head_dim)
        self.qk_nope_head_dim = getattr(module, "qk_nope_head_dim", config.qk_nope_head_dim)
        self.v_head_dim = getattr(module, "v_head_dim", config.v_head_dim)
        self.qk_head_dim = self.qk_nope_head_dim + self.qk_rope_head_dim
195
196
197
198
199
200
201
202
203
        self.qk_head_dim = self.qk_nope_head_dim + self.qk_rope_head_dim
        self.scaling = getattr(module, "scaling", self.qk_head_dim**-0.5)
        self.attention_dropout = getattr(module, "attention_dropout", 0.0)
        if isinstance(self.attention_dropout, nn.Dropout):
            self.attention_dropout = self.attention_dropout.p
        self.is_causal = getattr(module, "is_causal", True)
        self.sliding_window = getattr(
            module, "sliding_window", getattr(config, "sliding_window", None)
        )
206
207
208
209
210
211
212
213
214
215
216
217
218
        )
        # These source variants change attention math; ignoring the flags would
        # silently replace them with ordinary MLA. Keep only construction guards.
        if getattr(module, "param_sink_number", 0) > 0:
            raise ValueError(
                "MultiLatentAttention only implements standard MLA; parameter-sink attention "
                "must use the sink high-performance interfaces."
            )
        if getattr(module, "use_mome", False):
            raise NotImplementedError(
                "MultiLatentAttention only implements standard MLA; MOME must use the "
                "aggregate_hidden high-performance path."
            )
245
246
247
248
249
250
251
252
253
                and (q_a.bias is None) == (kv_a.bias is None)
                and (q_a.bias is None or q_a.bias.requires_grad == kv_a.bias.requires_grad)
            )
            if not can_fuse:
                raise ValueError(
                    "Q and KV latent projections cannot be represented by one fused projection"
                )
            self.linear_qkv = nn.Linear(
                q_a.in_features,
272
273
274
275
276
277
278
279
280
281
282
            "qk_rope_head_dim": config.qk_rope_head_dim,
            "v_head_dim": config.v_head_dim,
        }
        if any(not isinstance(value, int) or isinstance(value, bool) or value <= 0 for value in dimensions.values()):
            raise ValueError("MLA dimensions must be positive integers, including q_lora_rank")
        if dimensions["qk_rope_head_dim"] % 2:
            raise ValueError("MLA rotary dimension must be even")
        for name, value in dimensions.items():
            setattr(self, name, value)
        self.num_key_value_heads = self.num_heads
        self.num_key_value_groups = 1
287
288
289
290
291
292
293
294
295
        self.is_causal = getattr(config, "is_causal", True)
        self.sliding_window = getattr(config, "sliding_window", None)
        self.rotary_interleaved = bool(getattr(config, "rotary_interleaved", getattr(config, "rope_interleave", False)))
        if getattr(config, "use_mome", False) or getattr(config, "param_sink_number", 0) > 0:
            raise ValueError("MLA config construction does not support MOME or parameter-sink attention")
        self._source_is_fused = True
        self._q_latent_output_size = self.q_lora_rank
        self._kv_latent_output_size = self.kv_lora_rank + self.qk_rope_head_dim
        hidden_size = config.hidden_size
293
294
295
296
297
298
299
300
301
        self._q_latent_output_size = self.q_lora_rank
        self._kv_latent_output_size = self.kv_lora_rank + self.qk_rope_head_dim
        hidden_size = config.hidden_size
        if not isinstance(hidden_size, int) or isinstance(hidden_size, bool) or hidden_size <= 0:
            raise ValueError("MLA hidden_size must be a positive integer")
        dtype = getattr(config, "dtype", None)
        bias = getattr(config, "attention_bias", False)
        self.linear_qkv = nn.Linear(
            hidden_size, self._q_latent_output_size + self._kv_latent_output_size, bias=bias, dtype=dtype,
313
314
315
316
317
318
319
320
321
322
323
    def _config_scaling(config: Any, default: float) -> float:
        """Keep the source-compatible YaRN factor and explicit scaling override."""
        rope = getattr(config, "rope_parameters", None) or getattr(config, "rope_scaling", None) or {}
        if rope.get("rope_type", rope.get("type", "default")) != "default" and rope.get("mscale_all_dim", 0):
            factor = rope["factor"]
            mscale = 1.0 if factor <= 1 else 0.1 * rope["mscale_all_dim"] * math.log(factor) + 1.0
            default *= mscale * mscale
        return getattr(config, "scaling", default)

    def reset_parameters(self) -> None:
        """Initialize direct-config parameters without overriding a source model's initializer."""
324
325
326
327
328
329
330
331
332
333
334
335
        # HF initialization already visits converted Linear/Norm children. The replacement
        # framework also invokes this method after it, so source conversions must not reset them again.
        if not self._config_initialization:
            return
        for projection in (self.linear_qkv, self.q_b_proj, self.kv_b_proj, self.o_proj):
            projection.reset_parameters()
        nn.init.ones_(self.q_a_layernorm.weight)
        nn.init.ones_(self.kv_a_layernorm.weight)

    def make_transforms(self) -> list[WeightConverter]:
        """Describe reversible source-checkpoint to high-performance conversion."""
        if self._source_is_fused:
418
419
420
421
422
423
424
425
426
427
428
429
430

        Returns:
            K content and V views in BSND. A scheduler can consume V before K is assembled.
        """
        heads = self.num_heads if heads is None else heads
        projected = self.kv_b_proj(latent).view(
            *latent.shape[:2], heads, self.qk_nope_head_dim + self.v_head_dim,
        )
        return projected.split((self.qk_nope_head_dim, self.v_head_dim), dim=-1)

    @staticmethod
    def assemble_key(content: torch.Tensor, shared_key: torch.Tensor) -> torch.Tensor:
        """Join BSND content with one already-transformed shared key per token.
523
524
525
526
527
528
529
530
531
532
533
        query = self.project_query(state.query_latent, position_embeddings)
        shared_key = self.apply_position(state.key_rope.unsqueeze(2), position_embeddings).squeeze(2)
        kv_latent = state.kv_latent
        if past_key_values is not None:
            kv_latent, shared_key = past_key_values.update(
                kv_latent.unsqueeze(1), shared_key.unsqueeze(1), self.layer_idx,
            )
            kv_latent, shared_key = kv_latent.squeeze(1), shared_key.squeeze(1)
        key, value = self.project_key_value(kv_latent, shared_key)
        return MLAInputs(query, key, value, attention_mask,
                         dict(kwargs, actual_seq_len=actual_seq_len), output_state)
hyper_parallel/components/modules/attention/multi_latent_attention/multi_latent_attention_family.py
35
36
37
38
39
40
41
42
43
    """Validate logical ownership before allocating or replacing any parameter."""
    del context
    config = config if module is None else module.config
    if config is None:
        raise ValueError("MLA requires source configuration")
    options = member.arguments
    if not isinstance(options.get("overlap", False), bool):
        raise TypeError("MLA overlap must be boolean")
    if options.get("strategy", "expanded_ulysses") not in ("expanded_ulysses", "latent_kv_head"):
50
51
52
53
54
55
56
57
58
    degree = cp if degree is None else degree
    if not isinstance(degree, int) or isinstance(degree, bool) or degree < 1:
        raise ValueError("MLA Ulysses degree must be a positive integer")
    if config.num_attention_heads % tp:
        raise ValueError("MLA heads must be divisible by TP degree")
    _validate_mla_topology(config, cp, tp, degree, exchange)


def _validate_mla_topology(config, cp, tp, degree, exchange):
60
61
62
63
64
65
66
67
68
69
70
71
72
    # CP1 may retain a valid future CP request for promotion. Validate syntax
    # now; validate the actual factorization once CP topology becomes active.
    if cp > 1:
        if cp % degree or config.num_attention_heads % (tp * degree):
            raise ValueError("MLA Ulysses degree must divide CP degree and TP-local heads")
        if exchange == "all_to_all_v" and cp == degree:
            raise ValueError("selective KV exchange requires query-partitioned MLA CP")
        if tp > 1 and cp != degree:
            raise ValueError("MLA KV AllGather and hybrid CP currently require TP1")

def mla_sharding(*, module: nn.Module, module_fqn: str, member: Any, context: Any) -> ModuleShardingSpec:
    """Own all MLA parameters and declare the actual external token layout."""
    del module_fqn, member
107
108
109
110
111
112
113
114
115
    """Reject incompatible parameter overrides before any collectives can be bound."""
    expected = mla_sharding(module=module, module_fqn="", member=None, context=context)
    validate_tp_boundary(sharding, expected, name="MLA")
    if context.axis_sizes["tp"] > 1 and getattr(module, "use_nope", False):
        raise ValueError("Gated MLA currently supports TP1 only")


def mla_local_parallel_spec(*, module: nn.Module, member: Any) -> ParallelModuleSpec:
    """Keep layout validation outside the standalone algorithm."""
hyper_parallel/components/modules/family.py
153
154
155
156
157
158
159
160
161
    if id(family.baseline) in _FAMILIES:
        raise ValueError(f"Module family already registered: {family.name}")
    _validate_family_rules(family)
    if family.parallel_spec is not None and not callable(family.parallel_spec):
        raise TypeError("ModuleFamily parallel_spec must be callable")
    snapshot = replace(
        family,
        variants=MappingProxyType(dict(family.variants)),
        selector_specs=MappingProxyType(dict(family.selector_specs)),
hyper_parallel/components/modules/rms_norm.py
127
128
129
130
131
132
133

    def forward(self, x: torch.Tensor) -> torch.Tensor:
        """Apply unit-offset RMSNorm with the NPU operator."""
        # Resolve the callable directly; a same-named package attribute may be a submodule.
        from hyper_parallel.components.functional.rms_norm import rms_norm  # pylint: disable=C0415

        return rms_norm(x, self.weight, self.eps)
hyper_parallel/data/batching/runtime_input.py
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
    """

    def runtime_input_fields(self) -> tuple[str, ...]:
        """Declare the two leading-zero FA length arguments."""
        return ("cu_seq_lens_q", "cu_seq_lens_k")

    def build_runtime_inputs(self, *, batch: Mapping[str, Any], context: RuntimeInputContext) -> Mapping[str, Any]:
        """Read boundaries once per batch and preserve their global batch-major coordinates."""
        if context.options.get("attention_mode") != "compressed":
            raise ValueError("PackedAttentionRuntime requires attention_mode='compressed'")
        shape = context.local_input_shape
        if len(shape) != 2:
            raise ValueError("Packed attention requires input_ids with shape [batch, local_sequence]")
        boundaries = batch.get("cu_seq_lens")
        if isinstance(boundaries, torch.Tensor):
            if boundaries.ndim != 1:
                raise ValueError("Packed attention requires one-dimensional global cu_seq_lens")
            boundaries = boundaries.tolist()
        if boundaries is None or any(isinstance(value, bool) or not isinstance(value, Integral)
                                     for value in boundaries):
            raise ValueError("Packed attention requires integer global cu_seq_lens")
218
219
220
221
222
223
224
225
226
                                     for value in boundaries):
            raise ValueError("Packed attention requires integer global cu_seq_lens")
        boundaries = tuple(int(value) for value in boundaries)
        if len(boundaries) < 2 or boundaries[0] != 0:
            raise ValueError("Packed attention cu_seq_lens must start with zero and contain a document")
        if any(end <= start for start, end in zip(boundaries, boundaries[1:])):
            raise ValueError("Packed attention cu_seq_lens must be strictly increasing")
        tokens = shape[0] * shape[1] * context.parallel_sizes["cp"]
        if boundaries[-1] != tokens:
hyper_parallel/distributed/_builder/applier.py
360
361
362
363
364
365
366
367
    """Identify class-owned execution and boundaries already applied or unnecessary."""
    family = get_family_build_info(module)
    owned = module_fqn in plan.parallel_modules or (family is not None and family.resolved.sharding is not None)
    if family is not None and family.layout is not None and family.layout.applied:
        return owned, True
    unchanged = spec.in_src == spec.in_dst and (spec.out_dst is None or spec.out_src == spec.out_dst)
    return owned, owned and unchanged

hyper_parallel/distributed/_builder/family_layout.py
122
123
124
125
126
127
128
129
130
        expected: Supported parameter roles and computation layouts from the family.
        name: Family name used in construction-time errors.
    """
    if set(sharding.params) != set(expected.params):
        raise ValueError(f"{name} must declare every owned parameter")
    for parameter, required in expected.params.items():
        actual = sharding.params[parameter]
        if any(actual.get(axis, Replicate()) != required.get(axis, Replicate())
               for axis in set(actual) | set(required)):
130
131
132
133
134
135
136
137
138
139
140
141
142
143
               for axis in set(actual) | set(required)):
            raise ValueError(f"{name} unsupported parameter layout for {parameter}: "
                             f"expected {required}, got {actual}")
    if not sharding.is_boundary:
        raise ValueError(f"{name} requires a TP redistribution boundary")
    for field_name, key in (("in_src", "hidden_states"), ("in_dst", "hidden_states"),
                            ("out_src", "output"), ("out_dst", "output")):
        ports, required_ports = getattr(sharding, field_name), getattr(expected, field_name)
        if ports is None or set(ports) != {key}:
            raise ValueError(f"{name} {field_name} must declare its named token port")
        actual, required = ports[key], required_ports[key]
        for axis in set(actual) | set(required):
            placement = actual.get(axis, Replicate())
            if axis == "tp" and field_name in ("in_src", "out_dst"):
141
142
143
144
145
146
147
148
        for axis in set(actual) | set(required):
            placement = actual.get(axis, Replicate())
            if axis == "tp" and field_name in ("in_src", "out_dst"):
                if placement not in (Replicate(), Shard(1)):
                    raise ValueError(f"{name} TP token ports support Replicate or sequence Shard(1)")
            elif placement != required.get(axis, Replicate()):
                raise ValueError(f"{name} {field_name} cannot change {axis} computation/ownership")

hyper_parallel/distributed/context_parallel/cp_context.py
 95
 96
 97
 98
 99
100
101
102
103
            tensor: Input in sequence-partitioned layout.
            seq_dim: Sequence axis.
            head_dim: Head axis.
        """
        return async_ulysses_seq_to_head_launch(tensor, seq_dim, head_dim, self.mesh)

    def launch_head_to_sequence(self, tensor: Tensor, *, seq_dim: int = 1,
                                head_dim: int = 2) -> AsyncCPCollective:
        """Launch the inverse A2A without changing its existing autograd contract.
106
107
108
109
110
111
112
113
            tensor: Input in head-partitioned layout.
            seq_dim: Sequence axis.
            head_dim: Head axis.
        """
        return async_ulysses_seq_to_head_launch(tensor, head_dim, seq_dim, self.mesh)

    def split(self, ulysses_degree: int) -> tuple[CPContext, CPContext]:
        """Borrow cached Ulysses/query-owner submeshes at bind time, never in forward.
115
116
117
118
119
120
121
122
123
        Args:
            ulysses_degree: Number of CP ranks partitioning attention heads.
        """
        if ulysses_degree < 1 or self.size % ulysses_degree:
            raise ValueError("Ulysses degree must divide the CP mesh size")
        if ulysses_degree in (1, self.size):
            return self, self
        heads, queries = _build_hybrid_cp_submeshes(self.mesh, ulysses_degree)
        return CPContext(heads), CPContext(queries)
hyper_parallel/distributed/module_binding.py
136
137
138
139
140
141
142
143
144
145
146
147
148
149
    for source, destination in ((sharding.in_src, sharding.in_dst), (sharding.out_src, sharding.out_dst)):
        if source == destination or destination is None:
            continue
        if not sharding.is_boundary or source is None or set(source) != set(destination):
            raise ValueError("explicit stage modules require complete TP boundary ports")
        for port, begin in source.items():
            end = destination[port]
            if any(begin.get(axis, Replicate()) != end.get(axis, Replicate())
                   for axis in (set(begin) | set(end)) - {"tp"}):
                raise ValueError("explicit stages must restore non-TP ownership themselves")


def _validate_expert_ownership(module, parallel, sharding, context):
    """Check the bound EP mesh against local and global expert ownership."""