Diff Coverage

Diff: origin/master...HEAD, staged and unstaged changes

Source File Diff Coverage (%) Missing Lines
hyper_parallel/components/modules/attention/__init__.py 100%  
hyper_parallel/components/modules/attention/deepseek_sparse_attention/__init__.py 100%  
hyper_parallel/components/modules/attention/deepseek_sparse_attention/cp_deepseek_sparse_attention.py 57.0% 81,83,95,111-116,118-119,122,143-144,152-157,169,173,176,180-185,191,193,205-210,231,237-238,245-248,255-257,265,268
hyper_parallel/components/modules/attention/deepseek_sparse_attention/deepseek_sparse_attention.py 50.5% 73,92,96-98,114,118-124,154-157,180,191-194,199-202,204,208-210,215,217-219,242-246,248,255-257,261-265,271-272,282-283,286-289,293,296,302-304,307,311,317-318,345,353,364,366,388,390,397-399,406,440-442,450-452,456-469,474,507,509,518,526,532-534,539,543-545,563-564,580,589-593,604-614,618-619,686-688,692,695,701,705,714-716,739,742,746,753-754,757,760,763,766-767,770,778,788-789,793,796-797,800-801,806,842,844,849,857,864-865,867-868,882,890,899,913,922,949-952,956,961,964,970-977,981,985-988,992,1000-1001,1005-1010,1016-1018,1021-1027,1030-1031,1036-1038,1045-1052,1056,1063-1066,1070-1071,1073,1077,1089-1091,1094,1097,1101,1108,1119-1121,1124-1125,1129,1136-1138,1141-1142,1145-1147,1157-1161,1172,1175-1178,1181-1182
hyper_parallel/components/modules/attention/deepseek_sparse_attention/deepseek_sparse_attention_family.py 46.1% 37,39,42,56,61,66-72,79-81,86,91-104,113-119,124-125,143-144
hyper_parallel/components/modules/attention/deepseek_sparse_attention/cp_deepseek_sparse_attention.py
77
78
79
80
81
82
83
84
85
86
87
    if actual_seq_len is not None:
        packed["actual_seq_len"] = actual_seq_len
    queries, keys = resolve_packed_sequence_lengths(packed, total, total)
    if queries is None:
        queries = keys = [(row + 1) * global_length for row in range(batch_size)]
    if len(queries) != len(keys):
        raise ValueError("Sparse attention requires the same number of packed Q and KV documents")
    projected = []
    for rank in range(cp_size):
        query = _project_ends(
            queries, global_seq_len=global_length, local_seq_len=local_seq_len, rank=rank, query=True,
91
92
93
94
95
96
97
98
99
        )
        if any(end == previous for previous, end in zip((0, *query[:-1]), query)):
            raise ValueError(f"Sparse attention does not support zero-query packed documents on CP rank {rank}")
        if any(end == previous for previous, end in zip((0, *key[:-1]), key)):
            raise ValueError(f"Sparse attention has an empty packed KV document on CP rank {rank}")
        projected.append((query, key))
    local_query, local_key = projected[cp_rank]
    return SparseAttentionSequenceMetadata(
        (cp_rank + 1) * local_seq_len,
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
    index_states: tuple[torch.Tensor, ...], attention_states: tuple[torch.Tensor, ...],
    metadata: SparseAttentionSequenceMetadata,
) -> tuple[tuple[torch.Tensor, ...], tuple[torch.Tensor, ...]]:
    """Gather the three key banks in a fixed order, then select the causal prefix."""
    index_query, index_key, weights = index_states
    query, latent_kv, query_rope, key_rope = attention_states
    tensors = (index_key, latent_kv, key_rope)
    if overlap:
        pending = tuple(context.launch_gather_sequence(tensor, dim=1) for tensor in tensors)
        gathered = tuple(receipt.wait()[0] for receipt in pending)
    else:
        gathered = tuple(context.gather_sequence(tensor, dim=1)[0] for tensor in tensors)
    index_key, latent_kv, key_rope = (
        tensor[:, :metadata.kv_prefix_length].contiguous() for tensor in gathered
    )
    return (index_query, index_key, weights), (query, latent_kv, query_rope, key_rope)


class CPDeepseekV32SparseAttentionModule(DeepseekV32SparseAttentionModule):
    """Borrow CP transport while reusing the local projection and kernel stages."""
139
140
141
142
143
144
145
146
147
148

    @staticmethod
    def parallel_module_spec() -> ParallelModuleSpec:
        """Declare operand exchange while the local algorithm owns mathematical dispatch."""
        local_spec = v32_sparse_attention_parallel_spec(module=None, member=None)
        return ParallelModuleSpec(
            "attention.deepseek_v32_sparse_attention.cp", binding_kind="cp",
            stages=local_spec.stages,
            validate=validate_sparse_attention_layout,
        )
148
149
150
151
152
153
154
155
156
157
158
159
160
161
        )

    def bind_parallel(self, binding: ParallelBinding) -> None:
        """Borrow an existing CPContext before FSDP installs projection hooks."""
        context = binding.cp_context
        if context is None or context.size <= 1:
            raise ValueError("DeepSeek-V3.2 CP member requires an active CPContext")
        if self._cp_context is not None and self._cp_context.mesh is not context.mesh:
            raise ValueError("DeepSeek-V3.2 CP member already belongs to another mesh")
        self._cp_context = context

    def prepare_key_bank(
        self, index_states: tuple[torch.Tensor, ...], attention_states: tuple[torch.Tensor, ...],
        metadata: SparseAttentionSequenceMetadata,
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
        All readiness receipts belong to this invocation. The existing gather
        adjoint sums consumers into each source shard, including the zero
        gradients from unused suffix tokens.
        """
        return _gather_key_banks(self._cp_context, self.overlap, index_states, attention_states, metadata)

    def exchange_attention(self, inputs: SparseAttentionInputs) -> SparseAttentionInputs:
        """Return ready key banks without taking ownership of the attention algorithm."""
        index_states, attention_states = self.prepare_key_bank(
            inputs.index_states, inputs.attention_states, inputs.metadata,
        )
        return SparseAttentionInputs(index_states, attention_states, inputs.metadata, inputs.use_dense)

    def _validate_attention_mask(self, hidden_states: torch.Tensor, attention_mask: torch.Tensor | None) -> None:
        """Accept existing causal-mask carriers without reading their device contents."""
        if attention_mask is None:
            return
        batch_size, local_length = hidden_states.shape[:2]
        global_length = local_length * self._cp_context.size
        prefix_length = local_length * (self._cp_context.rank + 1)
        shapes = {
            (local_length, local_length), (local_length, global_length),
            (global_length, global_length), (local_length, prefix_length),
        }
        # TextParallelBatch constructs a global square mask after slicing token fields.
187
188
189
190
191
192
193
194
195
196
            (global_length, global_length), (local_length, prefix_length),
        }
        # TextParallelBatch constructs a global square mask after slicing token fields.
        # Packed endpoints, rather than mask values, remain the CANN causality contract.
        if (attention_mask.ndim != 4 or attention_mask.shape[0] not in (1, batch_size)
                or attention_mask.shape[1] != 1 or tuple(attention_mask.shape[-2:]) not in shapes):
            raise ValueError(
                "DeepSeek-V3.2 CP requires a standard 4D causal mask with local/global sequence dimensions; "
                "padding and custom masks are not supported"
            )
201
202
203
204
205
206
207
208
209
210
211
212
213
214
        actual_seq_len: torch.Tensor | Sequence[int] | None,
        kwargs: dict[str, Any], *, use_dense: bool,
    ) -> SparseAttentionSequenceMetadata:
        """Validate every rank's packed geometry before any operand collective."""
        del use_dense
        if self._cp_context is None:
            raise ValueError("DeepSeek-V3.2 CP member must be bound before execution")
        self._validate_forward_inputs(hidden_states, None, past_key_values, position_ids, kwargs)
        self._validate_attention_mask(hidden_states, attention_mask)
        return build_cp_sequence_metadata(
            batch_size=hidden_states.shape[0], local_seq_len=hidden_states.shape[1],
            device=hidden_states.device, cp_size=self._cp_context.size, cp_rank=self._cp_context.rank,
            actual_seq_len=actual_seq_len, kwargs=kwargs,
        )
227
228
229
230
231
232
233
234
235
            module=module, config=config, module_fqn=module_fqn, context=context,
            cp_strategy=cp_strategy, overlap=overlap, dense_warmup=dense_warmup,
        )
        if self.use_mome or self.param_sink_number > 0:
            raise ValueError("MOME and parameter-sink sparse attention require CP1")
        self._cp_context: CPContext | None = None

    @staticmethod
    def parallel_module_spec() -> ParallelModuleSpec:
233
234
235
236
237
238
239
240
241

    @staticmethod
    def parallel_module_spec() -> ParallelModuleSpec:
        """Keep the generic mathematical stages while declaring CP resource binding."""
        local_spec = sparse_attention_parallel_spec(module=None, member=None)
        return ParallelModuleSpec(
            "attention.deepseek_sparse_attention.cp", binding_kind="cp", stages=local_spec.stages,
            validate=validate_sparse_attention_layout,
        )
241
242
243
244
245
246
247
248
249
250
251
252
        )

    def bind_parallel(self, binding: ParallelBinding) -> None:
        """Borrow the already-created CP context before parameter materialization."""
        context = binding.cp_context
        if context is None or context.size <= 1:
            raise ValueError("Sparse-attention CP member requires an active CPContext")
        self._cp_context = context

    def preflight_attention(
        self, hidden_states: torch.Tensor, actual_seq_len: torch.Tensor | Sequence[int] | None,
        kwargs: Mapping[str, Any],
251
252
253
254
255
256
257
258
259
260
261
        self, hidden_states: torch.Tensor, actual_seq_len: torch.Tensor | Sequence[int] | None,
        kwargs: Mapping[str, Any],
    ) -> SparseAttentionSequenceMetadata:
        """Resolve global packed boundaries before the first key-bank collective."""
        if self._cp_context is None:
            raise ValueError("Sparse-attention CP member must be bound before execution")
        return build_cp_sequence_metadata(
            batch_size=hidden_states.shape[0], local_seq_len=hidden_states.shape[1],
            device=hidden_states.device, cp_size=self._cp_context.size, cp_rank=self._cp_context.rank,
            actual_seq_len=actual_seq_len, kwargs=kwargs,
        )
261
262
263
264
265
266
267
268
        )

    def exchange_attention(self, inputs: SparseAttentionInputs) -> SparseAttentionInputs:
        """Gather ready key banks while preserving the local query and packed metadata."""
        index_states, attention_states = _gather_key_banks(
            self._cp_context, self.overlap, inputs.index_states, inputs.attention_states, inputs.metadata,
        )
        return SparseAttentionInputs(index_states, attention_states, inputs.metadata, inputs.use_dense)
hyper_parallel/components/modules/attention/deepseek_sparse_attention/deepseek_sparse_attention.py
69
70
71
72
73
74
75
76
77
        queries, keys = resolve_packed_sequence_lengths(packed, total, total)
        if queries is None:
            queries = keys = [(row + 1) * local_seq_len for row in range(batch_size)]
        if len(queries) != len(keys):
            raise ValueError("Sparse attention requires the same number of packed Q and KV documents")
        return cls(
            local_seq_len,
            torch.tensor(queries, dtype=torch.int32, device=device),
            torch.tensor(keys, dtype=torch.int32, device=device),
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
    """Apply the configured RoPE convention to an attention query/key pair."""
    if position_embeddings is None:
        return query, key
    # The rotary backend imports torch_npu; defer it until positional math is requested.
    from hyper_parallel.components.functional import (  # pylint: disable=import-outside-toplevel
        apply_rotary_pos_emb, apply_rotary_pos_emb_interleave,
    )

    cos, sin = position_embeddings
    rotary_op = apply_rotary_pos_emb_interleave if interleaved else apply_rotary_pos_emb
    return rotary_op(query, key, cos, sin, unsqueeze_dim=2)


def _apply_index_rope(
    query: torch.Tensor,
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
    """Apply RoPE to the rotary portion of an indexer query/key pair."""
    if position_embeddings is None:
        return query, key
    # The rotary backend is optional for construction and CPU metadata/projection tests.
    from hyper_parallel.components.functional import (  # pylint: disable=import-outside-toplevel
        apply_rotary_pos_emb, apply_rotary_pos_emb_interleave,
    )

    cos, sin = position_embeddings
    split_sizes = (rope_head_dim, index_head_dim - rope_head_dim)
    query_rot, query_pass = torch.split(query, split_sizes, dim=-1)
    key_rot, key_pass = torch.split(key, split_sizes, dim=-1)
    rotary_op = apply_rotary_pos_emb_interleave if interleaved else apply_rotary_pos_emb
    query_rot, key_rot = rotary_op(query_rot, key_rot, cos, sin, unsqueeze_dim=2)
    return torch.cat((query_rot, query_pass), dim=-1), torch.cat((key_rot, key_pass), dim=-1)


def _absorb_query_projection(
    query: torch.Tensor,
150
151
152
153
154
155
156
157
158
159
160
    kv_lora_rank: int,
    value_head_dim: int,
) -> torch.Tensor:
    """Apply the absorbed value up-projection and restore the hidden layout."""
    output = output[..., :kv_lora_rank]
    output = output.permute(2, 0, 1, 3).reshape(num_heads, batch_size * seq_length, kv_lora_rank)
    output = torch.bmm(output, value_up_weight)
    return output.view(num_heads, batch_size, seq_length, value_head_dim).permute(1, 2, 0, 3).reshape(
        batch_size, seq_length, -1
    )

176
177
178
179
180
181
182
183
184

    Returns:
        Detached Q, K, Q-rope, K-rope, maximum and sum for the existing KL kernel.
    """
    return tuple(tensor.detach() for tensor in (query, key, query_rope, key_rope, maximum, total))


def apply_mome(
    hidden_states: torch.Tensor,
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
    *,
    fused: bool,
) -> torch.Tensor:
    """Apply masked causal depthwise convolution and residual."""
    if mome_mask is None:
        raise ValueError("mome_mask is required when MOME is enabled")
    if mome_mask.shape != hidden_states.shape[:2]:
        raise ValueError(
            f"mome_mask must have shape {hidden_states.shape[:2]}, "
            f"but got {tuple(mome_mask.shape)}"
        )

    mome_mask = mome_mask.to(device=hidden_states.device, dtype=torch.bool)
    padding = convolution.kernel_size[0] - 1
    padded_states = F.pad(hidden_states, (0, 0, padding, 0))
    if fused:
        # The MOME backend requires optional Omni operators, unlike V3.2 imports.
        from hyper_parallel.components.functional.aggregate_hidden import (  # pylint: disable=import-outside-toplevel
            aggregate_hidden,
        )

        padded_mask = F.pad(mome_mask, (padding, 0), value=False)
        weight = convolution.weight.squeeze(1).transpose(0, 1)
        mixed_states = aggregate_hidden(  # pylint: disable=not-callable
            padded_states.transpose(0, 1).contiguous(),
            weight,
            padded_mask,
        )
        mixed_states = mixed_states[padding:].transpose(0, 1).contiguous()
    else:
        mixed_states = convolution(padded_states.transpose(1, 2)).transpose(1, 2)
        mixed_states = mixed_states * mome_mask.unsqueeze(-1).to(mixed_states.dtype)
    return hidden_states + mixed_states


@dataclass(frozen=True)
class SparseAttentionInputs:
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
    *,
    need_stats: bool,
) -> tuple[torch.Tensor, torch.Tensor | None, torch.Tensor | None]:
    """Run one TND FA and return dense-indexer statistics."""
    query_ends, key_ends = (lengths.tolist() for lengths in sequence_lengths)
    q_flat, k_flat, v_flat = (tensor.flatten(0, 1) for tensor in (query, key, value))
    num_heads = q_flat.shape[1]
    causal_mask = torch.ones((2048, 2048), dtype=torch.bool, device=query.device).triu(diagonal=1)
    import torch_npu  # pylint: disable=import-outside-toplevel

    result = torch_npu.npu_fusion_attention(
        q_flat.contiguous(), k_flat.contiguous(), v_flat.contiguous(), num_heads, "TND",
        pse=None, padding_mask=None, atten_mask=causal_mask, scale=module.scaling,
        pre_tockens=getattr(module, "pre_tockens", 1048576),
        next_tockens=getattr(module, "next_tockens", 0), keep_prob=1.0, inner_precise=0,
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
        pre_tockens=getattr(module, "pre_tockens", 1048576),
        next_tockens=getattr(module, "next_tockens", 0), keep_prob=1.0, inner_precise=0,
        actual_seq_qlen=query_ends, actual_seq_kvlen=key_ends, sparse_mode=3,
    )
    output = result[0].reshape(*query.shape[:3], value.shape[-1])
    if not need_stats:
        return output, None, None

    # TND FA stores each packed segment as [head, token, 8], while exposing a
    # physical [token, head, 8] shape. Dense KL consumes [head, token, 1].
    statistics = []
    starts = [0, *query_ends[:-1]]
    for statistic in result[1:3]:
        rows = statistic.reshape(-1, statistic.shape[-1])
        segments = [
            rows[start * num_heads : end * num_heads].reshape(
                num_heads, end - start, statistic.shape[-1]
            )[..., :1]
            for start, end in zip(starts, query_ends)
267
268
269
270
271
272
273
274
275
276
                num_heads, end - start, statistic.shape[-1]
            )[..., :1]
            for start, end in zip(starts, query_ends)
        ]
        statistics.append(torch.cat(segments, dim=1).detach())
    return output, statistics[0], statistics[1]


def _deepseek_v32_dense_attention(
    module: nn.Module,
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
    attention_states: tuple[torch.Tensor, ...],
    sequence_lengths: tuple[torch.Tensor, torch.Tensor],
) -> torch.Tensor:
    """Run unabsorbed MLA FA and the fused dense indexer KL on local states."""
    query, compressed_kv, query_rope, key_rope = attention_states
    projected_kv = module.kv_b_proj(compressed_kv.squeeze(2)).view(
        *compressed_kv.shape[:2], query.shape[2], module.qk_nope_head_dim + module.v_head_dim,
    )
    key, value = projected_kv.split((module.qk_nope_head_dim, module.v_head_dim), dim=-1)
    key_rope = key_rope.expand(-1, -1, query.shape[2], -1)
    need_stats = bool(module.training and not module.freeze_dsa and module.dsa_loss_coeff)
    output, maximum, total = _dense_flash_attention(
        module, torch.cat((query, query_rope), dim=-1), torch.cat((key, key_rope), dim=-1),
        value, sequence_lengths, need_stats=need_stats,
    )
    if need_stats:
        # Dense CANN operators are optional for sparse and Omni execution.
        # pylint: disable=import-outside-toplevel
        from hyper_parallel.components.functional.dsa_dense import (
            dsa_dense_indexer_kl_loss,
            dsa_dense_indexer_lse,
        )
        # pylint: enable=import-outside-toplevel
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
            dsa_dense_indexer_lse,
        )
        # pylint: enable=import-outside-toplevel

        query_lengths, key_lengths = sequence_lengths
        index_query, index_key, weights = (tensor.flatten(0, 1) for tensor in index_states)
        indexer_stats = dsa_dense_indexer_lse(
            index_query, index_key, weights, query_lengths=query_lengths, key_lengths=key_lengths,
        )
        teacher_query, teacher_key, teacher_qrope, teacher_krope, maximum, total = prepare_indexer_teacher(
            query.flatten(0, 1), key.flatten(0, 1),
            query_rope.flatten(0, 1), key_rope.flatten(0, 1), maximum, total,
        )
        loss = dsa_dense_indexer_kl_loss(
            index_query, index_key, weights, teacher_query, teacher_key,
            maximum, total, indexer_stats, scale=module.scaling,
            query_rope=teacher_qrope, key_rope=teacher_krope,
            query_lengths=query_lengths, key_lengths=key_lengths, loss_coefficient=module.dsa_loss_coeff,
313
314
315
316
317
318
319
320
321
322
            maximum, total, indexer_stats, scale=module.scaling,
            query_rope=teacher_qrope, key_rope=teacher_krope,
            query_lengths=query_lengths, key_lengths=key_lengths, loss_coefficient=module.dsa_loss_coeff,
        )
        output = aux_loss_auto_scale(output, loss)
    return output


class DeepseekV32SparseAttentionModule(nn.Module):
    """DeepSeek-V3.2 sparse attention with native configuration and source-module construction.
341
342
343
344
345
346
347
348
349
        self.qk_nope_head_dim = getattr(module, "qk_nope_head_dim", config.qk_nope_head_dim)
        self.v_head_dim = getattr(module, "v_head_dim", config.v_head_dim)
        self.qk_head_dim = self.qk_nope_head_dim + self.qk_rope_head_dim
        if self.q_lora_rank is None:
            raise ValueError("DeepseekV32SparseAttention requires the Q-LoRA projection path")
        self.index_head_dim = getattr(indexer, "head_dim", config.index_head_dim)
        self.num_index_heads = getattr(
            indexer,
            "n_heads",
349
350
351
352
353
354
355
356
357
            "n_heads",
            getattr(config, "index_n_heads", getattr(config, "index_num_attention_heads", None)),
        )
        if self.num_index_heads is None:
            raise ValueError("DeepseekV32SparseAttention requires the number of indexer heads")
        self.index_topk = getattr(indexer, "index_topk", config.index_topk)

    def _initialize_options(self, module: nn.Module | None) -> None:
        """Read sparse attention options while preserving source scaling and RoPE conventions."""
360
361
362
363
364
365
366
367
368
369
370
        if self.scaling is None:
            self.scaling = self._config_scaling(config)
        attention_dropout = getattr(module, "attention_dropout", getattr(config, "attention_dropout", 0.0))
        if isinstance(attention_dropout, nn.Dropout):
            attention_dropout = attention_dropout.p
        if attention_dropout != 0.0:
            raise ValueError(
                "DeepseekV32SparseAttention requires attention_dropout=0 because "
                "the sparse-attention operator does not apply dropout"
            )
        source_rotary_interleaved = getattr(module, "rotary_interleaved", getattr(config, "rotary_interleaved", None))
384
385
386
387
388
389
390
391
392
393
394
        self.dsa_dense_warm_up = bool(
            getattr(module, "dsa_dense_warm_up", getattr(config, "dsa_dense_warm_up", False))
        )
        if getattr(module, "param_sink_number", getattr(config, "param_sink_number", 0)) > 0:
            raise ValueError("DeepseekV32SparseAttention does not support the parameter-sink path")
        if getattr(module, "use_mome", getattr(config, "use_mome", False)):
            raise ValueError("DeepseekV32SparseAttention does not support the MOME path")

    def _config_scaling(self, config: Any) -> float:
        """Apply the source-compatible YaRN factor for native construction."""
        scaling = self.qk_head_dim**-0.5
393
394
395
396
397
398
399
400
401
402
403
        """Apply the source-compatible YaRN factor for native construction."""
        scaling = self.qk_head_dim**-0.5
        rope = getattr(config, "rope_parameters", None) or getattr(config, "rope_scaling", None) or {}
        if rope.get("rope_type", rope.get("type", "default")) != "default" and rope.get("mscale_all_dim", 0):
            factor = rope["factor"]
            mscale = 1.0 if factor <= 1 else 1.0 + 0.1 * rope["mscale_all_dim"] * math.log(factor)
            scaling *= mscale * mscale
        return getattr(config, "scaling", scaling)

    def _initialize_config_layers(self, config: Any) -> None:
        """Allocate the native parameter tree without constructing a Transformers model."""
402
403
404
405
406
407
408
409
410
    def _initialize_config_layers(self, config: Any) -> None:
        """Allocate the native parameter tree without constructing a Transformers model."""
        dtype = getattr(config, "dtype", None)
        if isinstance(dtype, str):
            dtype = getattr(torch, dtype.removeprefix("torch."))
        factory_kwargs = {"device": getattr(config, "device", None), "dtype": dtype}
        bias = getattr(config, "attention_bias", False)
        self._q_latent_output_size = self.q_lora_rank
        self._kv_latent_output_size = self.kv_lora_rank + self.qk_rope_head_dim
436
437
438
439
440
441
442
443
444
445
446
        self.indexer.weights_proj = nn.Linear(config.hidden_size, self.num_index_heads, bias=False, **factory_kwargs)

    def _initialize_source_layers(self, module: nn.Module) -> None:
        """Fuse loaded down-projection values and retain the remaining Parameters."""
        q_a, kv_a = module.q_a_proj, module.kv_a_proj_with_mqa
        self._q_latent_output_size, self._kv_latent_output_size = q_a.out_features, kv_a.out_features
        can_fuse = (
            q_a.in_features == kv_a.in_features
            and q_a.weight.requires_grad == kv_a.weight.requires_grad
            and q_a.weight.device == kv_a.weight.device
            and q_a.weight.dtype == kv_a.weight.dtype
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
            and q_a.weight.dtype == kv_a.weight.dtype
            and (q_a.bias is None) == (kv_a.bias is None)
            and (q_a.bias is None or q_a.bias.requires_grad == kv_a.bias.requires_grad)
        )
        if not can_fuse:
            raise ValueError("Q and KV latent projections cannot be represented by one fused projection")
        self.linear_qkv = nn.Linear(
            q_a.in_features, q_a.out_features + kv_a.out_features, bias=q_a.bias is not None,
            device=q_a.weight.device, dtype=q_a.weight.dtype,
        )
        self.linear_qkv.weight.requires_grad_(q_a.weight.requires_grad)
        if self.linear_qkv.bias is not None:
            self.linear_qkv.bias.requires_grad_(q_a.bias.requires_grad)
        if not q_a.weight.is_meta:
            with torch.no_grad():
                self.linear_qkv.weight.copy_(torch.cat((q_a.weight, kv_a.weight), dim=0))
                if self.linear_qkv.bias is not None:
                    self.linear_qkv.bias.copy_(torch.cat((q_a.bias, kv_a.bias), dim=0))
        for name in ("q_a_layernorm", "kv_a_layernorm", "q_b_proj"):
            setattr(self, name, getattr(module, name))
        if module.kv_b_proj.bias is not None:
            raise ValueError("Absorbed KV projection requires a bias-free Linear")
        self.kv_b_proj = module.kv_b_proj
        self.o_proj, self.indexer = module.o_proj, module.indexer

    def _reuse_local_member(self, module: DeepseekV32SparseAttentionModule) -> None:
        """Promote a loaded unbound member without re-fusing or reinitializing values."""
        if getattr(module, "_hp_parallel_binding", None) is not None:
            raise ValueError("DeepSeek-V3.2 promotion must precede parallel binding and FSDP")
        fields = (
            "config", "layer_idx", "num_heads", "q_lora_rank", "kv_lora_rank",
            "qk_rope_head_dim", "qk_nope_head_dim", "v_head_dim", "qk_head_dim", "index_head_dim",
            "num_index_heads", "index_topk", "scaling", "rotary_interleaved", "index_rotary_interleaved",
503
504
505
506
507
508
509
510
511
512
513
        """
        super().__init__()
        del module_fqn, context
        if (config is None) == (module is None):
            raise ValueError("DeepSeek-V3.2 sparse attention requires exactly one of config or module")
        if cp_strategy != "kv_allgather":
            raise ValueError("DeepSeek-V3.2 supports only cp_strategy='kv_allgather'")
        self._config_initialization = module is None or getattr(module, "_config_initialization", False)
        self.cp_strategy, self.overlap = cp_strategy, overlap
        self._dense_warmup_override = dense_warmup
        if isinstance(module, DeepseekV32SparseAttentionModule):
514
515
516
517
518
519
520
521
522
            self._reuse_local_member(module)
        else:
            self.config = config if module is None else module.config
            if hasattr(self.config, "indexer_types"):
                raise TypeError(
                    "DeepSeek-V3.2 sparse attention does not support shared-indexer or three-value output contracts"
                )
            self._initialize_dimensions(module)
            self._initialize_options(module)
522
523
524
525
526
527
528
529
530
            self._initialize_options(module)
            if module is None:
                self._initialize_config_layers(config)
            else:
                self._initialize_source_layers(module)
        if module is not None:
            self.train(module.training)

    def reset_parameters(self) -> None:
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
            self.train(module.training)

    def reset_parameters(self) -> None:
        """Initialize native/meta parameters without resetting converted source weights."""
        if not self._config_initialization:
            return
        for child in (
            self.linear_qkv, self.q_a_layernorm, self.kv_a_layernorm, self.q_b_proj,
            self.kv_b_proj, self.o_proj, self.indexer.wq_b, self.indexer.wk,
            self.indexer.k_norm, self.indexer.weights_proj,
        ):
            child.reset_parameters()

    def make_transforms(self) -> list[WeightConverter]:
        """Describe reversible source-checkpoint to high-performance conversion."""
        if self._config_initialization:
            return []
        transforms: list[WeightConverter] = [
            WeightConverter(
                source_patterns=[
                    "q_a_proj.weight",
                    "kv_a_proj_with_mqa.weight",
559
560
561
562
563
564
565
566
567
568
                    )
                ],
            ),
        ]
        if self.linear_qkv.bias is not None:
            transforms.insert(
                1,
                WeightConverter(
                    source_patterns=["q_a_proj.bias", "kv_a_proj_with_mqa.bias"],
                    target_patterns="linear_qkv.bias",
576
577
578
579
580
581
582
583
584
                        )
                    ],
                ),
            )
        return transforms

    @staticmethod
    def _invalid_position_ids(
        position_ids: torch.Tensor | None,
585
586
587
588
589
590
591
592
593
594
595
596
597
        batch_size: int,
        seq_length: int,
    ) -> bool:
        """Return whether position IDs violate the supported dense layout."""
        if position_ids is None:
            return False
        if position_ids.ndim != 2:
            return True
        return position_ids.shape[0] not in (1, batch_size) or position_ids.shape[1] != seq_length

    @staticmethod
    def _validate_forward_inputs(
        hidden_states: torch.Tensor,
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
        position_ids: torch.Tensor | None,
        kwargs: dict[str, Any],
    ) -> None:
        """Validate inputs supported by the DeepSeek sparse-attention path."""
        if hidden_states.ndim != 3:
            raise ValueError("DeepSeek-V3.2 hidden_states must have shape [batch, sequence, hidden]")
        batch_size, seq_length = hidden_states.shape[:-1]
        if past_key_values is not None:
            raise NotImplementedError("DeepseekV32SparseAttention does not support KV cache")
        if DeepseekV32SparseAttentionModule._invalid_position_ids(position_ids, batch_size, seq_length):
            raise ValueError("position_ids must have shape [1 or batch_size, sequence_length]")
        if attention_mask is not None:
            expected_shape = (batch_size, 1, seq_length, seq_length)
            if tuple(attention_mask.shape) != expected_shape:
                raise ValueError(
                    "DeepseekV32SparseAttention supports only a Transformers 4D causal mask; "
                    "padding and custom masks are not supported"
                )
        if kwargs.pop("output_attentions", False):
            raise NotImplementedError("DeepseekV32SparseAttention does not expose attention weights")

    def _project_attention_states(
        self, hidden_states: torch.Tensor, *, use_dense: bool = False,
    ) -> tuple[torch.Tensor, tuple[torch.Tensor, ...]]:
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
        actual_q_len: torch.Tensor,
        actual_kv_len: torch.Tensor,
    ) -> torch.Tensor:
        """Attach the sparse attention KL auxiliary loss when training enables it."""
        if not (self.training and not self.freeze_dsa and self.dsa_loss_coeff):
            return attn_output
        query_tnd, key_tnd, q_rot_tnd, k_rot_tnd = (
            tensor.reshape(-1, tensor.shape[2], tensor.shape[3])
            for tensor in attention_states
        )
        query_tnd, key_tnd, q_rot_tnd, k_rot_tnd, maximum, total = prepare_indexer_teacher(
            query_tnd, key_tnd, q_rot_tnd, k_rot_tnd, *softmax_stats,
        )
        aux_loss = dsa_kl_loss(
            *index_states, query_tnd, key_tnd,
            topk_indices, maximum, total, q_rot_tnd, k_rot_tnd,
            actual_q_len, actual_kv_len, self.scaling, self.dsa_loss_coeff,
            backend="cann",
697
698
699
700
701
702
703
704
705
706
707
708
709
            topk_indices, maximum, total, q_rot_tnd, k_rot_tnd,
            actual_q_len, actual_kv_len, self.scaling, self.dsa_loss_coeff,
            backend="cann",
        )
        return aux_loss_auto_scale(attn_output, aux_loss)

    def use_dense_attention(self) -> bool:
        """Read the current training stage unless an explicit override pins it."""
        return self.dsa_dense_warm_up if self._dense_warmup_override is None else self._dense_warmup_override

    def preflight_attention(
        self, hidden_states: torch.Tensor, attention_mask: torch.Tensor | None,
        past_key_values: Any | None, position_ids: torch.Tensor | None,
710
711
712
713
714
715
716
717
718
719
        actual_seq_len: torch.Tensor | Sequence[int] | None,
        kwargs: dict[str, Any], *, use_dense: bool,
    ) -> SparseAttentionSequenceMetadata:
        """Validate local inputs and construct packed geometry before projection."""
        del use_dense
        self._validate_forward_inputs(hidden_states, attention_mask, past_key_values, position_ids, kwargs)
        return SparseAttentionSequenceMetadata.build(
            batch_size=hidden_states.shape[0], local_seq_len=hidden_states.shape[1],
            device=hidden_states.device, actual_seq_len=actual_seq_len, kwargs=kwargs,
        )
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
        self, hidden_states: torch.Tensor, position_embeddings: tuple[torch.Tensor, torch.Tensor] | None,
        metadata: SparseAttentionSequenceMetadata, *, use_dense: bool,
    ) -> SparseAttentionInputs:
        """Prepare local operands while retaining one invocation's resolved mode."""
        index_states, attention_states = self.prepare_attention(
            hidden_states, position_embeddings, use_dense=use_dense,
        )
        return SparseAttentionInputs(index_states, attention_states, metadata, use_dense)

    def exchange_attention(self, inputs: SparseAttentionInputs) -> SparseAttentionInputs:
        """Return ready operands; local attention needs no sequence exchange."""
        return inputs

    def run_attention(
        self, index_states: tuple[torch.Tensor, ...], attention_states: tuple[torch.Tensor, ...],
        sequence_lengths: tuple[torch.Tensor, torch.Tensor], *, use_dense: bool,
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
        self, index_states: tuple[torch.Tensor, ...], attention_states: tuple[torch.Tensor, ...],
        sequence_lengths: tuple[torch.Tensor, torch.Tensor], *, use_dense: bool,
    ) -> torch.Tensor:
        """Execute the shared dense/sparse kernel and restore local-head output."""
        if use_dense:
            return _deepseek_v32_dense_attention(
                self, index_states, attention_states, sequence_lengths,
            ).flatten(2)
        topk, *flattened = dsa_indexer(
            *index_states, *sequence_lengths, self.index_topk, backend="cann",
        )
        output, maximum, total = dsa_sparse_attention(
            *attention_states, topk, self.scaling, *sequence_lengths, backend="cann",
        )
        output = self._apply_auxiliary_loss(
            output, tuple(flattened), attention_states, topk, (maximum, total), *sequence_lengths,
        )
        heads = output.shape[2]
        weight = self.kv_b_proj.weight.view(
            heads, self.qk_nope_head_dim + self.v_head_dim, self.kv_lora_rank,
        )
        return _restore_attention_projection(
            output, weight[:, self.qk_nope_head_dim:].transpose(1, 2), num_heads=heads,
            batch_size=output.shape[0], seq_length=output.shape[1],
            kv_lora_rank=self.kv_lora_rank, value_head_dim=self.v_head_dim,
        )
774
775
776
777
778
779
780
781
782
        )

    def project_output(self, output: torch.Tensor) -> torch.Tensor:
        """Run the output Linear before the shared TP boundary."""
        return self.o_proj(output)

    def forward(
        self, hidden_states: torch.Tensor,
        position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None,
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
        position_ids: torch.Tensor | None = None,
        actual_seq_len: torch.Tensor | Sequence[int] | None = None, **kwargs: Any,
    ) -> tuple[torch.Tensor, None]:
        """Run causal or packed sparse attention while retaining the Transformers output ABI."""
        use_dense = self.use_dense_attention()
        metadata = self.preflight_attention(
            hidden_states, attention_mask, past_key_values, position_ids, actual_seq_len, kwargs,
            use_dense=use_dense,
        )
        inputs = self.prepare_attention_inputs(
            hidden_states, position_embeddings, metadata, use_dense=use_dense,
        )
        inputs = self.exchange_attention(inputs)
        output = self.run_attention(
            inputs.index_states, inputs.attention_states, inputs.metadata.sequence_lengths, use_dense=inputs.use_dense,
        )
        del inputs
        return self.project_output(output), None


def _projection_output(result: torch.Tensor | tuple[torch.Tensor, torch.Tensor | None]) -> torch.Tensor:
    """Read the activation from native Linear or source tuple-returning projections."""
    return result[0] if isinstance(result, tuple) else result


class DeepseekSparseAttentionModule(nn.Module):
    """Sparse attention with flat source-compatible parameters and native construction.
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
        """
        super().__init__()
        del module_fqn, context
        if (config is None) == (module is None):
            raise ValueError("DeepseekSparseAttention requires exactly one of config or module")
        if cp_strategy != "kv_allgather":
            raise ValueError("DeepseekSparseAttention supports only cp_strategy='kv_allgather'")
        if (
            isinstance(module, DeepseekSparseAttentionModule)
            and getattr(module, "_hp_parallel_binding", None) is not None
        ):
            raise ValueError("Sparse-attention promotion must precede parallel binding and FSDP")
        self.config = config if module is None else module.config
        self._config_initialization = module is None or getattr(module, "_config_initialization", False)
        self.cp_strategy, self.overlap = cp_strategy, overlap
        self._dense_warmup_override = dense_warmup
853
854
855
856
857
858
859
860
861
        self._dense_warmup_override = dense_warmup
        self._initialize_dimensions(module)
        self._initialize_options(module)
        if dense_warmup or self.dsa_dense_warm_up:
            raise NotImplementedError("DeepseekSparseAttention does not support dense warm-up")
        if module is None:
            self._initialize_config_layers()
        else:
            for name in self._LAYER_NAMES:
860
861
862
863
864
865
866
867
868
869
870
871
872
        else:
            for name in self._LAYER_NAMES:
                setattr(self, name, getattr(module, name))
            if self.use_mome:
                for name in ("qa_conv", "compresskv_conv", "o_conv"):
                    setattr(self, name, getattr(module, name))
            if self.param_sink_number > 0:
                self.param_sink_k_pe = module.param_sink_k_pe
                self.param_sink_compressed_kv = module.param_sink_compressed_kv
            self.train(module.training)

    def _initialize_dimensions(self, module: nn.Module | None) -> None:
        """Read dimensions without requiring native-config aliases on source modules."""
878
879
880
881
882
883
884
885
886
        ):
            setattr(self, name, getattr(module, name, getattr(config, name, None)))
        self.qk_head_dim = self.qk_nope_head_dim + self.qk_rope_head_dim
        if self.q_lora_rank is None:
            raise ValueError("DeepseekSparseAttention requires the Q-LoRA projection path")
        self.num_index_heads = getattr(
            module, "num_index_heads",
            getattr(config, "index_n_heads", getattr(
                config, "index_num_attention_heads", getattr(config, "num_index_heads", None),
886
887
888
889
890
891
892
893
894
                config, "index_num_attention_heads", getattr(config, "num_index_heads", None),
            )),
        )
        if self.num_index_heads is None:
            raise ValueError("DeepseekSparseAttention requires the number of indexer heads")
        self.index_topk = getattr(module, "index_topk", getattr(config, "index_topk", None))

    def _initialize_options(self, module: nn.Module | None) -> None:
        """Preserve source options and supply matching defaults for native construction."""
895
896
897
898
899
900
901
902
903
        def option(name: str, default: Any) -> Any:
            return getattr(module, name, getattr(self.config, name, default))

        if option("mla_mm_split", False):
            raise NotImplementedError("DeepseekSparseAttention does not support split latent projections")
        self.rotary_interleaved = bool(option("rotary_interleaved", False))
        self.dsa_loss_coeff = option("dsa_loss_coeff", 0.0)
        self.freeze_dsa = bool(option("freeze_dsa", getattr(self.config, "freeze_DSA", False)))
        self.dsa_dense_warm_up = bool(option("dsa_dense_warm_up", False))
909
910
911
912
913
914
915
916
917
        self.apply_fa_rescale = bool(option("apply_fa_rescale", False))
        dropout = option("attention_dropout", 0.0)
        self.attention_dropout = dropout if isinstance(dropout, nn.Dropout) else nn.Dropout(dropout)
        if self.param_sink_number > 0 and (self.param_sink_scalar or not self.apply_fa_rescale):
            raise NotImplementedError(
                "DeepseekSparseAttention parameter sinks require apply_fa_rescale=True and no scalar sink"
            )

    def _initialize_config_layers(self) -> None:
918
919
920
921
922
923
924
925
926
        """Allocate native Linear, normalization, and optional local MOME parameters."""
        config = self.config
        dtype = getattr(config, "dtype", None)
        if isinstance(dtype, str):
            dtype = getattr(torch, dtype.removeprefix("torch."))
        factory_kwargs = {"device": getattr(config, "device", None), "dtype": dtype}
        bias = getattr(config, "attention_bias", False)
        self.linear_qkv = nn.Linear(
            config.hidden_size, self.q_lora_rank + self.kv_lora_rank + self.qk_rope_head_dim,
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
        self.index_linear_k = nn.Linear(config.hidden_size, self.index_head_dim, bias=False, **factory_kwargs)
        self.index_k_layernorm = nn.LayerNorm(self.index_head_dim, eps=1e-6, **factory_kwargs)
        self.linear_merge_weight = nn.Linear(config.hidden_size, self.num_index_heads, bias=False, **factory_kwargs)
        if self.use_mome:
            kernel_size = getattr(config, "mome_kernel_size", None)
            if kernel_size is None:
                raise ValueError("Native MOME construction requires config.mome_kernel_size")
            for name, channels in (
                ("qa_conv", self.q_lora_rank), ("compresskv_conv", self.kv_lora_rank),
                ("o_conv", self.num_heads * self.v_head_dim),
            ):
                setattr(self, name, nn.Conv1d(
                    channels, channels, kernel_size, groups=channels,
                    bias=getattr(config, "mome_bias", False), **factory_kwargs,
                ))
        if self.param_sink_number > 0:
            self.param_sink_k_pe = nn.Parameter(torch.zeros(
                self.param_sink_number, self.qk_rope_head_dim, **factory_kwargs,
            ))
            self.param_sink_compressed_kv = nn.Parameter(torch.zeros(
                self.param_sink_number, self.kv_lora_rank, **factory_kwargs,
            ))

    def reset_parameters(self) -> None:
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
            ))

    def reset_parameters(self) -> None:
        """Initialize native/meta parameters without resetting converted source weights."""
        if not self._config_initialization:
            return
        names = self._LAYER_NAMES + (("qa_conv", "compresskv_conv", "o_conv") if self.use_mome else ())
        for name in names:
            getattr(self, name).reset_parameters()
        if self.param_sink_number > 0:
            nn.init.zeros_(self.param_sink_k_pe)
            nn.init.zeros_(self.param_sink_compressed_kv)

    def make_transforms(self) -> list[WeightConverter]:
        """Keep source checkpoint names because flat Parameters are reused unchanged."""
        return []

    def _prepare_param_sink(self, batch_size: int) -> tuple[torch.Tensor, torch.Tensor]:
        """Construct the local sink key and value in BSND layout."""
        sink_rotary_key = self.param_sink_k_pe.unsqueeze(0).unsqueeze(2).expand(batch_size, -1, -1, -1)
        sink_latent = self.param_sink_compressed_kv.unsqueeze(0).unsqueeze(2).expand(batch_size, -1, -1, -1)
        sink_latent = self.k_layernorm(sink_latent)
        sink_value = (
            F.pad(sink_latent, (0, self.qk_rope_head_dim))
            if self.use_flash_attn and self.qk_rope_head_dim > 0 else sink_latent
        )
        return torch.cat((sink_latent, sink_rotary_key), dim=-1), sink_value

    def _validate_forward_inputs(
        self, attention_mask: torch.Tensor | None, kv_reuse_states: Any | None,
        past_key_values: Any | None, cache_position: torch.Tensor | None,
 996
 997
 998
 999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
        past_key_values: Any | None, cache_position: torch.Tensor | None,
        output_attentions: bool,
    ) -> None:
        """Reject unsupported contracts before projection or communication."""
        if attention_mask is not None:
            raise ValueError(
                "DeepseekSparseAttention does not consume attention_mask; "
                "pass packed sequence boundaries through actual_seq_len"
            )
        if kv_reuse_states is not None:
            raise NotImplementedError("DeepseekSparseAttention does not support KV reuse")
        if past_key_values is not None or cache_position is not None:
            raise NotImplementedError("DeepseekSparseAttention does not support KV cache")
        if output_attentions:
            raise NotImplementedError("DeepseekSparseAttention does not expose attention weights")

    def _project_attention_states(
        self, hidden_states: torch.Tensor, mome_mask: torch.Tensor | None,
    ) -> tuple[torch.Tensor, tuple[torch.Tensor, ...]]:
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
    def _project_attention_states(
        self, hidden_states: torch.Tensor, mome_mask: torch.Tensor | None,
    ) -> tuple[torch.Tensor, tuple[torch.Tensor, ...]]:
        """Project local head shards inside the attention/block FSDP boundary."""
        batch_size, seq_length = hidden_states.shape[:-1]
        latent_states = _projection_output(self.linear_qkv(hidden_states))
        q_latent, kv_nope, key_rope = latent_states.split(
            (self.q_lora_rank, self.kv_lora_rank, self.qk_rope_head_dim), dim=-1,
        )
        if self.use_mome:
            q_latent = apply_mome(q_latent, mome_mask, self.qa_conv, fused=False)
            kv_nope = apply_mome(kv_nope, mome_mask, self.compresskv_conv, fused=False)
        q_resid = self.q_layernorm(q_latent)
        projected = _projection_output(self.linear_qb(q_resid.contiguous()))
        heads = projected.shape[-1] // self.qk_head_dim
        query, query_rope = projected.view(batch_size, seq_length, heads, self.qk_head_dim).split(
            (self.qk_nope_head_dim, self.qk_rope_head_dim), dim=-1,
        )
        weight = self.linear_kvb.weight.view(heads, self.qk_nope_head_dim + self.v_head_dim, self.kv_lora_rank)
        query = _absorb_query_projection(
            query, weight[:, :self.qk_nope_head_dim], num_heads=heads,
            batch_size=batch_size, seq_length=seq_length,
            input_dim=self.qk_nope_head_dim, output_dim=self.kv_lora_rank,
        )
        kv_nope = self.k_layernorm(kv_nope).view(batch_size, seq_length, 1, self.kv_lora_rank)
        key_rope = key_rope.view(batch_size, seq_length, 1, self.qk_rope_head_dim)
        return q_resid, (query, kv_nope, query_rope, key_rope)

    def _project_index_states(
        self, hidden_states: torch.Tensor, q_resid: torch.Tensor,
        position_embeddings: tuple[torch.Tensor, torch.Tensor] | None,
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
        self, hidden_states: torch.Tensor, q_resid: torch.Tensor,
        position_embeddings: tuple[torch.Tensor, torch.Tensor] | None,
    ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
        """Build replicated indexer states and apply their rotary embedding."""
        batch_size, seq_length = hidden_states.shape[:-1]
        index_query = _projection_output(self.index_linear_qb(q_resid.detach()))
        index_query = index_query.view(batch_size, seq_length, self.num_index_heads, self.index_head_dim)
        index_key = _projection_output(self.index_linear_k(hidden_states.detach()))
        index_key = self.index_k_layernorm(index_key.unsqueeze(2))
        merge_weight = _projection_output(self.linear_merge_weight(hidden_states.detach()))
        merge_weight = merge_weight * self.num_index_heads**-0.5 * self.index_head_dim**-0.5
        index_query, index_key = _apply_index_rope(
            index_query, index_key, position_embeddings, rope_head_dim=self.qk_rope_head_dim,
            index_head_dim=self.index_head_dim, interleaved=self.rotary_interleaved,
        )
        return index_query, index_key, merge_weight

    def _run_sparse_attention(
        self, attention_states: tuple[torch.Tensor, ...], topk_indices: torch.Tensor,
        actual_q_len: torch.Tensor, actual_kv_len: torch.Tensor,
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
        self, attention_states: tuple[torch.Tensor, ...], topk_indices: torch.Tensor,
        actual_q_len: torch.Tensor, actual_kv_len: torch.Tensor,
    ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
        """Run sparse attention with optional local parameter-sink rescaling."""
        query, latent_kv, query_rope, key_rope = attention_states
        sparse_scale = self.qk_head_dim**-0.5
        if self.param_sink_number <= 0:
            return dsa_sparse_attention(  # pylint: disable=not-callable
                query, latent_kv, query_rope, key_rope, topk_indices,
                sparse_scale, actual_q_len, actual_kv_len,
            )
        batch_size, seq_length, heads = query.shape[:3]
        sink_key, sink_value = self._prepare_param_sink(batch_size)
        # Sink rescaling owns optional Omni dependencies; ordinary sparse attention does not.
        from hyper_parallel.components.functional.dsa_sparse_attention_rescale import (  # pylint: disable=C0415
            dsa_sparse_attention_rescale,
        )

        return dsa_sparse_attention_rescale(  # pylint: disable=not-callable
            query, latent_kv, query_rope, key_rope, sink_key, sink_value,
            topk_indices, batch_size, seq_length, heads, sparse_scale,
            1 - self.attention_dropout.p, actual_q_len, actual_kv_len,
        )
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
        attention_states: tuple[torch.Tensor, ...], topk_indices: torch.Tensor,
        maximum: torch.Tensor, total: torch.Tensor, sequence_lengths: tuple[torch.Tensor, torch.Tensor],
    ) -> torch.Tensor:
        """Attach the local-teacher Indexer loss through the shared loss scaler."""
        if not (self.training and not self.freeze_dsa and self.dsa_loss_coeff):
            return output
        query, key, query_rope, key_rope = (
            tensor.flatten(0, 1) for tensor in attention_states
        )
        query, key, query_rope, key_rope, maximum, total = prepare_indexer_teacher(
            query, key, query_rope, key_rope, maximum, total,
        )
        loss = dsa_kl_loss(  # pylint: disable=not-callable
            *index_states, query, key, topk_indices, maximum, total, query_rope, key_rope,
            *sequence_lengths, self.qk_head_dim**-0.5, self.dsa_loss_coeff,
        )
        return aux_loss_auto_scale(output, loss)

    def preflight_attention(
        self, hidden_states: torch.Tensor, actual_seq_len: torch.Tensor | Sequence[int] | None,
        kwargs: Mapping[str, Any],
1104
1105
1106
1107
1108
1109
1110
1111
        self, hidden_states: torch.Tensor, actual_seq_len: torch.Tensor | Sequence[int] | None,
        kwargs: Mapping[str, Any],
    ) -> SparseAttentionSequenceMetadata:
        """Resolve local sequence boundaries before context-parallel communication."""
        return SparseAttentionSequenceMetadata.build(
            batch_size=hidden_states.shape[0], local_seq_len=hidden_states.shape[1],
            device=hidden_states.device, actual_seq_len=actual_seq_len, kwargs=kwargs,
        )
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
        position_embeddings: tuple[torch.Tensor, torch.Tensor] | None,
        metadata: SparseAttentionSequenceMetadata, mome_mask: torch.Tensor | None,
    ) -> SparseAttentionInputs:
        """Prepare only activations, keeping no parameter views across CP exchange."""
        q_resid, attention_states = self._project_attention_states(hidden_states, mome_mask)
        query, key, query_rope, key_rope = attention_states
        query_rope, key_rope = _apply_attention_rope(
            query_rope, key_rope, position_embeddings, interleaved=self.rotary_interleaved,
        )
        index_states = self._project_index_states(hidden_states, q_resid, position_embeddings)
        return SparseAttentionInputs(index_states, (query, key, query_rope, key_rope), metadata, False)

    def exchange_attention(self, inputs: SparseAttentionInputs) -> SparseAttentionInputs:
        """Keep local inputs unchanged; CP family members exchange key banks."""
        return inputs

    def run_attention(
        self, index_states: tuple[torch.Tensor, ...], attention_states: tuple[torch.Tensor, ...],
        sequence_lengths: tuple[torch.Tensor, torch.Tensor], *, use_dense: bool = False,
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
        self, index_states: tuple[torch.Tensor, ...], attention_states: tuple[torch.Tensor, ...],
        sequence_lengths: tuple[torch.Tensor, torch.Tensor], *, use_dense: bool = False,
    ) -> torch.Tensor:
        """Run sparse selection/attention and restore this rank's value-head outputs."""
        if use_dense:
            raise NotImplementedError("DeepseekSparseAttention does not support dense warm-up")
        indices, *flattened_index_states = dsa_indexer(  # pylint: disable=not-callable
            *index_states, *sequence_lengths, self.index_topk,
        )
        output, maximum, total = self._run_sparse_attention(attention_states, indices, *sequence_lengths)
        output = self._apply_auxiliary_loss(
            output, tuple(flattened_index_states), attention_states, indices, maximum, total, sequence_lengths,
        )
        batch_size, seq_length, heads = attention_states[0].shape[:3]
        weight = self.linear_kvb.weight.view(heads, self.qk_nope_head_dim + self.v_head_dim, self.kv_lora_rank)
        return _restore_attention_projection(
            output, weight[:, self.qk_nope_head_dim:].transpose(1, 2), num_heads=heads,
            batch_size=batch_size, seq_length=seq_length,
            kv_lora_rank=self.kv_lora_rank, value_head_dim=self.v_head_dim,
        )
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
    def project_output(
        self, output: torch.Tensor, mome_mask: torch.Tensor | None = None, return_bias: bool = False,
    ) -> tuple[torch.Tensor, torch.Tensor | None]:
        """Apply local MOME and output projection while preserving the source tuple ABI."""
        if self.use_mome:
            output = apply_mome(output, mome_mask, self.o_conv, fused=self.use_fused_mome)
        projected = self.linear_proj(output)
        output, bias = projected if isinstance(projected, tuple) else (projected, None)
        return (output, bias) if return_bias else (output, None)

    def forward(
        self, hidden_states: torch.Tensor,
        position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None,
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
        kv_reuse_states: Any | None = None, output_attentions: bool = False, return_bias: bool = False,
        mome_mask: torch.Tensor | None = None, **kwargs: Any,
    ) -> tuple[torch.Tensor, torch.Tensor | None]:
        """Run the shared attention stages and preserve local MOME/sink behavior."""
        self._validate_forward_inputs(
            attention_mask, kv_reuse_states, past_key_values, cache_position, output_attentions,
        )
        metadata = self.preflight_attention(hidden_states, actual_seq_len, kwargs)
        inputs = self.prepare_attention_inputs(hidden_states, position_embeddings, metadata, mome_mask)
        inputs = self.exchange_attention(inputs)
        output = self.run_attention(
            inputs.index_states, inputs.attention_states, inputs.metadata.sequence_lengths, use_dense=inputs.use_dense,
        )
        del inputs
        return self.project_output(output, mome_mask, return_bias)
hyper_parallel/components/modules/attention/deepseek_sparse_attention/deepseek_sparse_attention_family.py
33
34
35
36
37
38
39
40
41
42
43
44
45
46
def validate_sparse_attention_family(*, config: Any, module: nn.Module | None, context: Any, member: Any) -> None:
    """Check logical topology before allocating native or replacement parameters."""
    config = config if module is None else module.config
    if context.get("sequence_parallel", False):
        raise ValueError("DeepSeek sparse attention does not support sequence parallelism")
    if member.axis_sizes["pp"] > 1:
        raise ValueError("DeepSeek sparse attention does not support pipeline parallelism")
    heads = getattr(module, "num_heads", getattr(config, "num_attention_heads", None))
    if heads % member.axis_sizes["tp"]:
        raise ValueError("Sparse-attention heads must be divisible by TP degree")


def validate_generic_sparse_attention_family(
    *, config: Any, module: nn.Module | None, context: Any, member: Any,
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
    sink_count = getattr(module, "param_sink_number", getattr(config, "param_sink_number", 0))
    if (member.axis_sizes["tp"] > 1 or member.axis_sizes["cp"] > 1) and (
        use_mome or sink_count > 0
    ):
        raise ValueError("MOME and parameter-sink sparse attention require TP1 and CP1")
    projection = getattr(module, "linear_proj", None)
    if member.axis_sizes["tp"] > 1 and getattr(projection, "bias", None) is not None and (
        not isinstance(projection, nn.Linear) or getattr(projection, "skip_bias_add", False)
    ):
        raise ValueError("TP sparse attention requires standard nn.Linear output-bias semantics")


def sparse_attention_sharding(*, module: nn.Module, module_fqn: str, member: Any, context: Any) -> ModuleShardingSpec:
    """Declare every owned parameter and external token port without name fallbacks."""
    del module_fqn, member
    if context.sequence_parallel:
        raise ValueError("DeepSeek sparse attention does not support sequence parallelism")
    placements = {axis: Replicate() for axis in context.mesh_dim_names if axis in context.axis_sizes}
    generic = hasattr(module, "linear_kvb")
    column_names = ("linear_qb", "linear_kvb") if generic else ("q_b_proj", "kv_b_proj")
    replica_names = (
        "linear_qkv", "q_layernorm", "k_layernorm", "index_linear_qb", "index_linear_k",
        "index_k_layernorm", "linear_merge_weight", "qa_conv", "compresskv_conv", "o_conv",
    ) if generic else (
        "linear_qkv", "q_a_layernorm", "kv_a_layernorm",
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
    ) if generic else (
        "linear_qkv", "q_a_layernorm", "kv_a_layernorm",
        "indexer.wq_b", "indexer.wk", "indexer.k_norm", "indexer.weights_proj",
    )
    output_name = "linear_proj" if generic else "o_proj"
    columns = {f"{name}.{suffix}" for name in column_names for suffix in ("weight", "bias")}
    replicas = {
        f"{name}.{suffix}"
        for name in replica_names
        for suffix in ("weight", "bias")
    }
    roles = {
        **dict.fromkeys(columns, Shard(0)), **dict.fromkeys(replicas, Replicate()),
        f"{output_name}.weight": Shard(1), f"{output_name}.bias": Replicate(),
        "param_sink_k_pe": Replicate(), "param_sink_compressed_kv": Replicate(),
    }
    params = {}
    for name, _ in module.named_parameters(remove_duplicate=False):
        if name not in roles:
            raise ValueError(f"Sparse-attention parameter {name!r} has no declared sharding role")
        params[name] = dict(placements)
        if context.axis_sizes["tp"] > 1:
            params[name]["tp"] = roles[name]
    tokens = dict(placements)
    if context.axis_sizes["cp"] > 1:
        tokens["cp"] = Shard(1)
    output = dict(tokens)
    if context.axis_sizes["tp"] > 1:
        output["tp"] = Partial()
    return ModuleShardingSpec(
        params=params, in_src={"hidden_states": dict(tokens)}, in_dst={"hidden_states": dict(tokens)},
        out_src={"output": output}, out_dst={"output": dict(tokens)},
        out_names=("output", "bias" if generic else "attention_weights"), is_boundary=True,
    )
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129


def validate_sparse_attention_layout(*, module: nn.Module, sharding: Any, context: Any) -> None:
    """Validate the final overridden contract, including the unsupported SP boundary."""
    expected = sparse_attention_sharding(module=module, module_fqn="", member=None, context=context)
    validate_tp_boundary(sharding, expected, name="DeepSeek sparse attention")
    for field, port in (("in_src", "hidden_states"), ("out_dst", "output")):
        if getattr(sharding, field)[port].get("tp", Replicate()) != Replicate():
            raise ValueError("DeepSeek sparse attention does not support SP token-port overrides")
    if tuple(sharding.out_names or ()) != tuple(expected.out_names):
        raise ValueError("Sparse-attention output names must preserve the declared return contract")


def v32_sparse_attention_parallel_spec(*, module: nn.Module, member: Any) -> ParallelModuleSpec:
    """Declare shared algorithm stages and validate layouts outside local computation."""
    del module, member
    return ParallelModuleSpec(
        "attention.deepseek_v32_sparse_attention.local",
        stages={
            "preflight_attention": (
                "hidden_states", "attention_mask", "past_key_values", "position_ids",
139
140
141
142
143
144
145
146
147
148


def sparse_attention_parallel_spec(*, module: nn.Module, member: Any) -> ParallelModuleSpec:
    """Declare generic sparse-attention stages and the replicated Indexer contract."""
    del module, member
    return ParallelModuleSpec(
        "attention.deepseek_sparse_attention.local",
        stages={
            "preflight_attention": ("hidden_states", "actual_seq_len", "kwargs"),
            "prepare_attention_inputs": ("hidden_states", "position_embeddings", "metadata", "mome_mask"),