Diff Coverage

Diff: origin/master...HEAD, staged and unstaged changes

Source File Diff Coverage (%) Missing Lines
hyper_parallel/components/modules/engram/__init__.py 100%  
hyper_parallel/components/modules/engram/base_engram.py 96.6% 41,48,70
hyper_parallel/components/modules/engram/engram.py 100%  
hyper_parallel/components/modules/engram/host_engram.py 95.8% 48
hyper_parallel/components/modules/engram/host_engram_table.py 100%  
hyper_parallel/components/modules/engram/ngram_hash_mapping.py 91.9% 49,55,61,88,95,106
hyper_parallel/components/modules/engram/parallel_engram.py 61.4% 49-51,70-74,80-82,84,138,140,169,196-220,235-239,241-244,258-262,283,306-307,310-321,325-330,334-337,342,409,432,435,437,439
hyper_parallel/components/modules/gate.py 88.2% 49,51,54,56,82,95,138,151,153,168,178,190,203,249,258-260,265-267
hyper_parallel/components/modules/mhc/__init__.py 100%  
hyper_parallel/components/modules/mhc/_utils.py 76.7% 27,35-38,45,47,55,63,79
hyper_parallel/components/modules/mhc/mega_mhc.py 87.9% 33,46,49,60,65,69,98,103,138,146-147
hyper_parallel/components/modules/mhc/mhc.py 100%  
hyper_parallel/components/modules/mhc/mhc_post.py 95.7% 46
hyper_parallel/components/modules/mhc/mhc_pre.py 82.4% 51,74,76,78,83,86,93,95-96,103,105,122-123
hyper_parallel/components/modules/mhc/pipelined_mhc.py 90.4% 42,59,65,71,73,75,109
hyper_parallel/components/modules/rms_norm.py 91.0% 38,54,72,131,138,148,182,185,187,199,203,242
hyper_parallel/components/modules/rotary_embedding.py 85.9% 41,47,51,53,61,85,91,94,100,104,130,134,144,148
hyper_parallel/components/modules/router.py 82.0% 34,39,41,44,48,66,68,72,76
hyper_parallel/components/modules/shared_expert.py 72.9% 96,100,102,104,106,108,110,112,120,122,208,214,217,231-234,253,275,349-352
hyper_parallel/components/modules/spatial_merge_projector.py 84.3% 34,38,46,48,54,60,70,73
hyper_parallel/components/modules/swiglu_mlp.py 79.5% 45,48,59,61,74,77-78,96,118,121,210,216,235,237,239,252,296,317,325-327,329-333,347
hyper_parallel/trainer/config/resolver.py 100%  
hyper_parallel/components/modules/engram/base_engram.py
37
38
39
40
41
42
43
44
45
        """Reuse a source Engram module's parameters and hash mapping."""
        super().__init__()
        del module_fqn, context
        if (module is None) == (config is None):
            raise ValueError("provide exactly one of config or module")
        if config is not None:
            self._init_from_config(config)
            return
        required = ("hash_mapping", "embed", "wkv", "q_weight", "k_weight")
44
45
46
47
48
49
50
51
52
            return
        required = ("hash_mapping", "embed", "wkv", "q_weight", "k_weight")
        missing = [name for name in required if not hasattr(module, name)]
        if missing:
            raise TypeError(f"Engram source is missing required attributes: {missing}")
        self.hash_mapping = module.hash_mapping
        self.embed = module.embed
        self.wkv = module.wkv
        self.q_weight = module.q_weight
66
67
68
69
70
71
72
73
74
            value = getattr(config, name)
            if isinstance(value, bool) or not isinstance(value, int) or value <= 0:
                raise ValueError(f"{name} must be a positive integer")
        if isinstance(config.layer_id, bool) or not isinstance(config.layer_id, int):
            raise ValueError("layer_id must be an integer")
        for name in ("eps", "clamp_value"):
            value = getattr(config, name)
            if isinstance(value, bool) or not isinstance(value, (int, float)) or not isfinite(value) or value <= 0:
                raise ValueError(f"{name} must be finite and positive")
hyper_parallel/components/modules/engram/host_engram.py
44
45
46
47
48
49
50
51
52
        training = self.embed.training
        source = module if module is not None else config
        weight = self.embed.weight
        if tuple(weight.shape) != (self.padded_num_embeddings, self.embed.embedding_dim):
            raise ValueError("Host Engram replacement requires the full physical table before planning")
        if module is None:
            weight = nn.Parameter(torch.empty(weight.shape, device="meta", dtype=torch.float32))
        self.embed = HostEngramTable(
            source_weight=weight, logical_rows=self.logical_num_embeddings,
hyper_parallel/components/modules/engram/ngram_hash_mapping.py
45
46
47
48
49
50
51
52
53
        token_map = torch.tensor(assets["token_map"], dtype=torch.long, device="cpu")
        self.pad_id = int(token_map[int(assets["pad_token_id"])])
        primes = torch.tensor(assets["primes"][layer_index], dtype=torch.long, device="cpu")
        if tuple(primes.shape) != (self.max_ngram_size - 1, self.num_heads):
            raise ValueError(
                "Engram prime layout must be [max_ngram_size - 1, num_heads], "
                f"got {tuple(primes.shape)}"
            )
        flattened_primes = primes.flatten()
51
52
53
54
55
56
57
58
59
                f"got {tuple(primes.shape)}"
            )
        flattened_primes = primes.flatten()
        if self.max_ngram_size < 2 or self.num_heads <= 0 or bool((primes <= 0).any()):
            raise ValueError("Engram requires ngram size >=2, positive heads and positive hash moduli")
        offsets = torch.cat(
            [torch.zeros(1, dtype=torch.long, device="cpu"), flattened_primes.cumsum(0)[:-1]]
        ).view_as(primes)
        multipliers = torch.tensor(assets["multipliers"][layer_index], dtype=torch.long, device="cpu")
57
58
59
60
61
62
63
64
65
            [torch.zeros(1, dtype=torch.long, device="cpu"), flattened_primes.cumsum(0)[:-1]]
        ).view_as(primes)
        multipliers = torch.tensor(assets["multipliers"][layer_index], dtype=torch.long, device="cpu")
        if multipliers.numel() != self.max_ngram_size:
            raise ValueError("Engram multiplier count must equal max_ngram_size")
        self.logical_num_embeddings = int(flattened_primes.sum().item())
        initial_buffers = {
            "token_map": token_map,
            "primes": primes,
84
85
86
87
88
89
90
91
92
            segment_starts: Optional packed sequence boundaries.
            token_mask: Optional mask excluding image tokens.
        """
        if input_ids.ndim != 2:
            raise ValueError(f"Engram expects input_ids [B,S], got {tuple(input_ids.shape)}")
        if input_ids.shape[1] == 0:
            return input_ids.new_empty((input_ids.shape[0], 0, (self.max_ngram_size - 1) * self.num_heads),
                                       dtype=torch.long)
        compressed = self.token_map[input_ids]
91
92
93
94
95
96
97
98
99
                                       dtype=torch.long)
        compressed = self.token_map[input_ids]
        if token_mask is not None:
            if token_mask.shape != input_ids.shape:
                raise ValueError("Engram token_mask must have the same shape as input_ids")
            compressed = torch.where(
                token_mask.to(torch.bool),
                compressed,
                compressed.new_full((), self.DEAD_TOKEN),
102
103
104
105
106
107
108
109
        positions = torch.arange(sequence_length, device=input_ids.device).expand(batch_size, -1)
        segment_ids = None
        if segment_starts is not None:
            if segment_starts.shape != input_ids.shape:
                raise ValueError("Engram segment_starts must have the same shape as input_ids")
            starts = segment_starts.to(torch.bool).clone()
            starts[:, 0] = True
            segment_ids = starts.to(torch.long).cumsum(dim=1)
hyper_parallel/components/modules/engram/parallel_engram.py
45
46
47
48
49
50
51
52
53
54
55


def _all_gather_sequence(value: torch.Tensor, group: Any, group_size: int) -> torch.Tensor:
    """Gather equal-size sequence chunks and concatenate along dimension one."""
    gathered = [torch.empty_like(value) for _ in range(group_size)]
    dist.all_gather(gathered, value.contiguous(), group=group)
    return torch.cat(gathered, dim=1)


def aligned_hash_ids(
    self,
66
67
68
69
70
71
72
73
74
75
76
77
78
) -> torch.Tensor:
    """Hash the local sequence plus the left halo required by n-grams."""
    cp_local_ids = input_ids
    if cp_size > 1:
        input_ids = _all_gather_sequence(input_ids, cp_group, cp_size)
        if segment_starts is not None:
            segment_starts = _all_gather_sequence(segment_starts, cp_group, cp_size)
        if token_mask is not None:
            token_mask = _all_gather_sequence(token_mask, cp_group, cp_size)
    local_sequence = hidden_states.shape[1]
    cp_sequence = cp_local_ids.shape[1]
    if cp_sequence == local_sequence:
        sequence_parallel_parts = 1
76
77
78
79
80
81
82
83
84
85
86
87
88
    cp_sequence = cp_local_ids.shape[1]
    if cp_sequence == local_sequence:
        sequence_parallel_parts = 1
        local_tp_rank = 0
    elif cp_sequence % local_sequence == 0 and cp_sequence // local_sequence == tp_size:
        sequence_parallel_parts = tp_size
        local_tp_rank = tp_rank
    else:
        raise ValueError(
            "Engram input_ids length is incompatible with the local hidden sequence: "
            f"ids={cp_sequence}, hidden={local_sequence}, tp_size={tp_size}"
        )
    del sequence_parallel_parts
134
135
136
137
138
139
140
141
142
143
144
        table: Owner-local Host Engram table.
        global_ids: Global Engram row IDs to look up.
    """
    if table.rows_per_owner is None or table.global_row_start is None:
        raise RuntimeError("Host Engram table is not bound")
    if table.weight.device.type != "cpu" or table.weight.dtype != torch.float32:
        raise RuntimeError("Host Engram weight must remain CPU FP32")
    flat_ids = global_ids.reshape(-1).long()
    unique_ids, inverse = torch.unique(flat_ids, sorted=True, return_inverse=True)
    table.lookup_requests += flat_ids.numel()
    table.lookup_unique_ids += unique_ids.numel()
165
166
167
168
169
170
171
172
173
        owned_local_ids = (owned_global_ids - table.global_row_start).to("cpu")
        invalid_owner = bool(owned_local_ids.numel() and (
            owned_local_ids.min() < 0 or owned_local_ids.max() >= table.rows_per_owner))
        if stage_check(invalid_owner, table.mesh_context):
            raise IndexError("Engram EP routing delivered a row to the wrong owner")
        owner_values = HostRowsFn.apply(table.weight, owned_local_ids, table)
        transfer_values = owner_values.to(table.lookup_compute_dtype)
        table.forward_a2a_bytes += unique_ids.numel() * unique_ids.element_size()
        table.forward_a2a_bytes += transfer_values.numel() * transfer_values.element_size()
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
        coo_cpu: Owner-local sparse COO gradient on CPU.
        axis_mesh: Mesh axis used to synchronize sparse rows.
        width: Number of values in each table row.
    """
    if axis_mesh is None or axis_mesh.size() == 1:
        return coo_cpu
    group = axis_mesh.get_group()
    device = _communication_device(axis_mesh)
    coo_cpu = coo_cpu.coalesce()
    nnz = coo_cpu._nnz()  # pylint: disable=protected-access
    local_count = torch.tensor([nnz], dtype=torch.long, device=device)
    count_parts = [torch.empty_like(local_count) for _ in range(axis_mesh.size())]
    dist.all_gather(count_parts, local_count, group=group)
    counts = [int(part.item()) for part in count_parts]
    max_count = max(counts)
    if max_count == 0:
        return coo_cpu
    ids = torch.zeros(max_count, dtype=torch.long, device=device)
    values = torch.zeros((max_count, width), dtype=torch.float32, device=device)
    if nnz:
        ids[:nnz] = coo_cpu.indices()[0].to(device)
        values[:nnz] = coo_cpu.values().to(device)
    id_parts = [torch.empty_like(ids) for _ in counts]
    value_parts = [torch.empty_like(values) for _ in counts]
    dist.all_gather(id_parts, ids, group=group)
    dist.all_gather(value_parts, values, group=group)
    gathered_ids = torch.cat([part[:count].cpu() for part, count in zip(id_parts, counts)])
    gathered_values = torch.cat([part[:count].cpu() for part, count in zip(value_parts, counts)])
    return torch.sparse_coo_tensor(
        gathered_ids.unsqueeze(0), gathered_values, coo_cpu.shape,
    ).coalesce()

231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
        width: Number of values in each table row.
    """
    if mesh_context is None or mesh_context.device_mesh is None:
        return coo_cpu
    moe = mesh_context.fsdp_moe_mesh
    if moe is None:
        for name in ("dp", "cp", "tp"):
            if name in (mesh_context.device_mesh.mesh_dim_names or ()):
                coo_cpu = gather_sparse_along_axis(coo_cpu, mesh_context.device_mesh[name], width)
    else:
        for name in ("edp_shard", "edp_replicate"):
            if name in (moe.mesh_dim_names or ()):
                coo_cpu = gather_sparse_along_axis(coo_cpu, moe[name], width)
    return coo_cpu


def stage_reduce_scalar(value: torch.Tensor, mesh_context: Any,
                        op: Any = dist.ReduceOp.SUM) -> torch.Tensor:
254
255
256
257
258
259
260
261
262
263
264
265
266
        op: Distributed reduction operation.
    """
    if mesh_context is None or mesh_context.device_mesh is None:
        return value
    for name in mesh_context.device_mesh.mesh_dim_names or ():
        axis = mesh_context.device_mesh[name]
        if axis.size() > 1:
            dist.all_reduce(value, op=op, group=axis.get_group())
    return value


def stage_check(local_bad: bool, mesh_context: Any) -> bool:
    """Align validation failure across every rank in this PP stage.
279
280
281
282
283
284
285
286
287
                   ep_rank: int, ep_size: int, padded_num_embeddings: int) -> torch.Tensor:
    """Use the already-bound Host row route without applying the device-table route twice."""
    if (embedding.ep_rank != ep_rank or embedding.ep_size != ep_size or embedding.ep_group is not ep_group
            or embedding.physical_rows != padded_num_embeddings):
        raise ValueError("Host lookup must match the table's planned row partition")
    return embedding(hash_ids)


class ParallelEngram:
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
        ep_size: int,
        padded_num_embeddings: int,
    ) -> torch.Tensor:
        """Route requested global rows to their EP owners and restore order."""
        if padded_num_embeddings % ep_size:
            raise ValueError(
                f"Engram padded rows {padded_num_embeddings} are not divisible by ep_size={ep_size}"
            )
        num_local_rows = padded_num_embeddings // ep_size
        row_start = ep_rank * num_local_rows
        flat_ids = hash_ids.reshape(-1)
        owner = torch.div(flat_ids, num_local_rows, rounding_mode="floor")
        if owner.numel() and int(owner.max()) >= ep_size:
            raise IndexError("Engram hash requested a row outside the padded table")
        local_counts = torch.bincount(owner, minlength=ep_size).to(torch.long)
        gathered_counts = [torch.empty_like(local_counts) for _ in range(ep_size)]
        dist.all_gather(gathered_counts, local_counts, group=ep_group)
        all_counts = torch.stack(gathered_counts)
        send_counts = local_counts.cpu().tolist()
        recv_counts = all_counts[:, ep_rank].cpu().tolist()

        # Ascend ArgSort executes integer inputs on AiCPU. The owner range is
        # bounded by ep_size, so float32 preserves it exactly and stays on AiCore.
        sort_indices = torch.argsort(owner.float(), stable=True)
        sorted_ids = flat_ids.index_select(0, sort_indices)
        received_ids = ep_all_to_all(sorted_ids, send_counts, recv_counts, ep_group)
        local_ids = received_ids - row_start
        if local_ids.numel() and (int(local_ids.min()) < 0 or int(local_ids.max()) >= num_local_rows):
            raise IndexError("Engram EP routing delivered a row to the wrong owner")
        # Call the child module instead of reading ``embedding.weight``
        # directly. This preserves the FSDP pre/post-forward lifecycle when
        # the EP table is wrapped independently from the dense Engram gates.
        values = embedding(local_ids)
        returned = ep_all_to_all(values, recv_counts, send_counts, ep_group)
        restored = torch.zeros_like(returned)
        restored.scatter_add_(
            0,
            sort_indices.unsqueeze(1).expand(-1, returned.shape[1]),
            returned,
        )
        return restored.view(*hash_ids.shape, embedding.embedding_dim)

    def _aligned_hash_ids(
        self, hidden_states: torch.Tensor, input_ids: torch.Tensor, segment_starts: torch.Tensor | None,
        token_mask: torch.Tensor | None = None, *, cp_group: Any = None, cp_rank: int = 0, cp_size: int = 1,
405
406
407
408
409
410
411
412
413
        fused = self._fuse(hidden_states, embeddings)
        token_mask = local_token_mask(token_mask, hidden_states, tp_rank=tp_rank, tp_size=tp_size)
        if token_mask is None:
            return fused
        return torch.where(token_mask.to(torch.bool).unsqueeze(-1).unsqueeze(-1), fused, hidden_states)


class ParallelHostTable:
    """Own existing EP partition binding without placing it in local row storage."""
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
    def bind_planned_shard(self, *, ep_rank: int, ep_size: int, ep_group: Any = None,
                           compute_dtype: torch.dtype = torch.float32) -> None:
        """Bind the planner's single EP row shard to its global row interval."""
        if ep_size < 1 or not 0 <= ep_rank < ep_size or self.physical_rows % ep_size:
            raise ValueError("Invalid EP row partition")
        rows = self.physical_rows // ep_size
        if not self.weight.is_meta or self.weight.dtype != torch.float32:
            raise ValueError("Planner must leave a meta FP32 Host table")
        if tuple(self.weight.shape) != (rows, self.width):
            raise ValueError("Planner did not produce one EP row shard")
        if ep_size > 1 and ep_group is None:
            raise ValueError("EP routing requires an explicit process group")
        self.rows_per_owner = rows
        self.global_row_start = ep_rank * rows
        self.ep_rank = ep_rank
        self.ep_size = ep_size
hyper_parallel/components/modules/gate.py
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
    This contract has no group mask. TokenChoiceTopKRouter retains its separate
    scale-before-group/selection behavior and weights/indices/counts protocol.
    """
    if scores.ndim != 2 or not scores.is_floating_point():
        raise ValueError("scores must be a floating [tokens, experts] tensor")
    if isinstance(top_k, bool) or not isinstance(top_k, int) or not 1 <= top_k <= scores.shape[-1]:
        raise ValueError("top_k must be an integer in [1, num_experts]")
    if (isinstance(routed_scaling_factor, bool) or not isinstance(routed_scaling_factor, (int, float))
            or not isfinite(routed_scaling_factor)):
        raise ValueError("routed_scaling_factor must be finite")
    if not isinstance(normalize_topk, bool) or not isinstance(sorted_indices, bool):
        raise TypeError("normalize_topk and sorted_indices must be bool")
    selection = scores
    if correction_bias is not None:
        if (correction_bias.device != scores.device or not correction_bias.is_floating_point()
                or tuple(correction_bias.shape) not in (
78
79
80
81
82
83
84
85
86
    def __init__(self, *, config: Any = None, module: nn.Module | None = None) -> None:
        """Construct routing math or retain a public member; model state is not a replacement boundary."""
        super().__init__()
        if (config is None) == (module is None):
            raise ValueError("provide exactly one of config or module")
        if module is not None:
            self._validate_source(module)
        source = config if config is not None else module
        self._set_routing_settings(source)
91
92
93
94
95
96
97
98
99
        else:
            weight = getattr(module, "weight", None)
            if (not isinstance(weight, nn.Parameter) or not weight.is_floating_point()
                    or tuple(weight.shape) != (self.num_experts, self.hidden_size)):
                raise ValueError("source weight must be a floating Parameter [num_experts, hidden_size]")
            self.weight = weight
            self.train(module.training)

    def _set_routing_settings(self, source: Any) -> None:
134
135
136
137
138
139
140
141
142
    def forward(self, hidden_states: torch.Tensor,
                correction_bias: torch.Tensor | None = None) -> tuple[torch.Tensor, ...]:
        """Return flattened logits, scaled weights and selected expert IDs."""
        if hidden_states.ndim < 2 or hidden_states.shape[-1] != self.hidden_size:
            raise ValueError("hidden_states must end in hidden_size and include a token dimension")
        logits = F.linear(  # pylint: disable=not-callable
            hidden_states.reshape(-1, self.hidden_size).float(), self.weight.float())
        weights, indices = select_topk_scores(
            routing_scores(logits, self.scoring_func), top_k=self.top_k, correction_bias=correction_bias,
147
148
149
150
151
152
153
154
155
156
157

@lru_cache(maxsize=None)
def _plan_for_device(device: torch.device) -> Any:
    # HyperMulticore is optional and its runtime is needed only for native NPU calls.
    from hyper_multicore.modules.mega_gate.plan import build_mega_gate_plan  # pylint: disable=import-outside-toplevel

    return build_mega_gate_plan(device)


class MegaGateModule(GateModule):
    """Extend the ordinary gate with correction biases and a fused NPU route.
164
165
166
167
168
169
170
171
172
                 module_fqn: str = "", context: Mapping[str, Any] | None = None) -> None:
        """Construct or adopt the public MegaGate without accepting model-specific subclasses."""
        del module_fqn, context
        if (config is None) == (module is None):
            raise ValueError("provide exactly one of config or module")
        if module is not None:
            self._validate_source(module)
        source = config if config is not None else module
        if (getattr(source, "normalize_topk", source.top_k > 1) != (source.top_k > 1)
174
175
176
177
178
179
180
181
182
            raise ValueError("MegaGate requires normalize_topk=(top_k > 1) and unsorted indices")
        vision_enabled = (getattr(source, "vision_enabled", False)
                          if config is not None else getattr(source, "bias_vl", None) is not None)
        if not isinstance(vision_enabled, bool):
            raise TypeError("vision_enabled must be bool")
        super().__init__(config=config, module=module)
        for name in ("bias", "bias_vl"):
            if name == "bias_vl" and not vision_enabled:
                self.register_parameter(name, None)
186
187
188
189
190
191
192
193
194
            else:
                parameter = getattr(module, name, None)
                if (not isinstance(parameter, nn.Parameter) or tuple(parameter.shape) != (self.num_experts,)
                        or parameter.device != self.weight.device or parameter.dtype != torch.float32):
                    raise ValueError(f"source {name} must be an FP32 Parameter [num_experts] on the weight device")
            setattr(self, name, parameter)

    @classmethod
    def _validate_source(cls, module: nn.Module) -> None:
199
200
201
202
203
204
205
206
207
        # Existing HyperMulticore modules remain valid sources; this is not a base class.
        from hyper_multicore.modules.mega_gate import MegaGate as SourceMegaGate  # pylint: disable=import-outside-toplevel

        if module.__class__ is not SourceMegaGate:
            raise TypeError("MegaGate family accepts only the public MegaGate; use a model adapter for other routers")

    def reset_parameters(self) -> None:
        """Reset the common projection and MegaGate-specific correction state."""
        super().reset_parameters()
245
246
247
248
249
250
251
252
253
    def forward(self, hidden_states: torch.Tensor,
                image_mask: torch.Tensor | None = None) -> tuple[torch.Tensor, ...]:
        """Use shared Gate math or the equivalent native sqrt-softplus NPU route."""
        if hidden_states.ndim < 2 or hidden_states.shape[-1] != self.hidden_size:
            raise ValueError("hidden_states must end in hidden_size and include a token dimension")
        if image_mask is not None:
            if self.bias_vl is None:
                raise ValueError("image_mask requires vision_enabled=True when constructing MegaGate")
            if (image_mask.shape != hidden_states.shape[:2] or image_mask.dtype != torch.bool
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
                    or image_mask.device != hidden_states.device):
                raise ValueError("image_mask must be BOOL [batch, sequence] on the hidden_states device")
        if self.scoring_func != "sqrtsoftplus" or hidden_states.device.type != "npu" or hidden_states.numel() == 0:
            return self._torch_forward(hidden_states, image_mask)
        if image_mask is not None and not image_mask.is_contiguous():
            raise ValueError("image_mask must be contiguous for native MegaGate")
        return self._native_forward(hidden_states, image_mask)

    @classmethod
    def from_module(cls, module: nn.Module, **configuration: Any) -> MegaGateModule:
        """Retain the source's declared settings; overrides require a separate explicit construction."""
        if configuration:
            raise ValueError("family source construction preserves source settings; overrides are unsupported")
        return cls(module=module)


@module_replacement
def Gate(  # pylint: disable=invalid-name
hyper_parallel/components/modules/mhc/_utils.py
23
24
25
26
27
28
29
30
31

def _required_attribute(module: nn.Module, name: str) -> Any:
    """Return a required source-module attribute."""
    if not hasattr(module, name):
        raise TypeError(f"{module.__class__.__name__} is missing required attribute '{name}'")
    return getattr(module, name)


def _required_config_value(module: nn.Module, name: str) -> Any:
31
32
33
34
35
36
37
38
39
40
41
42
def _required_config_value(module: nn.Module, name: str) -> Any:
    """Return a required value from the source module or its config."""
    if hasattr(module, name):
        return getattr(module, name)
    config = getattr(module, "config", None)
    if config is None or not hasattr(config, name):
        raise TypeError(f"{module.__class__.__name__} is missing required MHC setting '{name}'")
    return getattr(config, name)


def _num_stream(module: nn.Module) -> int:
    """Return and validate the source module's residual-stream count."""
41
42
43
44
45
46
47
48
49
50
51
def _num_stream(module: nn.Module) -> int:
    """Return and validate the source module's residual-stream count."""
    value = getattr(module, "num_stream", None)
    if value is None:
        value = _required_config_value(module, "mhc_num_stream")
    if isinstance(value, bool) or not isinstance(value, int) or value <= 0:
        raise ValueError(f"MHC num_stream must be a positive integer, but got {value}")
    return value


def _parameter(module: nn.Module, name: str) -> nn.Parameter:
51
52
53
54
55
56
57
58
59
def _parameter(module: nn.Module, name: str) -> nn.Parameter:
    """Return a required source parameter."""
    value = _required_attribute(module, name)
    if not isinstance(value, nn.Parameter):
        raise TypeError(f"MHC attribute '{name}' must be an nn.Parameter")
    return value


def _validate_parameter_layout(phi: nn.Linear, parameters: tuple[nn.Parameter, ...]) -> None:
59
60
61
62
63
64
65
66
67
def _validate_parameter_layout(phi: nn.Linear, parameters: tuple[nn.Parameter, ...]) -> None:
    """Validate that MHC parameters share the projection's device and dtype."""
    for parameter in parameters:
        if parameter.device != phi.weight.device or parameter.dtype != phi.weight.dtype:
            raise ValueError("MHC projection and branch parameters must share device and dtype")


def _validate_settings(num_stream: int, hidden_size: int | None = None, sinkhorn_iters: int = 20,
                       hc_eps: float = 1e-6, norm_eps: float = 1e-6, use_gamma: bool = False) -> None:
75
76
77
78
79
    for name, value in (("hc_eps", hc_eps), ("norm_eps", norm_eps)):
        if isinstance(value, bool) or not isinstance(value, (int, float)) or not isfinite(value) or value <= 0:
            raise ValueError(f"{name} must be finite and positive")
    if not isinstance(use_gamma, bool):
        raise TypeError("use_gamma must be bool")
hyper_parallel/components/modules/mhc/mega_mhc.py
29
30
31
32
33
34
35
36
37
    def __init__(self, *, config: Any = None, module: nn.Module | None = None) -> None:
        """Initialize logical settings or retain compatible source Parameters."""
        super().__init__()
        if (config is None) == (module is None):
            raise ValueError("provide exactly one of config or module")
        source = config if config is not None else module
        self._set_settings(source)
        shapes = {"phi": (24, 4 * self.hidden_size), "alpha": (3,), "bias": (24,),
                  "norm_weight": (self.hidden_size,)}
42
43
44
45
46
47
48
49
50
51
52
53
            else:
                parameter = getattr(module, name, None)
                if (not isinstance(parameter, nn.Parameter) or tuple(parameter.shape) != shape
                        or parameter.dtype != dtype):
                    raise ValueError(f"source {name} must have shape {shape} and dtype {dtype}")
            setattr(self, name, parameter)
        if len({parameter.device for parameter in self.parameters()}) != 1:
            raise ValueError("MegaMhc parameters must share a device")
        if module is None:
            self.reset_parameters()
        else:
            self.train(module.training)
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
        """Validate and retain dimensions, normalization and tiling settings."""
        for name in ("hidden_size", "num_iters"):
            value = getattr(source, name)
            if isinstance(value, bool) or not isinstance(value, int) or value < 1:
                raise ValueError(f"{name} must be a positive integer")
            setattr(self, name, value)
        for name in ("hc_eps", "norm_eps"):
            value = getattr(source, name)
            if isinstance(value, bool) or not isinstance(value, (int, float)) or not isfinite(value) or value <= 0:
                raise ValueError(f"{name} must be finite and positive")
            setattr(self, name, value)
        self.token_tile = getattr(source, "token_tile", 32)
        if isinstance(self.token_tile, bool) or not isinstance(self.token_tile, int) or self.token_tile < 1:
            raise ValueError("token_tile must be a positive integer")

    def reset_parameters(self) -> None:
        """Restore predictor and normalization initialization after materialization."""
        nn.init.normal_(self.phi, std=(4 * self.hidden_size) ** -0.5)
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
                previous_pre_mix: torch.Tensor, previous_post_mix: torch.Tensor,
                previous_residual_mix: torch.Tensor) -> tuple[torch.Tensor, ...]:
        """Return residual, three next mappings and normalized next-block input."""
        if residual.shape[-2:] != (4, self.hidden_size):
            raise ValueError("residual must end in [4, hidden_size]")
        prefix = residual.shape[:-2]
        expected = ((previous_output, (*prefix, self.hidden_size)), (previous_pre_mix, (*prefix, 4)),
                    (previous_post_mix, (*prefix, 4)), (previous_residual_mix, (*prefix, 4, 4)))
        if any(tuple(value.shape) != shape for value, shape in expected):
            raise ValueError("shifted boundary tensors must share the same token dimensions")
        updated = (torch.einsum("...ji,...jh->...ih", previous_residual_mix.float(), residual.float())
                   + previous_post_mix.float().unsqueeze(-1) * previous_output.float().unsqueeze(-2)).to(residual.dtype)
        updated_fp32 = updated.float()
        variance = updated_fp32.square().mean((-2, -1), keepdim=True)
134
135
136
137
138
139
140
141
142
                validate_mega_mhc_hidden_size,
            )
            validate_mega_mhc_hidden_size(source.hidden_size, need_backward=True)
            if source.num_iters != 20:
                raise ValueError("fused MegaMhc requires num_iters=20")
        super().__init__(config=config, module=module)

    def forward(self, previous_output: torch.Tensor, residual: torch.Tensor,
                previous_pre_mix: torch.Tensor, previous_post_mix: torch.Tensor,
142
143
144
145
146
147
148
149
150
                previous_pre_mix: torch.Tensor, previous_post_mix: torch.Tensor,
                previous_residual_mix: torch.Tensor) -> tuple[torch.Tensor, ...]:
        """Delegate native shape/dtype validation and execution to multicore."""
        # Native operators are optional and loaded only when fused execution is requested.
        from hyper_multicore.modules.mega_mhc.function import hyper_mega_mhc  # pylint: disable=C0415
        return hyper_mega_mhc(previous_output, residual, previous_pre_mix, previous_post_mix,
                              previous_residual_mix, self.phi, self.alpha, self.bias, self.norm_weight,
                              hc_eps=self.hc_eps, norm_eps=self.norm_eps, num_iters=self.num_iters,
                              token_tile=self.token_tile)
hyper_parallel/components/modules/mhc/mhc_post.py
42
43
44
45
46
47
48
49
50
        """
        super().__init__()
        del module_fqn, context
        if (module is None) == (config is None):
            raise ValueError("provide exactly one of config or module")
        if config is not None:
            _validate_settings(config.num_stream)
            self.config = config
            self.num_stream = config.num_stream
hyper_parallel/components/modules/mhc/mhc_pre.py
47
48
49
50
51
52
53
54
55
        """
        super().__init__()
        del module_fqn, context
        if (module is None) == (config is None):
            raise ValueError("provide exactly one of config or module")
        if config is not None:
            _validate_settings(config.num_stream, config.hidden_size, config.sinkhorn_iters,
                               config.hc_eps, config.norm_eps, config.use_gamma)
            self.config = config
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
        self.num_stream = _num_stream(module)

        phi = _required_attribute(module, "phi")
        if not isinstance(phi, nn.Linear) or phi.bias is not None:
            raise TypeError("MhcPreModule requires a bias-free nn.Linear phi projection")
        if phi.in_features % self.num_stream != 0:
            raise ValueError("MHC pre phi input size must be divisible by num_stream")
        if phi.out_features != (self.num_stream + 2) * self.num_stream:
            raise ValueError("MHC pre phi output size must equal (num_stream + 2) * num_stream")
        self.phi = phi
        self.branch_alpha = _parameter(module, "branch_alpha")
        self.branch_beta = _parameter(module, "branch_beta")
        if self.branch_alpha.numel() != 3:
            raise ValueError("MHC pre branch_alpha must contain three values")
        expected_beta_size = 2 * self.num_stream + self.num_stream * self.num_stream
        if self.branch_beta.numel() != expected_beta_size:
            raise ValueError("MHC pre branch_beta has an incompatible size")

        self.mhc_use_gamma = bool(_required_config_value(module, "mhc_use_gamma"))
        parameters = [self.branch_alpha, self.branch_beta]
        if self.mhc_use_gamma:
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
        parameters = [self.branch_alpha, self.branch_beta]
        if self.mhc_use_gamma:
            self.norm_gamma = _parameter(module, "norm_gamma")
            if self.norm_gamma.numel() != phi.in_features:
                raise ValueError("MHC pre norm_gamma size must equal the phi input size")
            parameters.append(self.norm_gamma)
        elif hasattr(module, "norm_gamma"):
            raise ValueError("MHC pre source has norm_gamma while mhc_use_gamma is disabled")
        _validate_parameter_layout(phi, tuple(parameters))

        self.hc_eps = float(_required_attribute(module, "hc_eps"))
        self.norm_eps = float(_required_attribute(module, "norm_eps"))
 99
100
101
102
103
104
105
106
107
108
        self.hc_eps = float(_required_attribute(module, "hc_eps"))
        self.norm_eps = float(_required_attribute(module, "norm_eps"))
        self.mhc_recur_norm = int(_required_config_value(module, "mhc_recur_norm"))
        if self.hc_eps <= 0 or self.norm_eps <= 0:
            raise ValueError("MHC eps values must be positive")
        if self.mhc_recur_norm <= 0:
            raise ValueError("mhc_recur_norm must be a positive integer")
        _validate_settings(self.num_stream, phi.in_features // self.num_stream,
                     self.mhc_recur_norm, self.hc_eps, self.norm_eps, self.mhc_use_gamma)
        self.train(module.training)
118
119
120
121
122
123
124
125
126
127
    def forward(
        self, x: torch.Tensor
    ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor | None]:
        """Prepare hidden states and return the MHC mixing coefficients."""
        gamma = self.norm_gamma if self.mhc_use_gamma else None
        return mhc_pre(
            x,
            self.phi.weight,
            self.branch_alpha,
            self.branch_beta,
hyper_parallel/components/modules/mhc/pipelined_mhc.py
38
39
40
41
42
43
44
45
46
        """Reuse the source module's ``fn/base/scale`` parameter layout."""
        super().__init__()
        del module_fqn, context
        if (module is None) == (config is None):
            raise ValueError("provide exactly one of config or module")
        if config is not None:
            _validate_settings(config.num_stream, config.hidden_size, config.sinkhorn_iters,
                               config.hc_eps, config.norm_eps)
            self.hc_mult = config.num_stream
55
56
57
58
59
60
61
62
63
            return
        required = ("input_norm", "fn", "base", "scale", "hc_mult")
        missing = [name for name in required if not hasattr(module, name)]
        if missing:
            raise TypeError(f"pipelined MHC source is missing required attributes: {missing}")
        self.input_norm = module.input_norm
        self.fn = module.fn
        self.base = module.base
        self.scale = module.scale
61
62
63
64
65
66
67
68
69
        self.fn = module.fn
        self.base = module.base
        self.scale = module.scale
        if not all(isinstance(value, nn.Parameter) for value in (self.fn, self.base, self.scale)):
            raise TypeError("pipelined MHC fn/base/scale must be Parameters")
        self.hc_mult = int(module.hc_mult)
        self.hc_sinkhorn_iters = int(module.hc_sinkhorn_iters)
        self.hc_eps = float(module.hc_eps)
        expected_mix = (self.hc_mult + 2) * self.hc_mult
67
68
69
70
71
72
73
74
75
76
77
78
        self.hc_sinkhorn_iters = int(module.hc_sinkhorn_iters)
        self.hc_eps = float(module.hc_eps)
        expected_mix = (self.hc_mult + 2) * self.hc_mult
        if tuple(self.fn.shape)[0] != expected_mix or self.base.numel() != expected_mix:
            raise ValueError("pipelined MHC fn/base shapes do not match hc_mult")
        if self.scale.numel() != 3:
            raise ValueError("pipelined MHC scale must contain three values")
        if self.fn.ndim != 2 or self.fn.shape[1] % self.hc_mult:
            raise ValueError("pipelined MHC fn must be a matrix with stream-divisible input width")
        _validate_settings(self.hc_mult, self.fn.shape[1] // self.hc_mult,
                              self.hc_sinkhorn_iters, self.hc_eps)
        self.train(module.training)
105
106
107
108
109
110
111
112
        post = 2 * torch.sigmoid(post * post_scale + post_bias)
        residual = residual.view(*residual.shape[:-1], num_stream, num_stream)
        residual = residual * residual_scale + residual_bias.view(num_stream, num_stream)
        if hidden_streams.device.type == "npu" and hasattr(torch.ops.custom, "npu_sinkhorn"):
            residual = sinkhorn(residual, self.hc_sinkhorn_iters, self.hc_eps)
        else:
            residual = sinkhorn_knopps(residual, self.hc_sinkhorn_iters, self.hc_eps)
        return pre, post, residual
hyper_parallel/components/modules/rms_norm.py
34
35
36
37
38
39
40
41
    """Validate normalization semantics without requiring a concrete config class."""
    if isinstance(hidden_size, bool) or not isinstance(hidden_size, int) or hidden_size <= 0:
        raise ValueError("RMSNormModule hidden_size must be a positive integer")
    if isinstance(eps, bool) or not isinstance(eps, (int, float)):
        raise TypeError("RMSNormModule eps must be a real scalar")
    if not isfinite(eps) or eps <= 0:
        raise ValueError("RMSNormModule eps must be finite and positive")

50
51
52
53
54
55
56
57
58
    if source_type.__name__ in ("Qwen3RMSNorm", "Qwen3MoeRMSNorm", "Qwen2MoeRMSNorm",
                                "Qwen3VLMoeTextRMSNorm", "DeepseekV3RMSNorm", "DeepseekV32RMSNorm",
                                "DeepseekV4RMSNorm"):
        return "direct"
    return None


class _RMSNormPolicy(torch.nn.Module):
    """Declare rounding order and optional FP32 scale ownership at construction."""
68
69
70
71
72
73
74
75
            raise ValueError("cast_order must be before_weight or after_weight")
        if fp32_weight is None:
            fp32_weight = getattr(source, "fp32_weight", False)
        if not isinstance(fp32_weight, bool):
            raise TypeError("fp32_weight must be bool")
        self.rms_norm_cast_order = cast_order
        self.fp32_weight = fp32_weight
        self.fp32_parameter_names = ("weight",) if fp32_weight else ()
127
128
129
130
131
132
133
134
135
        del module_fqn, context
        self._set_policy(config if config is not None else module, cast_order, fp32_weight)
        if config is not None:
            if module is not None or hidden_size is not None:
                raise ValueError("config cannot be combined with module or hidden_size")
            _validate_rms_settings(config.hidden_size, config.eps)
            hidden_size, eps = config.hidden_size, config.eps
        if module is not None:
            if _hf_scale_kind(module) == "offset" or isinstance(module, OffsetRMSNormModule):
134
135
136
137
138
139
140
141
142
        if module is not None:
            if _hf_scale_kind(module) == "offset" or isinstance(module, OffsetRMSNormModule):
                raise ValueError("unit-offset source requires OffsetRMSNorm, not direct-scale RMSNorm")
            if not isinstance(getattr(module, "weight", None), torch.nn.Parameter):
                raise TypeError("RMSNormModule source module must expose an nn.Parameter weight")
            source_eps = getattr(module, "eps", None)
            if source_eps is None:
                source_eps = getattr(module, "variance_epsilon", None)
            if source_eps is None:
144
145
146
147
148
149
150
151
152
                    "RMSNormModule source module must expose eps or variance_epsilon"
                )
            self.eps = float(source_eps)
            if module.weight.ndim != 1:
                raise ValueError("RMSNormModule source weight must be one-dimensional")
            _validate_rms_settings(module.weight.numel(), self.eps)
            if self.fp32_weight and module.weight.dtype != torch.float32:
                raise ValueError("fp32_weight source must already own an FP32 scale")
            self.weight = module.weight
178
179
180
181
182
183
184
185
186
187
188
189
190
191
        """Retain offset Parameters; checkpoints use the source's offset representation."""
        super().__init__()
        del module_fqn, context
        if (config is None) == (module is None):
            raise ValueError("provide exactly one of config or module")
        self._set_policy(config if config is not None else module, cast_order, fp32_weight)
        if source_representation not in ("offset", "scale"):
            raise ValueError("source_representation must be offset or scale")
        if config is not None and source_representation != "offset":
            raise ValueError("direct construction stores offsets; scale conversion requires a source")
        self.source_representation = source_representation
        if _hf_scale_kind(module) == "direct" and source_representation != "scale":
            raise ValueError("direct-scale HF source requires RMSNorm, not OffsetRMSNorm")
        if config is not None:
195
196
197
198
199
200
201
202
203
204
205
206
207
                config.hidden_size, dtype=torch.float32 if self.fp32_weight else None))
        else:
            weight = getattr(module, "weight", None)
            if not isinstance(weight, torch.nn.Parameter) or weight.ndim != 1:
                raise TypeError("offset source must expose a one-dimensional Parameter weight")
            self.eps = module.eps
            _validate_rms_settings(weight.numel(), self.eps)
            if self.fp32_weight and weight.dtype != torch.float32:
                raise ValueError("fp32_weight source must already own an FP32 offset")
            if source_representation == "scale":
                if isinstance(module, OffsetRMSNormModule) or _hf_scale_kind(module) == "offset":
                    raise ValueError("offset source must not be declared as a direct scale")
                weight = torch.nn.Parameter(weight.detach() - 1.0, requires_grad=weight.requires_grad)
238
239
240
241
242
243
244
245
246
    if cast_order == "before_weight":
        # The NPU kernel accumulates internally; retain the input dtype for saved activations.
        normalized = rms_norm(x, torch.ones_like(scale, dtype=x.dtype), eps)
        return scale * normalized
    return rms_norm(x.float(), scale.float(), eps).to(x.dtype)


def _native_rms_norm(x: torch.Tensor, scale: torch.Tensor, eps: float, cast_order: str) -> torch.Tensor:
    """Preserve FP32 accumulation and the source rounding boundary using Torch operations."""
hyper_parallel/components/modules/rotary_embedding.py
37
38
39
40
41
42
43
44
45
    def __init__(self, *, config: Any = None, module: nn.Module | None = None) -> None:
        """Build from logical config or adopt a supported static source frequency buffer."""
        super().__init__()
        if (config is None) == (module is None):
            raise ValueError("provide exactly one of config or module")
        if module is not None:
            source_config = getattr(module, "config", None)
            params = getattr(source_config, "rope_parameters", None) or getattr(source_config, "rope_scaling", {}) or {}
            self.rope_type = getattr(module, "rope_type", params.get("rope_type", "default"))
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
            source_config = getattr(module, "config", None)
            params = getattr(source_config, "rope_parameters", None) or getattr(source_config, "rope_scaling", {}) or {}
            self.rope_type = getattr(module, "rope_type", params.get("rope_type", "default"))
            if self.rope_type not in ("default", "linear"):
                raise ValueError("source conversion supports static default/linear RoPE; "
                                 "dynamic cache needs explicit config")
            frequency = getattr(module, "inv_freq", None)
            if not isinstance(frequency, torch.Tensor) or frequency.ndim != 1 or frequency.is_meta:
                raise ValueError("source requires a materialized one-dimensional inv_freq buffer")
            if any(True for _ in module.parameters()):
                raise ValueError("parameterized rotary sources require a separate state contract")
            self.dim = 2 * frequency.numel()
            self.base = float(params.get("rope_theta", getattr(source_config, "rope_theta", 10000.0)))
            self.factor = float(params.get("factor", 1.0))
            self.max_position_embeddings = int(getattr(source_config, "max_position_embeddings", 2048))
57
58
59
60
61
62
63
64
65
            self.max_position_embeddings = int(getattr(source_config, "max_position_embeddings", 2048))
            self.attention_factor = float(getattr(module, "attention_scaling", 1.0))
            self.mrope_section = tuple(getattr(module, "mrope_section", params.get("mrope_section", ())))
            if self.mrope_section:
                raise ValueError("multi-axis source requires an explicit position layout config")
            register_rebuildable_buffer(self, "inv_freq", value=frequency.detach().cpu(), persistent=False)
            self.inv_freq = frequency
            self.train(module.training)
        else:
81
82
83
84
85
86
87
88
89

    def _validate(self) -> None:
        """Validate supported frequency and position layouts before execution."""
        if isinstance(self.dim, bool) or not isinstance(self.dim, int) or self.dim < 2 or self.dim % 2:
            raise ValueError("rotary dim must be a positive even integer")
        if self.rope_type not in ("default", "linear", "dynamic"):
            raise ValueError("supported rope_type values are default, linear and dynamic")
        for name in ("base", "factor", "attention_factor"):
            value = getattr(self, name)
87
88
89
90
91
92
93
94
95
96
97
98
            raise ValueError("supported rope_type values are default, linear and dynamic")
        for name in ("base", "factor", "attention_factor"):
            value = getattr(self, name)
            if isinstance(value, bool) or not isinstance(value, (float, int)) or not isfinite(value) or value <= 0:
                raise ValueError(f"{name} must be finite and positive")
        if (isinstance(self.max_position_embeddings, bool)
                or not isinstance(self.max_position_embeddings, int) or self.max_position_embeddings < 1):
            raise ValueError("max_position_embeddings must be positive")
        self._validate_position_layout()

    def _validate_position_layout(self) -> None:
        """Validate dynamic NTK extent constraints and multi-axis section coverage."""
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108

    def _validate_position_layout(self) -> None:
        """Validate dynamic NTK extent constraints and multi-axis section coverage."""
        if self.rope_type == "dynamic" and (self.dim <= 2 or self.factor < 1):
            raise ValueError("dynamic NTK requires dim > 2 and factor >= 1")
        if self.mrope_section and (
                any(isinstance(x, bool) or not isinstance(x, int) or x < 1 for x in self.mrope_section)
                or sum(self.mrope_section) != self.dim // 2):
            raise ValueError("mrope_section must partition the half-dimension into positive axis sections")

    def _apply(self, fn: Any, recurse: bool = True) -> RotaryEmbeddingModule:
        # Frequencies must not be rounded by model-wide BF16 conversion.
        original = self.inv_freq
126
127
128
129
130
131
132
133
134
135
136
137
138
        Returns:
            Cosine and sine tensors shaped [B,S,dim].
        """
        if position_ids.device != x.device or self.inv_freq.device != x.device:
            raise ValueError("rotary buffers, positions and activations must share a device")
        frequency = self.inv_freq.float()
        if self.rope_type == "dynamic":
            if isinstance(sequence_length, bool) or not isinstance(sequence_length, int) or sequence_length < 1:
                raise ValueError("dynamic NTK requires explicit positive sequence_length")
            extent = max(sequence_length, self.max_position_embeddings)
            adjusted = self.base * (
                self.factor * extent / self.max_position_embeddings - (self.factor - 1)
            ) ** (self.dim / (self.dim - 2))
140
141
142
143
144
145
146
147
148
149
150
151
        # Explicit elementwise FP32 multiplication is unaffected by GEMM autocast.
        angles = position_ids.float().unsqueeze(-1) * frequency
        if position_ids.ndim == 3 and self.mrope_section:
            if position_ids.shape[0] != len(self.mrope_section):
                raise ValueError("position axes must match mrope_section")
            sections = angles.split(self.mrope_section, dim=-1)
            angles = torch.cat([section[axis] for axis, section in enumerate(sections)], dim=-1)
        elif position_ids.ndim != 2 or self.mrope_section:
            raise ValueError("positions must be [B,S], or sectioned [axes,B,S] with mrope_section")
        embedding = torch.cat((angles, angles), dim=-1)
        return ((embedding.cos() * self.attention_factor).to(x.dtype),
                (embedding.sin() * self.attention_factor).to(x.dtype))
hyper_parallel/components/modules/router.py
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52

    def __init__(self, *, config: Any = None, module: nn.Module | None = None) -> None:
        """Create a public router or reuse its gate Parameters and routing settings."""
        if (config is None) == (module is None):
            raise ValueError("provide exactly one of config or module")
        source = config if config is not None else module
        dim = config.dim if config is not None else module.gate.in_features
        for name, value in (("dim", dim), ("num_experts", source.num_experts), ("top_k", source.top_k)):
            if isinstance(value, bool) or not isinstance(value, int) or value < 1:
                raise ValueError(f"{name} must be a positive integer")
        if source.top_k > source.num_experts:
            raise ValueError("top_k cannot exceed num_experts")
        if (isinstance(source.route_scale, bool) or not isinstance(source.route_scale, (float, int))
                or not isfinite(source.route_scale)):
            raise ValueError("route_scale must be finite")
        self._validate_groups(source)
        if module is not None:
            if type(module) not in (SourceTokenChoiceTopKRouter, TokenChoiceTopKRouterModule):
                raise TypeError("model-specific routers require their own adapter")
            nn.Module.__init__(self)  # pylint: disable=non-parent-init-called
            self.gate = module.gate
            for name in ("num_experts", "top_k", "score_func", "num_expert_groups",
                         "num_limited_groups", "route_scale"):
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
        """Validate expert-group coverage and the number of selectable experts."""
        groups, limited = source.num_expert_groups, source.num_limited_groups
        if groups is not None:
            if any(isinstance(value, bool) or not isinstance(value, int) or value < 1 for value in (groups, limited)):
                raise ValueError("expert groups and limited groups must be positive integers")
            if source.num_experts % groups or limited > groups:
                raise ValueError("expert groups must divide experts and cover the selected limited groups")
            if source.top_k > limited * (source.num_experts // groups):
                raise ValueError("top_k exceeds the number of experts in selected groups")
        elif limited is not None:
            raise ValueError("num_limited_groups requires num_expert_groups")

    def reset_parameters(self) -> None:
        """Initialize the gate after meta materialization."""
        self.gate.reset_parameters()


@module_replacement
def TokenChoiceTopKRouter(  # pylint: disable=invalid-name
hyper_parallel/components/modules/shared_expert.py
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
    def _separate_projections(module: nn.Module) -> tuple[nn.Linear, nn.Linear, nn.Linear]:
        """Validate and return separate source projections."""
        source_linears = (module.gate_proj, module.up_proj, module.down_proj)
        if any(not isinstance(layer, nn.Linear) for layer in source_linears):
            raise TypeError("SharedExpertModule source projections must be nn.Linear instances")
        gate_proj, up_proj, down_proj = source_linears
        SwiGLUMLPModule._validate_projection_contract(gate_proj, up_proj, down_proj)  # pylint: disable=protected-access
        if gate_proj.in_features != up_proj.in_features:
            raise ValueError("SharedExpertModule gate and up projections must have the same input size")
        if gate_proj.out_features != up_proj.out_features:
            raise ValueError("SharedExpertModule gate and up projections must have the same output size")
        if down_proj.in_features != gate_proj.out_features:
            raise ValueError("SharedExpertModule down projection input size is incompatible with gate/up")
        if down_proj.out_features != gate_proj.in_features:
            raise ValueError("SharedExpertModule down projection output size is incompatible with gate/up")
        if gate_proj.weight.requires_grad != up_proj.weight.requires_grad:
            raise ValueError("SharedExpertModule gate and up weights must use the same training policy")
        if (gate_proj.bias is None) != (up_proj.bias is None):
            raise ValueError("SharedExpertModule gate and up projections must use the same bias policy")
        if gate_proj.bias is not None and gate_proj.bias.requires_grad != up_proj.bias.requires_grad:
            raise ValueError("SharedExpertModule gate and up biases must use the same training policy")
        return gate_proj, up_proj, down_proj

    def _initialize_fused_source(self, module: nn.Module) -> None:
        """Reuse an already fused source module without converting parameters."""
116
117
118
119
120
121
122
123
124
125
126
        """Reuse an already fused source module without converting parameters."""
        self.linear_fc1 = module.linear_fc1
        self.linear_fc2 = module.linear_fc2
        if not all(hasattr(layer, "skip_bias_add") for layer in (self.linear_fc1, self.linear_fc2)):
            raise TypeError("fused SharedExpertModule projections must return output and bias separately")
        if not hasattr(module, "activation_func"):
            raise TypeError("fused SharedExpertModule source module is missing activation_func")
        self.activation_func = module.activation_func
        self._return_bias_tuple = getattr(module, "_return_bias_tuple", True)
        self._gate_size = 0
        self._up_size = 0
204
205
206
207
208
209
210
211
212
        """Build the high-performance module from a Transformers shared expert."""
        super().__init__()
        del module_fqn, context
        if not isinstance(preserve_source_layout, bool):
            raise TypeError("preserve_source_layout must be bool")
        self._preserve_source_layout = preserve_source_layout
        self.swiglu_limit, self.activation_dtype = _activation_settings(
            config if config is not None else module, limit=swiglu_limit, activation_dtype=activation_dtype)
        self.limit = self.swiglu_limit
210
211
212
213
214
215
216
217
218
219
220
221
        self.swiglu_limit, self.activation_dtype = _activation_settings(
            config if config is not None else module, limit=swiglu_limit, activation_dtype=activation_dtype)
        self.limit = self.swiglu_limit
        if config is None and module is None:
            raise ValueError("provide exactly one of config or module")
        if config is not None:
            if module is not None:
                raise ValueError("provide config or module, not both")
            _validate_mlp_config(config)
            if not isinstance(config.return_bias_tuple, bool):
                raise TypeError("return_bias_tuple must be bool")
            self.config = config
227
228
229
230
231
232
233
234
235
236
237
238
                                           activation_dtype=self.activation_dtype)
            if self._preserve_source_layout:
                if config.return_bias_tuple:
                    raise ValueError("separate SharedExpert returns a tensor, not a bias tuple")
                self.gate_proj = nn.Linear(config.hidden_size, config.intermediate_size, bias=config.bias)
                self.up_proj = nn.Linear(config.hidden_size, config.intermediate_size, bias=config.bias)
                self.down_proj = nn.Linear(config.intermediate_size, config.hidden_size, bias=config.bias)
                return
            self.linear_fc1 = LinearWithMatmul(config.hidden_size, 2 * config.intermediate_size, bias=config.bias)
            self.linear_fc2 = LinearWithMatmul(config.intermediate_size, config.hidden_size, bias=config.bias)
            return
        config = getattr(module, "config", None)
249
250
251
252
253
254
255
256
257
            )

        if self._source_is_fused:
            if self._preserve_source_layout:
                raise ValueError("preserve_source_layout requires separate gate/up/down source projections")
            self._initialize_fused_source(module)
            return

        gate_proj, up_proj, down_proj = self._separate_projections(module)
271
272
273
274
275
276
277
278
279
    def _set_clamped_activation(self, config: Any) -> None:
        if self.swiglu_limit > 0 or self.activation_dtype == "float32":
            hidden_act = getattr(config, "hidden_act", getattr(config, "hidden_activation", None))
            if hidden_act not in (None, "silu", "swiglu"):
                raise ValueError("clamped/FP32 SharedExpert requires SwiGLU activation")
            self.activation_func = partial(local_swiglu, limit=self.swiglu_limit,
                                           activation_dtype=self.activation_dtype)

    def make_transforms(self) -> list[WeightRenaming | WeightConverter]:
345
346
347
348
349
350
351
352
353
354
355

    def reset_parameters(self) -> None:
        """Initialize owned projections after meta materialization."""
        if self._preserve_source_layout:
            self.gate_proj.reset_parameters()
            self.up_proj.reset_parameters()
            self.down_proj.reset_parameters()
            return
        self.linear_fc1.reset_parameters()
        self.linear_fc2.reset_parameters()

hyper_parallel/components/modules/spatial_merge_projector.py
30
31
32
33
34
35
36
37
38
39
40
41
42
    def __init__(self, *, config: Any = None, module: nn.Module | None = None) -> None:
        """Construct logical dimensions or reuse a compatible source's loaded projections."""
        super().__init__()
        if (config is None) == (module is None):
            raise ValueError("provide exactly one of config or module")
        if module is not None:
            self.downsample_ratio = module.downsample_ratio
            if not all(isinstance(getattr(module, key, None), nn.Linear) for key in ("w1", "w2")):
                raise TypeError("spatial projector source requires w1/w2 linear projections")
            self.w1, self.w2 = module.w1, module.w2
            self.train(module.training)
        else:
            self.downsample_ratio = config.downsample_ratio
42
43
44
45
46
47
48
49
50
51
52
            self.downsample_ratio = config.downsample_ratio
            for key in ("input_size", "hidden_size", "output_size"):
                value = getattr(config, key)
                if isinstance(value, bool) or not isinstance(value, int) or value < 1:
                    raise ValueError(f"{key} must be a positive integer")
            if not isinstance(config.bias, bool):
                raise TypeError("bias must be bool")
            self._validate_ratio()
            self.w1 = nn.Linear(config.input_size * self.downsample_ratio**2, config.hidden_size, bias=config.bias)
            self.w2 = nn.Linear(config.hidden_size, config.output_size, bias=config.bias)
        self._validate_ratio()
50
51
52
53
54
55
56
57
58
            self.w1 = nn.Linear(config.input_size * self.downsample_ratio**2, config.hidden_size, bias=config.bias)
            self.w2 = nn.Linear(config.hidden_size, config.output_size, bias=config.bias)
        self._validate_ratio()
        if self.w1.in_features % self.downsample_ratio**2 or self.w1.out_features != self.w2.in_features:
            raise ValueError("spatial projector dimensions do not match the merge ratio and hidden projection")

    def _validate_ratio(self) -> None:
        """Reject fractional, boolean or nonpositive spatial ratios."""
        if (isinstance(self.downsample_ratio, bool) or not isinstance(self.downsample_ratio, int)
56
57
58
59
60
61
62
63
64
    def _validate_ratio(self) -> None:
        """Reject fractional, boolean or nonpositive spatial ratios."""
        if (isinstance(self.downsample_ratio, bool) or not isinstance(self.downsample_ratio, int)
                or self.downsample_ratio < 1):
            raise ValueError("downsample_ratio must be a positive integer")

    def reset_parameters(self) -> None:
        """Initialize projections after direct meta construction."""
        self.w1.reset_parameters()
66
67
68
69
70
71
72
73
74
75
76
77

    def forward(self, hidden_states: Tensor, grid_height: int, grid_width: int) -> Tensor:
        """Pad one image grid, merge channel-major patches and project image-token rows."""
        if any(isinstance(size, bool) or not isinstance(size, int) or size < 1 for size in (grid_height, grid_width)):
            raise ValueError("grid dimensions must be positive integers")
        channels = self.w1.in_features // self.downsample_ratio**2
        if hidden_states.ndim != 2 or hidden_states.shape != (grid_height * grid_width, channels):
            raise ValueError("features must have shape [grid_height * grid_width, input_size]")
        ratio = self.downsample_ratio
        grid = hidden_states.reshape(grid_height, grid_width, channels).permute(2, 0, 1)
        grid = F.pad(grid, (0, -grid_width % ratio, 0, -grid_height % ratio))
        merged = F.unfold(grid.unsqueeze(0), ratio, stride=ratio).squeeze(0).transpose(0, 1)
hyper_parallel/components/modules/swiglu_mlp.py
41
42
43
44
45
46
47
48
49
50
51
52
        value = getattr(config, name)
        if isinstance(value, bool) or not isinstance(value, int) or value <= 0:
            raise ValueError(f"{name} must be a positive integer")
    if not isinstance(config.bias, bool):
        raise TypeError("bias must be bool")
    limit = getattr(config, "swiglu_limit", 0.0)
    if isinstance(limit, bool) or not isinstance(limit, (int, float)) or not isfinite(limit):
        raise ValueError("swiglu_limit must be finite; nonpositive values disable clamping")


def swiglu_activation(gate: torch.Tensor, up: torch.Tensor, *, limit: float = 0.0,
                       activation_dtype: str = "input") -> torch.Tensor:
55
56
57
58
59
60
61
62
63
64
65
    FP32 activation remains FP32 even when clamp is disabled. The projection
    owner decides when to cast back before its down projection.
    """
    if activation_dtype not in ("input", "float32"):
        raise ValueError("activation_dtype must be input or float32")
    if isinstance(limit, bool) or not isinstance(limit, (int, float)) or not isfinite(limit):
        raise ValueError("SwiGLU limit must be finite")
    if activation_dtype == "float32":
        gate, up = gate.float(), up.float()
    if limit > 0:
        gate, up = gate.clamp(max=limit), up.clamp(-limit, limit)
70
71
72
73
74
75
76
77
78
79
80
81
    """Apply packed SwiGLU, retaining the historical FP32 clamp default."""
    if activation_dtype is None:
        activation_dtype = "float32" if limit > 0 else "input"
    if activation_dtype not in ("input", "float32"):
        raise ValueError("activation_dtype must be input or float32")
    if x.device.type == "npu" and limit <= 0 and activation_dtype == "input":
        # Optional NPU operators are required only by NPU execution.
        from hyper_parallel.components.functional.swiglu import swiglu  # pylint: disable=C0415
        return swiglu(x)
    gate, up = x.chunk(2, dim=-1)
    return swiglu_activation(gate, up, limit=limit, activation_dtype=activation_dtype).to(x.dtype)

 92
 93
 94
 95
 96
 97
 98
 99
100
    if limit is None:
        limit = source.limit if hf_clamped_source else getattr(
            source, "swiglu_limit", 0.0 if hf_source else getattr(config, "swiglu_limit", 0.0))
    if isinstance(limit, bool) or not isinstance(limit, (int, float)) or not isfinite(limit):
        raise ValueError("swiglu_limit must be finite")
    if literal_source_limit and limit <= 0:
        raise ValueError("HF DeepseekV4 literal nonpositive clamp is not a disabled-clamp source contract")
    if activation_dtype is None:
        activation_dtype = getattr(source, "activation_dtype",
114
115
116
117
118
119
120
121
122
123
124
125
        """Validate and return the source Gate, Up, and Down projections."""
        required = ("gate_proj", "up_proj", "down_proj")
        missing = [name for name in required if not hasattr(module, name)]
        if missing:
            raise TypeError(f"SwiGLUMLPModule source module is missing: {missing}")
        projections = (module.gate_proj, module.up_proj, module.down_proj)
        if not all(isinstance(projection, nn.Linear) for projection in projections):
            raise TypeError("SwiGLUMLPModule requires Gate, Up, and Down projections to be nn.Linear")
        return projections

    @staticmethod
    def _validate_projection_contract(
206
207
208
209
210
211
212
213
214
        """
        super().__init__()
        del module_fqn, context
        if config is None and module is None:
            raise ValueError("provide exactly one of config or module")
        source = config if config is not None else module
        self.swiglu_limit, self.activation_dtype = _activation_settings(
            source, limit=swiglu_limit, activation_dtype=activation_dtype)
        if config is not None:
212
213
214
215
216
217
218
219
220
        self.swiglu_limit, self.activation_dtype = _activation_settings(
            source, limit=swiglu_limit, activation_dtype=activation_dtype)
        if config is not None:
            if module is not None:
                raise ValueError("provide config or module, not both")
            _validate_mlp_config(config)
            self.config = config
            self.hidden_size = config.hidden_size
            self.intermediate_size = config.intermediate_size
231
232
233
234
235
236
237
238
239
240
241
242
243
        if all(isinstance(getattr(module, name, None), nn.Linear) for name in ("w1", "w2")):
            source_config = getattr(module, "config", None)
            if (not hasattr(module, "swiglu_limit")
                    and getattr(source_config, "hidden_act", None) not in ("silu", "swiglu")):
                raise TypeError("packed w1/w2 source must explicitly declare SwiGLU activation semantics")
            if module.w1.out_features != 2 * module.w2.in_features or module.w2.out_features != module.w1.in_features:
                raise ValueError("packed SwiGLU source must contain [gate, up] rows followed by a down projection")
            if module.w1.weight.device != module.w2.weight.device or module.w1.weight.dtype != module.w2.weight.dtype:
                raise ValueError("packed SwiGLU projections must share device and dtype")
            self.config = getattr(module, "config", None)
            self.hidden_size, self.intermediate_size = module.w1.in_features, module.w2.in_features
            self.linear_fc1, self.linear_fc2 = module.w1, module.w2
            self._packed_source = True
248
249
250
251
252
253
254
255

        config = getattr(module, "config", None)
        hidden_act = getattr(config, "hidden_act", getattr(config, "hidden_activation", None))
        if hidden_act is not None and hidden_act not in ("silu", "swiglu"):
            raise ValueError(f"SwiGLUMLPModule requires a SiLU activation, but got {hidden_act}")
        self.config = config
        self.hidden_size = gate_proj.in_features
        self.intermediate_size = gate_proj.out_features
292
293
294
295
296
297
298
299
300

    def forward(self, x: torch.Tensor | None = None, *, hidden_states: torch.Tensor | None = None) -> torch.Tensor:
        """Apply the fused Gate/Up projection, SwiGLU, and Down projection."""
        if (x is None) == (hidden_states is None):
            raise ValueError("provide exactly one of x or hidden_states")
        x = hidden_states if x is None else x
        intermediate_parallel = self.linear_fc1(x)
        intermediate_parallel = local_swiglu(intermediate_parallel, limit=self.swiglu_limit,
                                             activation_dtype=self.activation_dtype)
313
314
315
316
317
318
319
320
321
                 activation_dtype: str | None = None, swiglu_limit: float | None = None) -> None:
        """Construct vision projections or retain a declared packed source unchanged."""
        super().__init__()
        if (config is None) == (module is None):
            raise ValueError("provide exactly one of config or module")
        self.swiglu_limit, self.activation_dtype = _activation_settings(
            config if config is not None else module, limit=swiglu_limit, activation_dtype=activation_dtype)
        if config is not None:
            _validate_mlp_config(config)
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
            _validate_mlp_config(config)
            self.w1 = nn.Linear(config.hidden_size, 2 * config.intermediate_size, bias=config.bias)
            self.w2 = nn.Linear(config.intermediate_size, config.hidden_size, bias=config.bias)
        else:
            if not all(isinstance(getattr(module, name, None), nn.Linear) for name in ("w1", "w2")):
                raise TypeError("w1_w2 source requires two Linear projections")
            if (module.w1.out_features != 2 * module.w2.in_features
                    or module.w1.in_features != module.w2.out_features):
                raise ValueError("w1_w2 source must contain gate/up rows and a compatible down projection")
            if module.w1.weight.dtype != module.w2.weight.dtype or module.w1.weight.device != module.w2.weight.device:
                raise ValueError("w1/w2 must share dtype and device")
            self.w1, self.w2 = module.w1, module.w2
            self.train(module.training)

    def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
        """Apply the shared SwiGLU math without changing source checkpoint names."""
        return self.w2(local_swiglu(self.w1(hidden_states), limit=self.swiglu_limit,
343
344
345
346
347
348
349
350
351
        self.w2.reset_parameters()

    def make_transforms(self) -> list[WeightRenaming | WeightConverter]:
        """Packed source and target share the same logical state names."""
        return []


@module_replacement
def SwiGLUMLP(  # pylint: disable=invalid-name