Diff Coverage

Diff: origin/master...HEAD, staged and unstaged changes

Source File Diff Coverage (%) Missing Lines
hyper_parallel/components/modules/attention/grouped_query_attention/cp_grouped_query_attention.py 0.0% 22,149,172-174,181,184
hyper_parallel/components/modules/attention/grouped_query_attention/grouped_query_attention_family.py 0.0% 67-68
hyper_parallel/components/modules/attention/grouped_query_attention/cp_grouped_query_attention.py
18
19
20
21
22
23
24
25

from dataclasses import dataclass
from functools import lru_cache
from itertools import accumulate
from math import gcd
from typing import Any

import torch
145
146
147
148
149
150
151
152
153
        del module_fqn, context
        super().__init__(config=config, module=module, fuse_qkv=fuse_qkv)
        self.overlap = overlap
        self.ulysses_degree = ulysses_degree
        self.kv_head_repeats = 1

    def parallel_module_spec(self) -> ParallelModuleSpec:
        """Request borrowed CP resources in addition to the shared TP boundary."""
        return ParallelModuleSpec("attention.gqa.cp", "cp",
168
169
170
171
172
173
174
175
176
177
178
        degree = self.cp_context.size
        self.ulysses_degree = degree if self.ulysses_degree is None else self.ulysses_degree
        if self.ulysses_degree < 1 or degree % self.ulysses_degree:
            raise ValueError("GQA Ulysses degree must divide CP degree")
        if self.num_heads % self.ulysses_degree:
            raise ValueError("GQA Ulysses degree must divide TP-local Q heads")
        self.kv_head_repeats = self.ulysses_degree // gcd(self.ulysses_degree, self.num_key_value_heads)
        self.ulysses_context, self.query_context = self.cp_context.split(self.ulysses_degree)
        self.all_gather_degree = degree // self.ulysses_degree

    def _transfer_input(self, tensor: torch.Tensor, *, is_query: bool) -> tuple[torch.Tensor, Any]:
177
178
179
180
181
182
183
184
185
186
187
188

    def _transfer_input(self, tensor: torch.Tensor, *, is_query: bool) -> tuple[torch.Tensor, Any]:
        """Keep Q local for KV AG; otherwise exchange this ready operand along U."""
        if self.ulysses_degree > 1:
            if not is_query and self.kv_head_repeats > 1:
                # Keep contiguous query groups mapped to their original KV head.
                # Repeat only to lcm(KV heads, U), not Q heads; autograd sums replica gradients.
                tensor = tensor.repeat_interleave(self.kv_head_repeats, dim=2)
            if self.overlap:
                return tensor, self.ulysses_context.launch_sequence_to_head(tensor)
            return self.ulysses_context.sequence_to_head(tensor), None
        if self.overlap and self.all_gather_degree > 1 and not is_query:
hyper_parallel/components/modules/attention/grouped_query_attention/grouped_query_attention_family.py
63
64
65
66
67
68
69
70
71
72
            "GQA KV heads must be divisible by TP degree for the current whole-KV-group layout; "
            "TP KV replication is not implemented"
        )

    if cp > 1 and config.num_attention_heads // tp % degree:
        raise ValueError("GQA Ulysses degree must divide TP-local Q heads")

def gqa_sharding(*, module: nn.Module, module_fqn: str, member: Any, context: Any) -> ModuleShardingSpec:
    """Own all GQA parameters and declare the actual external token layout."""
    del module_fqn, member