Diff Coverage

Diff: origin/master...HEAD, staged and unstaged changes

Source File Diff Coverage (%) Missing Lines
hyper_parallel/compile/compiler.py 93.8% 818
hyper_parallel/compile/passes/bucketing/bucketing_pass.py 0.0% 281-285
hyper_parallel/compile/passes/parallel/__init__.py 100%  
hyper_parallel/compile/passes/parallel/fsdp_pass.py 84.7% 94,96,98,601,664,778,1147,1155-1156
hyper_parallel/compile/compiler.py
814
815
816
817
818
819
820
821
822
            for name, (group, degree) in self._parameter_groups.items()
        }
        if self._expert_param_names:
            if self._expert_fsdp_mesh is None:
                raise ValueError("Expert parameters require fsdp_moe_mesh['edp_shard']")
            expert_degree = self._expert_fsdp_mesh.size()
            for name in self._expert_param_names:
                bound = specs.get(name)
                group = bound.group_name if bound is not None else "fsdp_expert"
hyper_parallel/compile/passes/bucketing/bucketing_pass.py
277
278
279
280
281
282
283
284
285
286
287
288
289
            return graph_module
        modules = pass_config.fsdp_bucket_modules
        if modules is not None:
            modules = build_module_bucket_plans(kwargs["model"], module_bucket_plans=modules)
        names = {kwargs.get("fsdp_group_name", "fsdp")}
        fsdp_plan = kwargs.get("fsdp_plan")
        if fsdp_plan is not None:
            names.update(spec.group_name for spec in fsdp_plan.params.values())
        names = sorted(names)
        groups = {name: _resolve_process_group(name) for name in names}
        ranks = {name: tuple(dist.get_process_group_ranks(group)) for name, group in groups.items()}
        for apply in (apply_ag_buckets, apply_rs_buckets):
            plan = plan_fsdp_buckets(graph_module, group_ranks=ranks, module_bucket_plans=modules)
hyper_parallel/compile/passes/parallel/fsdp_pass.py
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
    pad_policy: str = "pad"

    def __post_init__(self) -> None:
        if self.mode not in ("shard", "replicate", "skip"):
            raise ValueError(f"Invalid FSDP shard mode: {self.mode!r}")
        if self.pad_policy not in ("pad", "error", "replicate"):
            raise ValueError(f"Invalid FSDP pad policy: {self.pad_policy!r}")
        if isinstance(self.degree, bool) or not isinstance(self.degree, int) or self.degree < 1:
            raise ValueError(f"FSDP shard degree must be a positive integer, got {self.degree!r}")


@dataclass(frozen=True)
class FsdpPlan:
597
598
599
600
601
602
603
604
605
            spec = self._fsdp_plan.spec(name)
            if spec is not None:
                return spec
        if self._default_spec is None:
            raise RuntimeError("FSDPPass default shard spec is not initialized")
        return self._default_spec

    def _param_group(self, name: str) -> tuple[str, int]:
        """Choose the shard group and degree for one parameter."""
660
661
662
663
664
665
666
667
668
                continue

            spec = self._param_spec(fqn)
            if spec.mode == "skip":
                continue
            _, degree = self._param_group(fqn)
            if spec.mode == "replicate":
                replicated_indices.add(idx)
                continue
774
775
776
777
778
779
780
781
782
        spec = self._param_spec(fqn)
        _, degree = self._param_group(fqn)
        shape = param.shape
        if spec.pad_policy == "replicate":
            return True
        if not shape:
            if spec.pad_policy == "error":
                raise ValueError(
                    f"Parameter {fqn} with shape {tuple(shape)} cannot be sharded "
1143
1144
1145
1146
1147
1148
1149
1150
1151
            state_idx = trainable_state_indices[i - 1]
            fqn = state_fqns[state_idx]
            spec = self._param_spec(fqn)
            if spec.mode == "skip":
                continue
            group_name, degree = spec.group_name, spec.degree
            if state_idx in sharded_param_indices:
                replicated = False
                pad_info = self._padded_params.get(fqn)
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
                pad_info = self._padded_params.get(fqn)
            elif state_idx in replicated_param_indices:
                replicated = True
                pad_info = None
            elif spec.mode == "replicate":
                replicated = True
                pad_info = None
            elif degree == 1 and fqn in self._gradient_reductions:
                replicated = True
                pad_info = None