Coverage for  / home / jenkins / .local / lib / python3.10 / site-packages / hyper_parallel / compile / passes / overlap / schedule.py: 100%

1# Copyright 2026 Huawei Technologies Co., Ltd 

2# 

3# Licensed under the Apache License, Version 2.0 (the "License"); 

4# you may not use this file except in compliance with the License. 

5# You may obtain a copy of the License at 

6# 

7# http://www.apache.org/licenses/LICENSE-2.0 

8# 

9# Unless required by applicable law or agreed to in writing, software 

10# distributed under the License is distributed on an "AS IS" BASIS, 

11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 

12# See the License for the specific language governing permissions and 

13# limitations under the License. 

14# ============================================================================ 

15""" 

16Auto Overlap Pass - Communication-Compute Overlap Optimization 

17 

18Responsibilities: 

191. Find wait_tensor nodes for communication operations 

202. Analyze dependencies to find overlap opportunities 

213. Move wait_tensor to later position to maximize overlap 

22 

23Optimization Strategies: 

24- Move wait_tensor as late as possible (just before first use) 

25- Allow communication to overlap with independent computation 

26""" 

27 

28__all__ = ["AutoOverlapPass", "FsdpOverlapPass"] 

29 

30from typing import Any, List 

31from torch import fx 

32 

33from ...pass_config import PassConfig 

34from ..base import GraphPass 

35from .overlap_param_gather import overlap_param_gather 

36from .overlap_grad_comm import overlap_grad_comm 

37 

38 

39class AutoOverlapPass(GraphPass): 

40 """ 

41 Auto Overlap Pass 

42 

43 Automatically identifies opportunities to overlap communication with computation 

44 by reordering operations and issuing async communication early. 

45 """ 

46 

47 name = "auto_overlap" 

48 

49 def run( 

50 self, 

51 graph_module: fx.GraphModule, 

52 pass_config: PassConfig, 

53 **kwargs: Any, 

54 ) -> fx.GraphModule: 

55 """Optimize communication-compute overlap by moving wait_tensor nodes. 

56 

57 FSDPPass inserts wait_tensor immediately after communication (for correctness). 

58 This pass moves wait_tensor later to allow overlap with independent computation. 

59 """ 

60 if not pass_config.enable_overlap: 

61 return graph_module 

62 

63 graph = graph_module.graph 

64 wait_nodes = self._find_wait_nodes(graph) 

65 for wait_node in wait_nodes: 

66 graph_module = self._move_wait_later(graph_module, wait_node) 

67 

68 graph_module.recompile() 

69 return graph_module 

70 

71 def _find_wait_nodes(self, graph: fx.Graph) -> List[fx.Node]: 

72 """Find all wait_tensor nodes in the graph.""" 

73 wait_nodes = [] 

74 for node in graph.nodes: 

75 if node.op == "call_function": 

76 target_str = str(node.target) 

77 if ( 

78 "wait_tensor" in target_str 

79 or "torch.distributed._functional_collectives.wait_tensor" 

80 == target_str 

81 ): 

82 wait_nodes.append(node) 

83 return wait_nodes 

84 

85 def _move_wait_later( 

86 self, 

87 graph_module: fx.GraphModule, 

88 wait_node: fx.Node, 

89 ) -> fx.GraphModule: 

90 """Move wait_tensor later so it overlaps with independent compute. 

91 

92 Placeholder: keeps the wait where FSDPPass placed it (immediate 

93 wait). A future implementation will sink each wait past independent 

94 nodes up to its first dependent user. 

95 """ 

96 _ = wait_node # intentional no-op until the scheduler lands 

97 return graph_module 

98 

99 

100class FsdpOverlapPass(GraphPass): 

101 """Overlap FSDP parameter gathers and gradient reductions with computation.""" 

102 

103 name = "fsdp_overlap" 

104 

105 def run( 

106 self, 

107 graph_module: fx.GraphModule, 

108 pass_config: PassConfig, 

109 **kwargs: Any, 

110 ) -> fx.GraphModule: 

111 """Prefetch FSDP launches and sink their complete wait/view DAGs. 

112 

113 Args: 

114 graph_module: Graph transformed by the parallel passes. 

115 pass_config: Independent FSDP switches and all-gather memory limit. 

116 **kwargs: Pipeline context, unused by this graph-only scheduler. 

117 

118 Returns: 

119 The transformed graph module. 

120 """ 

121 if pass_config.fsdp_allgather_prefetch: 

122 graph_module = overlap_param_gather(graph_module, pass_config) 

123 if pass_config.fsdp_overlap_grad_reduce: 

124 graph_module = overlap_grad_comm(graph_module, pass_config) 

125 return graph_module