Coverage for  / home / jenkins / .local / lib / python3.10 / site-packages / hyper_parallel / compile / passes / overlap / schedule.py: 100%

1# Copyright 2026 Huawei Technologies Co., Ltd 

2# 

3# Licensed under the Apache License, Version 2.0 (the "License"); 

4# you may not use this file except in compliance with the License. 

5# You may obtain a copy of the License at 

6# 

7# http://www.apache.org/licenses/LICENSE-2.0 

8# 

9# Unless required by applicable law or agreed to in writing, software 

10# distributed under the License is distributed on an "AS IS" BASIS, 

11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 

12# See the License for the specific language governing permissions and 

13# limitations under the License. 

14# ============================================================================ 

15""" 

16Auto Overlap Pass - Communication-Compute Overlap Optimization 

17 

18Responsibilities: 

191. Find wait_tensor nodes for communication operations 

202. Analyze dependencies to find overlap opportunities 

213. Move wait_tensor to later position to maximize overlap 

22 

23Optimization Strategies: 

24- Move wait_tensor as late as possible (just before first use) 

25- Allow communication to overlap with independent computation 

26""" 

27 

28__all__ = ["AutoOverlapPass"] 

29 

30from typing import Any, List 

31from torch import fx 

32 

33from ...pass_config import PassConfig 

34from ..base import GraphPass 

35 

36 

37class AutoOverlapPass(GraphPass): 

38 """ 

39 Auto Overlap Pass 

40 

41 Automatically identifies opportunities to overlap communication with computation 

42 by reordering operations and issuing async communication early. 

43 """ 

44 

45 name = "auto_overlap" 

46 

47 def run( 

48 self, 

49 graph_module: fx.GraphModule, 

50 pass_config: PassConfig, 

51 **kwargs: Any, 

52 ) -> fx.GraphModule: 

53 """Optimize communication-compute overlap by moving wait_tensor nodes. 

54 

55 FSDPPass inserts wait_tensor immediately after communication (for correctness). 

56 This pass moves wait_tensor later to allow overlap with independent computation. 

57 """ 

58 if not pass_config.enable_overlap: 

59 return graph_module 

60 

61 graph = graph_module.graph 

62 wait_nodes = self._find_wait_nodes(graph) 

63 for wait_node in wait_nodes: 

64 graph_module = self._move_wait_later(graph_module, wait_node) 

65 

66 graph_module.recompile() 

67 return graph_module 

68 

69 def _find_wait_nodes(self, graph: fx.Graph) -> List[fx.Node]: 

70 """Find all wait_tensor nodes in the graph.""" 

71 wait_nodes = [] 

72 for node in graph.nodes: 

73 if node.op == "call_function": 

74 target_str = str(node.target) 

75 if ( 

76 "wait_tensor" in target_str 

77 or "torch.distributed._functional_collectives.wait_tensor" 

78 == target_str 

79 ): 

80 wait_nodes.append(node) 

81 return wait_nodes 

82 

83 def _move_wait_later( 

84 self, 

85 graph_module: fx.GraphModule, 

86 wait_node: fx.Node, 

87 ) -> fx.GraphModule: 

88 """Move wait_tensor later so it overlaps with independent compute. 

89 

90 Placeholder: keeps the wait where FSDPPass placed it (immediate 

91 wait). A future implementation will sink each wait past independent 

92 nodes up to its first dependent user. 

93 """ 

94 _ = wait_node # intentional no-op until the scheduler lands 

95 return graph_module