Coverage for  / home / jenkins / .local / lib / python3.10 / site-packages / hyper_parallel / __init__.py: 96%

1# Copyright 2026 Huawei Technologies Co., Ltd 

2# 

3# Licensed under the Apache License, Version 2.0 (the "License"); 

4# you may not use this file except in compliance with the License. 

5# You may obtain a copy of the License at 

6# 

7# http://www.apache.org/licenses/LICENSE-2.0 

8# 

9# Unless required by applicable law or agreed to in writing, software 

10# distributed under the License is distributed on an "AS IS" BASIS, 

11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 

12# See the License for the specific language governing permissions and 

13# limitations under the License. 

14# ============================================================================ 

15# pylint: disable=undefined-all-variable 

16"""hyper parallel interface""" 

17 

18__all__ = ["DFunction", "fully_shard", "hsdp_sync_stream", "HSDPModule", "DTensor", 

19 "Layout", "DeviceMesh", "init_device_mesh", "get_current_mesh", "distribute_module", 

20 "distribute_tensor", "ones", "zeros", "empty", "full", "rand", "randn", 

21 "Shard", "RaggedShard", "Replicate", "Partial", "Placement", 

22 "init_parameters", "init_empty_weights", "init_on_device", 

23 "shard_module", "custom_shard", "SkipDTensorDispatch", 

24 "MetaStep", "MetaStepType", "BatchDimSpec", "PipelineStage", "ScheduleInterleaved1F1B", 

25 "ScheduleMPipeTranspose", 

26 "init_process_group", "destroy_process_group", "get_process_group_ranks", "get_backend", "split_group", 

27 "get_group_local_rank", "mark_created_groups", 

28 "ContextParallel", "AsyncContextParallel", 

29 "AsyncDSAIndexerContextParallel", "AsyncDSAIndexerLossContextParallel", 

30 "AsyncDSASparseAttentionContextParallel", 

31 "DSAIndexerContextParallel", "DSAIndexerLossContextParallel", "DSASparseAttentionContextParallel", 

32 "ColwiseParallel", "MC2ColwiseParallel", "MC2RowwiseParallel", "MC2Linear", 

33 "NoParallel", "RowwiseParallel", "SequenceParallel", 

34 "PrepareModuleInput", "PrepareModuleInputOutput", "PrepareModuleOutput", 

35 "ParallelStyle", "parallelize_module", "manual_seed"] 

36 

37from importlib import import_module as _import_module # pylint: disable=invalid-name 

38 

39from hyper_parallel.core.shard.dfunction import DFunction 

40from hyper_parallel.core.dtensor.layout import Layout 

41from hyper_parallel.core.dtensor.device_mesh import DeviceMesh, _mesh_resources, init_device_mesh 

42from hyper_parallel.core.dtensor.dtensor import ( 

43 DTensor, 

44 SkipDTensorDispatch, 

45 distribute_module, 

46 distribute_tensor, 

47 ones, 

48 zeros, 

49 empty, 

50 full, 

51 rand, 

52 randn, 

53) 

54from hyper_parallel.core.dtensor.placement_types import ( 

55 Placement, 

56 RaggedShard, 

57 Replicate, 

58 Partial, 

59 Shard, 

60) 

61from hyper_parallel.core.dtensor.parameter_init import init_parameters 

62from hyper_parallel.core.dtensor.init_weights import init_empty_weights, init_on_device 

63from hyper_parallel.core.shard.api import shard_module 

64from hyper_parallel.core.shard.custom_shard import custom_shard 

65from hyper_parallel.collectives.cc import (init_process_group, destroy_process_group, get_process_group_ranks, 

66 get_backend, split_group, get_group_local_rank, mark_created_groups) 

67from hyper_parallel.core.context_parallel import ( 

68 AsyncDSAIndexerContextParallel, 

69 AsyncDSAIndexerLossContextParallel, 

70 AsyncDSASparseAttentionContextParallel, 

71 ContextParallel, 

72 AsyncContextParallel, 

73 DSAIndexerContextParallel, 

74 DSAIndexerLossContextParallel, 

75 DSASparseAttentionContextParallel, 

76) 

77from hyper_parallel.core.tensor_parallel import ( 

78 ColwiseParallel, 

79 NoParallel, 

80 ParallelStyle, 

81 PrepareModuleInput, 

82 PrepareModuleInputOutput, 

83 PrepareModuleOutput, 

84 RowwiseParallel, 

85 SequenceParallel, 

86 parallelize_module, 

87) 

88from hyper_parallel.core.dtensor.random import manual_seed 

89from hyper_parallel.core.fully_shard.api import fully_shard, hsdp_sync_stream, HSDPModule 

90 

91get_current_mesh = _mesh_resources.get_current_mesh 

92 

93# Load Torch-only pipeline and MC2 components only when their public APIs 

94# are requested, preserving imports for other components and backends. 

95_LAZY_EXPORTS = { 

96 "PipelineStage": "hyper_parallel.core.pipeline_parallel", 

97 "ScheduleInterleaved1F1B": "hyper_parallel.core.pipeline_parallel", 

98 "ScheduleMPipeTranspose": "hyper_parallel.core.pipeline_parallel", 

99 "MetaStep": "hyper_parallel.core.pipeline_parallel", 

100 "MetaStepType": "hyper_parallel.core.pipeline_parallel", 

101 "BatchDimSpec": "hyper_parallel.core.pipeline_parallel", 

102 "MC2Linear": "hyper_parallel.core.tensor_parallel", 

103 "MC2ColwiseParallel": "hyper_parallel.core.tensor_parallel", 

104 "MC2RowwiseParallel": "hyper_parallel.core.tensor_parallel", 

105} 

106 

107 

108def __getattr__(name): # pylint: disable=invalid-name 

109 """Lazily import symbols of Torch-only components.""" 

110 if name not in _LAZY_EXPORTS: 

111 raise AttributeError(f"module {__name__!r} has no attribute {name!r}") 

112 

113 module = _import_module(_LAZY_EXPORTS[name]) 

114 value = getattr(module, name) 

115 globals()[name] = value 

116 return value 

117 

118 

119def __dir__(): # pylint: disable=invalid-name 

120 """Include lazy Torch-only exports in ``dir()``.""" 

121 return sorted(set(globals()) | set(_LAZY_EXPORTS))