Coverage for  / home / jenkins / .local / lib / python3.10 / site-packages / hyper_parallel / __init__.py: 100%

1# Copyright 2026 Huawei Technologies Co., Ltd 

2# 

3# Licensed under the Apache License, Version 2.0 (the "License"); 

4# you may not use this file except in compliance with the License. 

5# You may obtain a copy of the License at 

6# 

7# http://www.apache.org/licenses/LICENSE-2.0 

8# 

9# Unless required by applicable law or agreed to in writing, software 

10# distributed under the License is distributed on an "AS IS" BASIS, 

11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 

12# See the License for the specific language governing permissions and 

13# limitations under the License. 

14# ============================================================================ 

15"""hyper parallel interface""" 

16 

17__all__ = ["get_platform", "DFunction", "fully_shard", "hsdp_sync_stream", "HSDPModule", "DTensor", 

18 "Layout", "DeviceMesh", "init_device_mesh", "get_current_mesh", "distribute_module", 

19 "distribute_tensor", "ones", "zeros", "empty", "full", "rand", "randn", 

20 "Shard", "Replicate", "Partial", "Placement", 

21 "init_parameters", "init_empty_weights", "init_on_device", 

22 "shard_module", "custom_shard", "parallelize_value_and_grad", "SkipDTensorDispatch", 

23 "MetaStep", "MetaStepType", "BatchDimSpec", "PipelineStage", "ScheduleInterleaved1F1B", 

24 "ScheduleMPipeTranspose", 

25 "init_process_group", "destroy_process_group", "get_process_group_ranks", "get_backend", "split_group", 

26 "get_group_local_rank", "mark_created_groups", 

27 "ContextParallel", "AsyncContextParallel", 

28 "AsyncDSAIndexerContextParallel", "AsyncDSAIndexerLossContextParallel", 

29 "AsyncDSASparseAttentionContextParallel", 

30 "DSAIndexerContextParallel", "DSAIndexerLossContextParallel", "DSASparseAttentionContextParallel", 

31 "ColwiseParallel", "NoParallel", "RowwiseParallel", "SequenceParallel", 

32 "PrepareModuleInput", "PrepareModuleInputOutput", "PrepareModuleOutput", 

33 "ParallelStyle", "parallelize_module", "manual_seed"] 

34 

35from hyper_parallel.platform import get_platform 

36from hyper_parallel.core.shard.dfunction import DFunction 

37from hyper_parallel.core.dtensor.layout import Layout 

38from hyper_parallel.core.dtensor.device_mesh import DeviceMesh, _mesh_resources, init_device_mesh 

39from hyper_parallel.core.dtensor.dtensor import ( 

40 DTensor, 

41 SkipDTensorDispatch, 

42 distribute_module, 

43 distribute_tensor, 

44 ones, 

45 zeros, 

46 empty, 

47 full, 

48 rand, 

49 randn, 

50) 

51from hyper_parallel.core.dtensor.placement_types import Shard, Replicate, Partial, Placement 

52from hyper_parallel.core.dtensor.parameter_init import init_parameters 

53from hyper_parallel.core.dtensor.init_weights import init_empty_weights, init_on_device 

54from hyper_parallel.core.shard.api import shard_module 

55from hyper_parallel.core.shard.api import parallelize_value_and_grad 

56from hyper_parallel.core.shard.custom_shard import custom_shard 

57from hyper_parallel.core.pipeline_parallel import (PipelineStage, ScheduleInterleaved1F1B, ScheduleMPipeTranspose, 

58 MetaStep, MetaStepType, BatchDimSpec) 

59from hyper_parallel.collectives.cc import (init_process_group, destroy_process_group, get_process_group_ranks, 

60 get_backend, split_group, get_group_local_rank, mark_created_groups) 

61from hyper_parallel.core.context_parallel import ( 

62 AsyncDSAIndexerContextParallel, 

63 AsyncDSAIndexerLossContextParallel, 

64 AsyncDSASparseAttentionContextParallel, 

65 ContextParallel, 

66 AsyncContextParallel, 

67 DSAIndexerContextParallel, 

68 DSAIndexerLossContextParallel, 

69 DSASparseAttentionContextParallel, 

70) 

71from hyper_parallel.core.tensor_parallel import ( 

72 ColwiseParallel, 

73 NoParallel, 

74 ParallelStyle, 

75 PrepareModuleInput, 

76 PrepareModuleInputOutput, 

77 PrepareModuleOutput, 

78 RowwiseParallel, 

79 SequenceParallel, 

80 parallelize_module, 

81) 

82from hyper_parallel.core.dtensor.random import manual_seed 

83from hyper_parallel.core.fully_shard.api import fully_shard, hsdp_sync_stream, HSDPModule 

84 

85get_current_mesh = _mesh_resources.get_current_mesh