Coverage for  / home / jenkins / .local / lib / python3.10 / site-packages / hyper_parallel / distributed_data / __init__.py: 100%

1# Copyright 2026 Huawei Technologies Co., Ltd 

2# 

3# Licensed under the Apache License, Version 2.0 (the "License"); 

4# you may not use this file except in compliance with the License. 

5# You may obtain a copy of the License at 

6# 

7# http://www.apache.org/licenses/LICENSE-2.0 

8# 

9# Unless required by applicable law or agreed to in writing, software 

10# distributed under the License is distributed on an "AS IS" BASIS, 

11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 

12# See the License for the specific language governing permissions and 

13# limitations under the License. 

14# ============================================================================ 

15"""Dataset Reader, Planner, and Data Constructor distributed data pipeline.""" 

16 

17from hyper_parallel.distributed_data.api import DistributedDatasetConfig, build_distributed_dataloader 

18from hyper_parallel.distributed_data.balance_logging import format_balance_stats, log_balance_stats 

19from hyper_parallel.distributed_data.balancing_algorithm import ( 

20 BalancingAlgorithm, 

21 FeasibleLPTBalancingAlgorithm, 

22 LPTBalancingAlgorithm, 

23) 

24from hyper_parallel.distributed_data.cost_model import ( 

25 BackboneFlopsConfig, CausalAttentionCostModel, CostModel, DefaultCostModel, 

26) 

27from hyper_parallel.distributed_data.data_constructor import default_collate_fn, default_pack_fn 

28from hyper_parallel.distributed_data.distributed_dataloader import DistributedDataLoader 

29from hyper_parallel.distributed_data.external_step import ExternalStepSource 

30from hyper_parallel.distributed_data.model_batch import TensorFieldSpec 

31from hyper_parallel.distributed_data.indexed_text import ( 

32 collate_indexed_text_sequences, 

33 pack_indexed_text_samples, 

34 split_packed_text_sample, 

35) 

36from hyper_parallel.distributed_data.packed_balancing import LocalBalancingDataLoader, build_local_balancing_dataloader 

37from hyper_parallel.distributed_data.schema import ( 

38 DistributedPackingPlan, 

39 PackingBinPlan, 

40 PackingConstraints, 

41 SampleKey, 

42 SampleMetadata, 

43 WorkloadCost, 

44) 

45 

46__all__ = [ 

47 "BackboneFlopsConfig", 

48 "BalancingAlgorithm", 

49 "CausalAttentionCostModel", 

50 "LPTBalancingAlgorithm", 

51 "CostModel", 

52 "DefaultCostModel", 

53 "DistributedDataLoader", 

54 "DistributedDatasetConfig", 

55 "DistributedPackingPlan", 

56 "ExternalStepSource", 

57 "FeasibleLPTBalancingAlgorithm", 

58 "LocalBalancingDataLoader", 

59 "PackingBinPlan", 

60 "PackingConstraints", 

61 "SampleKey", 

62 "SampleMetadata", 

63 "WorkloadCost", 

64 "TensorFieldSpec", 

65 "build_distributed_dataloader", 

66 "build_local_balancing_dataloader", 

67 "collate_indexed_text_sequences", 

68 "default_collate_fn", 

69 "default_pack_fn", 

70 "format_balance_stats", 

71 "log_balance_stats", 

72 "pack_indexed_text_samples", 

73 "split_packed_text_sample", 

74]