Coverage for  / home / jenkins / .local / lib / python3.10 / site-packages / hyper_parallel / data / batching / __init__.py: 100%

1# Copyright 2026 Huawei Technologies Co., Ltd 

2# 

3# Licensed under the Apache License, Version 2.0 (the "License"); 

4# you may not use this file except in compliance with the License. 

5# You may obtain a copy of the License at 

6# 

7# http://www.apache.org/licenses/LICENSE-2.0 

8# 

9# Unless required by applicable law or agreed to in writing, software 

10# distributed under the License is distributed on an "AS IS" BASIS, 

11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 

12# See the License for the specific language governing permissions and 

13# limitations under the License. 

14# ============================================================================ 

15"""Batch collation, dynamic packing, and DataLoader construction.""" 

16 

17from hyper_parallel.data.batching.build_collate_fn import ( 

18 build_default_collate_fn, 

19 build_sft_collate_fn, 

20 build_indexed_collate_fn, 

21 build_omni_collate_fn, 

22 build_online_text_collate_fn, 

23) 

24from hyper_parallel.data.batching.build_dataloader import ( 

25 DynamicBatchDataLoader, 

26 FixedBatchDataLoader, 

27 OmniPackingLoader, 

28 TokenBatchLoader, 

29 build_dataloader, 

30 calculate_num_micro_batches, 

31) 

32from hyper_parallel.data.batching.get_batch import OmniParallelBatch, TextParallelBatch 

33from hyper_parallel.data.batching.packing import ( 

34 FirstFitPackingSelector, 

35 PackingSelector, 

36 SamplePacker, 

37) 

38from hyper_parallel.data.batching.runtime_input import ( 

39 RuntimeInputAdapter, 

40 RuntimeInputContext, 

41) 

42 

43__all__ = [ 

44 "FirstFitPackingSelector", 

45 "DynamicBatchDataLoader", 

46 "FixedBatchDataLoader", 

47 "OmniParallelBatch", 

48 "OmniPackingLoader", 

49 "PackingSelector", 

50 "TextParallelBatch", 

51 "RuntimeInputAdapter", 

52 "RuntimeInputContext", 

53 "SamplePacker", 

54 "TokenBatchLoader", 

55 "build_default_collate_fn", 

56 "build_sft_collate_fn", 

57 "build_dataloader", 

58 "build_indexed_collate_fn", 

59 "build_omni_collate_fn", 

60 "build_online_text_collate_fn", 

61 "calculate_num_micro_batches", 

62]