Coverage for  / home / jenkins / .local / lib / python3.10 / site-packages / hyper_parallel / data / batching / __init__.py: 100%

1# Copyright 2026 Huawei Technologies Co., Ltd 

2# 

3# Licensed under the Apache License, Version 2.0 (the "License"); 

4# you may not use this file except in compliance with the License. 

5# You may obtain a copy of the License at 

6# 

7# http://www.apache.org/licenses/LICENSE-2.0 

8# 

9# Unless required by applicable law or agreed to in writing, software 

10# distributed under the License is distributed on an "AS IS" BASIS, 

11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 

12# See the License for the specific language governing permissions and 

13# limitations under the License. 

14# ============================================================================ 

15"""Batch collation, dynamic packing, and DataLoader construction.""" 

16 

17from hyper_parallel.data.batching.build_collate_fn import ( 

18 build_default_collate_fn, 

19 build_indexed_collate_fn, 

20 build_omni_collate_fn, 

21 build_online_text_collate_fn, 

22) 

23from hyper_parallel.data.batching.build_dataloader import ( 

24 FixedBatchDataLoader, 

25 OmniPackingLoader, 

26 TokenBatchLoader, 

27 build_dataloader, 

28 calculate_num_micro_batches, 

29) 

30from hyper_parallel.data.batching.get_batch import OmniParallelBatch, TextParallelBatch 

31from hyper_parallel.data.batching.packing import ( 

32 FirstFitPackingSelector, 

33 PackingSelector, 

34 SamplePacker, 

35) 

36from hyper_parallel.data.batching.runtime_input import ( 

37 RuntimeInputAdapter, 

38 RuntimeInputContext, 

39) 

40 

41__all__ = [ 

42 "FirstFitPackingSelector", 

43 "FixedBatchDataLoader", 

44 "OmniParallelBatch", 

45 "OmniPackingLoader", 

46 "PackingSelector", 

47 "TextParallelBatch", 

48 "RuntimeInputAdapter", 

49 "RuntimeInputContext", 

50 "SamplePacker", 

51 "TokenBatchLoader", 

52 "build_default_collate_fn", 

53 "build_dataloader", 

54 "build_indexed_collate_fn", 

55 "build_omni_collate_fn", 

56 "build_online_text_collate_fn", 

57 "calculate_num_micro_batches", 

58]