Coverage for  / home / jenkins / .local / lib / python3.10 / site-packages / hyper_parallel / data / megatron / __init__.py: 100%

1# Copyright 2026 Huawei Technologies Co., Ltd 

2# 

3# Licensed under the Apache License, Version 2.0 (the "License"); 

4# you may not use this file except in compliance with the License. 

5# You may obtain a copy of the License at 

6# 

7# http://www.apache.org/licenses/LICENSE-2.0 

8# 

9# Unless required by applicable law or agreed to in writing, software 

10# distributed under the License is distributed on an "AS IS" BASIS, 

11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 

12# See the License for the specific language governing permissions and 

13# limitations under the License. 

14# ============================================================================ 

15"""Megatron ``.bin`` / ``.idx`` dataset support. 

16 

17Public API: 

18 

19- :class:`IndexedDataset` + :class:`IndexedDatasetBuilder` — wire-format 

20 compatible reader/writer for Megatron-LM's ``.bin`` / ``.idx``. 

21- :class:`GPTDataset` — fixed-length sampling on top of an IndexedDataset. 

22- :class:`BlendableDataset` — weighted multi-source blend. 

23 

24Importing this package registers ``data.type='megatron'`` with 

25:data:`hyper_parallel.data.DATASET_REGISTRY` via ``builder``. 

26""" 

27from hyper_parallel.data.megatron.blendable_dataset import BlendableDataset 

28from hyper_parallel.data.megatron.gpt_dataset import GPTDataset 

29from hyper_parallel.data.megatron.indexed_dataset import ( 

30 IndexedDataset, 

31 IndexedDatasetBuilder, 

32 get_bin_path, 

33 get_idx_path, 

34 strip_suffix, 

35) 

36 

37from hyper_parallel.data.megatron import builder # noqa: F401 # registers builder 

38 

39 

40__all__ = [ 

41 "BlendableDataset", 

42 "GPTDataset", 

43 "IndexedDataset", 

44 "IndexedDatasetBuilder", 

45 "get_bin_path", 

46 "get_idx_path", 

47 "strip_suffix", 

48]