Coverage for  / home / jenkins / .local / lib / python3.10 / site-packages / hyper_parallel / data / __init__.py: 100%

1# Copyright 2026 Huawei Technologies Co., Ltd 

2# 

3# Licensed under the Apache License, Version 2.0 (the "License"); 

4# you may not use this file except in compliance with the License. 

5# You may obtain a copy of the License at 

6# 

7# http://www.apache.org/licenses/LICENSE-2.0 

8# 

9# Unless required by applicable law or agreed to in writing, software 

10# distributed under the License is distributed on an "AS IS" BASIS, 

11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 

12# See the License for the specific language governing permissions and 

13# limitations under the License. 

14# ============================================================================ 

15"""HyperParallel dataset module for the unified ``data.type`` registry. 

16 

17Public API: 

18 

19- :func:`build_dataset`: registry-driven dispatch on ``data.type``. 

20- :data:`DATASET_REGISTRY`: registry instance; register custom formats 

21 with ``@DATASET_REGISTRY.register("<name>")``. 

22- :class:`DummyDataset`, :class:`DummyVLDataset`, :class:`TokenizedDataset`, 

23 :class:`PresetPtDataset`: first-party dataset classes. 

24- :class:`IndexedDataset`, :class:`IndexedDatasetBuilder`, 

25 :class:`GPTDataset`, :class:`BlendableDataset`: Megatron support. 

26 

27Importing this package registers every first-party builder 

28(dummy, vl_dummy, hf_datasets, json_file, preset_pt, megatron). 

29External plugins can register additional builders by importing their 

30modules before the trainer is constructed. 

31""" 

32from hyper_parallel.data.dummy import DummyDataset 

33from hyper_parallel.data.hf import TokenizedDataset 

34from hyper_parallel.data.megatron import ( 

35 BlendableDataset, 

36 GPTDataset, 

37 IndexedDataset, 

38 IndexedDatasetBuilder, 

39) 

40from hyper_parallel.data.preset_pt import PresetPtDataset 

41from hyper_parallel.data.registry import DATASET_REGISTRY, build_dataset 

42from hyper_parallel.data.vl_dummy import DummyVLDataset 

43 

44# Each module registers a builder when imported. 

45from hyper_parallel.data import ( # noqa: F401 # registers builders 

46 dummy, 

47 hf, 

48 preset_pt, 

49 vl_dummy, 

50) 

51 

52 

53__all__ = [ 

54 "BlendableDataset", 

55 "DATASET_REGISTRY", 

56 "DummyDataset", 

57 "DummyVLDataset", 

58 "GPTDataset", 

59 "IndexedDataset", 

60 "IndexedDatasetBuilder", 

61 "PresetPtDataset", 

62 "TokenizedDataset", 

63 "build_dataset", 

64]