Coverage for  / home / jenkins / .local / lib / python3.10 / site-packages / hyper_parallel / data / indexed / __init__.py: 100%

1# Copyright 2026 Huawei Technologies Co., Ltd 

2# 

3# Licensed under the Apache License, Version 2.0 (the "License"); 

4# you may not use this file except in compliance with the License. 

5# You may obtain a copy of the License at 

6# 

7# http://www.apache.org/licenses/LICENSE-2.0 

8# 

9# Unless required by applicable law or agreed to in writing, software 

10# distributed under the License is distributed on an "AS IS" BASIS, 

11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 

12# See the License for the specific language governing permissions and 

13# limitations under the License. 

14# ============================================================================ 

15"""Indexed token and prepared-record datasets, with native sampling helpers. 

16 

17Prepared Arrow datasets share the Indexed split and sampling pipeline. 

18Offline indexed I/O tools live in ``hyper_parallel.data.tools.io``. 

19""" 

20 

21from pathlib import Path 

22 

23from hyper_parallel.data.indexed.indexed_pretrain_dataset import SFTFromMRDataset 

24 

25__all__ = ["SFTFromMRDataset"] 

26 

27 

28def _extend_native_path(package_file: str, package_path: list[str]) -> None: 

29 """Make the source-build helper visible without copying binaries into source.""" 

30 repository = Path(package_file).resolve().parents[3] 

31 native = repository / "build/native/payload/hyper_parallel/data/indexed" 

32 if (repository / "setup.py").is_file() and native.is_dir() and str(native) not in package_path: 

33 package_path.append(str(native)) 

34 

35 

36_extend_native_path(__file__, __path__)