Coverage for  / home / jenkins / .local / lib / python3.10 / site-packages / hyper_parallel / data / text / __init__.py: 92%

1# Copyright 2026 Huawei Technologies Co., Ltd 

2# 

3# Licensed under the Apache License, Version 2.0 (the "License"); 

4# you may not use this file except in compliance with the License. 

5# You may obtain a copy of the License at 

6# 

7# http://www.apache.org/licenses/LICENSE-2.0 

8# 

9# Unless required by applicable law or agreed to in writing, software 

10# distributed under the License is distributed on an "AS IS" BASIS, 

11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 

12# See the License for the specific language governing permissions and 

13# limitations under the License. 

14# ============================================================================ 

15"""Text (LLM) dataset builders, transforms, and chat templates. 

16 

17Moved from ``components/datasets/llm`` in stage 6 (05 §11.3). The public 

18names are re-exported lazily: ``build_dataset`` pulls in the natively built 

19indexed-dataset helpers, which are unavailable on a plain CPU checkout, so 

20importing this package must stay cheap. 

21""" 

22 

23import importlib 

24from typing import Any 

25 

26from hyper_parallel.data.constants import TEXT_LAZY_EXPORTS 

27 

28__all__ = sorted(TEXT_LAZY_EXPORTS) 

29 

30 

31def __getattr__(name: str) -> Any: 

32 """Resolve a public name through its owning submodule on first use.""" 

33 module_path = TEXT_LAZY_EXPORTS.get(name) 

34 if module_path is None: 

35 raise AttributeError(f"module {__name__!r} has no attribute {name!r}") 

36 value = getattr(importlib.import_module(module_path), name) 

37 globals()[name] = value 

38 return value 

39 

40 

41def __dir__() -> list[str]: 

42 """List the lazily exported public names.""" 

43 return sorted(__all__)