Coverage for / home / jenkins / .local / lib / python3.10 / site-packages / hyper_parallel / trainer / config / training.py: 100%
66 statements
« prev ^ index » next coverage.py v7.13.1, created at 2026-10-25 05:45 +0800
1# Copyright 2026 Huawei Technologies Co., Ltd
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14# ============================================================================
15"""Training-loop, debug, profiler, and remote-logging configuration sections."""
17from dataclasses import dataclass, field
18from typing import Literal, Optional
20from hyper_parallel.components.quantization.config import LowPrecisionConfig
23@dataclass
24class TrainingConfig:
25 """Training-loop parameters exposed by the initial YAML schema."""
27 train_iters: Optional[int] = None
28 train_samples: Optional[int] = None
29 lr_scheduler_iters: Optional[int] = None
30 eval_iters: int = 0
32 global_batch_size: int = 8
33 micro_batch_size: int = 1
35 backend: Literal["nccl", "hccl", "gloo"] = "nccl"
36 max_grad_norm: float = 1.0
37 init_device: Literal["meta", "cpu", "cuda", "npu"] = "meta"
38 loss_aggregation: Literal["token_weighted", "rank_average"] = "token_weighted"
39 seed: Optional[int] = None
40 enable_full_determinism: bool = False
41 gc_steps: int = 0
42 empty_cache_steps: int = 0
43 empty_cache_before_backward: bool = False
44 eval_steps: int = 0
45 eval_epochs: int = 0
46 logging_steps: int = 1
47 low_precision: LowPrecisionConfig = field(default_factory=LowPrecisionConfig)
50@dataclass
51class DebugConfig:
52 """Debug parameters exposed by the initial YAML schema."""
54 check_dataset: Optional[Literal["debug", "info", "warn"]] = None
55 check_nan_inf: bool = False
56 check_fsdp_runtime: bool = False
59@dataclass
60class ModelIntegrationConfig:
61 """Enable model-integration checks at one well-defined lifecycle scope.
63 ``build`` validates the final module tree, sharding ownership, data
64 contract, optimizer layout, and checkpoint coverage. ``runtime`` includes
65 those checks and records one-step input, gradient, shared-state, and layout
66 evidence. Output location and probe selection are framework policy rather
67 than user-tunable training semantics.
68 """
70 mode: Literal["off", "build", "runtime"] = "off"
72 def __post_init__(self) -> None:
73 """Reject ambiguous validation scopes at the configuration boundary."""
74 if self.mode not in ("off", "build", "runtime"):
75 raise ValueError(
76 "model_integration.mode must be one of: off, build, runtime"
77 )
80@dataclass
81class MemoryConfig:
82 """Configure allocator memory-history snapshots during training.
84 Args:
85 enable: Whether allocator history recording is enabled.
86 start_step: First session-relative step included in the snapshot; zero includes model construction.
87 end_step: Last session-relative step included in the snapshot.
88 save_path: Directory used for snapshot files.
89 dump_ranks: Global ranks that write snapshot files.
90 stacks: Stack information recorded by the allocator.
91 mem_info: Whether peak allocator statistics are logged before each step.
92 """
94 enable: bool = False
95 start_step: int = 1
96 end_step: int = 2
97 save_path: str = "./memory_snapshot"
98 dump_ranks: list[int] = field(default_factory=lambda: [0])
99 stacks: Literal["python", "all"] = "all"
100 mem_info: bool = False
103@dataclass
104class WandbConfig:
105 """WandB remote-logging parameters (03 §4.2.5: read by build_callback_manager)."""
107 enabled: bool = False
108 project: str = ""
109 entity: Optional[str] = None
112@dataclass
113class ProfilingAnalysisConfig:
114 """Configure Ascend NPU trace analysis after collection.
116 Args:
117 enable: Whether collected data is automatically analyzed.
118 async_mode: Whether automatic analysis runs asynchronously; ignored when disabled.
119 """
121 enable: bool = True
122 async_mode: bool = False
125@dataclass
126class ProfilingConfig:
127 """Configure bounded PyTorch profiler collection during training.
129 HyperParallel groups these options under ``TrainerConfig.profiler``, so
130 their names describe only the option itself rather than repeating a
131 ``profile_`` prefix.
133 Args:
134 enable: Whether profiler collection is enabled.
135 start_step: First training step included in the profile.
136 stop_step: Last training step included in the profile.
137 warmup_steps: Number of warmup steps immediately before ``start_step``; must be smaller than it.
138 memory: Whether tensor memory events are collected.
139 rank_ids: Explicit ranks that collect a profile. ``None`` or an empty list selects all ranks.
140 output_path: Output root. Each rank writes under ``profile/rank_<rank>``.
141 level: Ascend collection level, from 0 through 2. ``None`` selects level 0.
142 with_stack: Whether Python stack traces are collected.
143 data_simplification: Whether torch-npu removes selected auxiliary CANN data after export.
144 analysis: Ascend NPU automatic trace analysis options; ignored on CPU/CUDA.
145 """
147 enable: bool = False
148 start_step: int = 1
149 stop_step: int = 10
150 warmup_steps: int = 0
151 memory: bool = True
152 rank_ids: Optional[list[int]] = None
153 output_path: Optional[str] = None
154 level: Optional[int] = 1
155 with_stack: bool = False
156 data_simplification: bool = False
157 analysis: ProfilingAnalysisConfig = field(default_factory=ProfilingAnalysisConfig)