Coverage for /home/jenkins/.local/lib/python3.10/site-packages/hyper

3# Licensed under the Apache License, Version 2.0 (the "License");

4# you may not use this file except in compliance with the License.

5# You may obtain a copy of the License at

7# http://www.apache.org/licenses/LICENSE-2.0

9# Unless required by applicable law or agreed to in writing, software

10# distributed under the License is distributed on an "AS IS" BASIS,

11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

12# See the License for the specific language governing permissions and

13# limitations under the License.

14# ============================================================================

15"""Train-time integrity checks over a codegen artifact bundle.

17The manager's ``preflight_integrity_check`` is the gate; this module holds

18the per-zone checks it composes — signature, output hashes, importability,

19meta schema, skipped-override warnings, and parameter-plan coverage. Each

20verify_* function is a standalone unit so the CLI can call them

21individually.

22"""

24from __future__ import annotations

26import logging

27import os

28from typing import Any

30from hyper_parallel.codegen.hash import sha256_file

32logger = logging.getLogger(__name__)

35def verify_signature(meta: Any, current_signature: str) -> None:

36 """Fail fast when the stored signature no longer matches the current one.

38 A mismatch means the YAML / source / env changed since generation — the

39 bundle is stale and must be regenerated before training.

40 """

41 if meta.signature != current_signature:

42 raise RuntimeError(

43 f"codegen preflight: signature mismatch for {meta.yaml_path} "

44 f"(stored {meta.signature}, current {current_signature}); regenerate"

45 )

48def verify_output_hashes(meta: Any, layout: Any) -> None:

49 """Fail fast when any recorded output file's sha256 drifted.

51 Checks the same file set that ``record_output_hashes`` records — the

52 generated modeling file, its diff, ``__init__.py``, and any copied remote

53 source siblings — so a post-generation edit surfaces here instead of being

54 silently imported.

55 The meta file itself is not in that set: a file cannot carry its own

56 sha256 (writing the value changes the bytes it hashes), so meta

57 integrity is covered by the signature and schema checks instead.

58 """

59 core_paths = {

60 "modeling": layout.modeling_path,

61 "diff": layout.diff_path,

62 "init": layout.init_path,

63 }

64 for key in core_paths:

65 if key not in meta.outputs:

66 raise RuntimeError(f"codegen preflight: missing {key} output hash in meta")

68 for sibling in getattr(meta, "remote_siblings", None) or []:

69 if sibling not in meta.outputs:

70 raise RuntimeError(

71 f"codegen preflight: missing output hash for remote sibling {sibling!r}"

72 )

74 for key, recorded in meta.outputs.items():

75 path = core_paths.get(key)

76 if path is None:

77 if (

78 not key

79 or os.path.basename(key) != key

80 or "/" in key

81 or "\\" in key

82 or key in {".", ".."}

83 ):

84 raise RuntimeError(

85 f"codegen preflight: invalid output filename {key!r}"

86 )

87 path = os.path.join(layout.artifact_dir, key)

88 if not os.path.isfile(path):

89 raise RuntimeError(

90 f"codegen preflight: {key} file missing at {path} "

91 f"(recorded sha256 {recorded.get('sha256')})"

92 )

93 actual = sha256_file(path)

94 if recorded.get("sha256") != actual:

95 raise RuntimeError(

96 f"codegen preflight: {key} hash drift at {path} "

97 f"(recorded {recorded.get('sha256')}, actual {actual})"

98 )

100

101def verify_generated_import(layout: Any) -> None:

102 """Fail fast when the generated modeling file cannot be imported.

103

104 Importing the generated module is the strongest single check that the

105 bundle is usable: it exercises the module's own imports (runtime, torch,

106 transformers) and top-level class definitions. The module is cached in

107 ``sys.modules`` by the loader, so preflight and the later build reuse the

108 same import. A minimal placeholder also imports because preflight checks

109 bundle structure, while the emit layer owns lowering completeness.

110 """

111 from hyper_parallel.codegen.loader import import_generated_module

112

113 import_generated_module(layout.artifact_dir)

114

115

116def verify_meta_required_fields(meta: Any) -> None:

117 """Fail fast when the meta record is missing required schema fields.

118

119 Delegates to the meta module's schema validation; a stale or hand-edited

120 meta file that lost a required field must not pass preflight.

121 """

122 from hyper_parallel.codegen.meta import validate_meta_schema

123

124 validate_meta_schema(meta_to_dict(meta))

125

126

127def warn_skipped_overrides(meta: Any) -> None:

128 """Warn (not error) when generation skipped any plan_overrides entries.

129

130 An entry skipped during generation (or a ``when``-gated entry that never

131 applied) must be surfaced

132 so the user knows the artifact does not contain what the YAML declared.

133 """

134 skipped = getattr(meta, "skipped_overrides", None) or []

135 if not skipped:

136 return

137 lines = [f"codegen: {len(skipped)} plan_overrides entries were skipped:"]

138 for item in skipped:

139 match = item.get("match") if isinstance(item, dict) else None

140 reason = item.get("reason") if isinstance(item, dict) else item

141 lines.append(f" - match={match!r}: {reason}")

142 logger.warning("\n".join(lines))

143

144

145def verify_boundary_forms(meta: Any, layout: Any) -> None:

146 """Fail fast when persisted forward forms disagree with the frozen plan.

147

148 Re-derives every boundary class's emitted form from the persisted meta —

149 through ``iter_emitted_forms``, the emitter's own decision path — and

150 checks the generated source carries exactly that structure:

151

152 - ``identity``: the original forward is untouched (no extracted

153 ``_forward_impl``, no form marker);

154 - ``tp_collective``: the class carries a form marker (``TP_FORM_MARKER``

155 from the static template, or ``TOGGLE_FORM_MARKER`` from the opt-in

156 switch template) and the extracted ``_forward_impl``, and the rewritten

157 forward references only the static template's own attributes

158 (``self._hp_tp`` / ``self._forward_impl``) — plus the D-22 exit call

159 ``self._hp_deferred_bias`` when the class's frozen entry declares

160 deferred bias params;

161 - ``generic`` / ``region``: the extracted ``_forward_impl`` without the

162 marker (the compiled-plan / local-compute engines own the forward);

163 - a fully inlined component body (no ``_forward_impl``, no marker): its

164 forward must expose the instance-bound channel (``self._hp_tp`` /

165 ``self._hp_cp_mesh`` / ``self._hp_ep_group`` /

166 ``self.<axis>_enable``) and must not read the removed module-level

167 parallel-state accessor. A class whose forward an inline *strategy*

168 body replaced (the MoE EP shell) is stripped from ``boundary_classes``,

169 so it is not visited here; this shape covers an artifact that still

170 marks its inline orchestration explicitly.

171

172 A drift means the modeling file was edited after generation (or the

173 emitter and this check disagree); the install-time validation would

174 silently fall back, masking a broken artifact, so fail fast instead.

175 """

176 import re

177

178 from hyper_parallel.codegen.astkit.index import build_source_index

179 from hyper_parallel.codegen.emit.parallel import (

180 TOGGLE_FORM_MARKER,

181 TP_FORM_ATTRIBUTE,

182 TP_FORM_MARKER,

183 iter_emitted_forms,

184 )

185 from hyper_parallel.codegen.plan.boundary_forms import (

186 FORM_IDENTITY,

187 FORM_TP_COLLECTIVE,

188 )

189

190 with open(layout.modeling_path, "r", encoding="utf-8") as handle:

191 source_text = handle.read()

192 index = build_source_index(source_text)

193 boundary_classes = getattr(meta, "boundary_classes", None) or {}

194 source = getattr(meta, "source", None)

195 module_name = (

196 source.get("module_name")

197 if isinstance(source, dict)

198 else getattr(source, "module_name", "")

199 )

200

201 for class_name, _, emitted, func, _, deferred_bias in iter_emitted_forms(

202 source_text,

203 meta,

204 boundary_classes=boundary_classes,

205 module_name=module_name,

206 ):

207 info = index.find_class(class_name)

208 has_impl = info is not None and "_forward_impl" in info.methods

209 class_body = source_text[info.class_offset:info.end_offset] if info else ""

210 has_marker = f'{TP_FORM_ATTRIBUTE} = "{TP_FORM_MARKER}"' in class_body

211 # The opt-in toggle template marks its classes with TOGGLE_FORM_MARKER:

212 # a boundary the classifier calls tp_collective carries either marker

213 # depending on which generation mode emitted it.

214 has_toggle_marker = (

215 f'{TP_FORM_ATTRIBUTE} = "{TOGGLE_FORM_MARKER}"' in class_body

216 )

217 has_inline_marker = "_hp_inline_boundary = True" in class_body

218 body = source_text[func.body_start:func.body_end]

219 # Every generated communication path — inlined or lowered — reads the

220 # instance-attribute channel (``self._hp_tp`` / ``self._hp_cp_mesh``

221 # / ``self._hp_ep_group`` / ``self.<axis>_enable``) the runtime binds

222 # at install time. The retired module-level accessor must never come

223 # back: a body still reading it was emitted by the accessor era, and the

224 # class would raise NameError at train time.

225 stray_accessor = re.search(r"get_parallel_state|get_inline_parallel_state", body)

226 if stray_accessor is not None:

227 raise RuntimeError(

228 f"codegen preflight: class {class_name} still reads the removed "

229 f"module-level accessor {stray_accessor.group(0)!r}; the parallel "

230 "channel is bound on the instance at install time — regenerate "

231 "the artifact"

232 )

233 has_inline_channel = any(

234 token in body

235 for token in (

236 "self._hp_tp",

237 "self._hp_cp_mesh",

238 "self._hp_ep_group",

239 "self.tp_enable",

240 "self.cp_enable",

241 "self.ep_enable",

242 "ep_all_to_all",

243 )

244 )

245 if has_inline_marker or (

246 has_inline_channel and not (has_impl or has_marker or has_toggle_marker)

247 ):

248 # A fully inlined component body (the inline-marker shape): it

249 # keeps no extracted ``_forward_impl`` and no form marker, and

250 # must expose the generated communication path through the channel.

251 if not has_inline_channel:

252 raise RuntimeError(

253 f"codegen preflight: inline boundary class {class_name} "

254 "does not expose a generated communication path; "

255 "regenerate the artifact"

256 )

257 if has_marker:

258 raise RuntimeError(

259 f"codegen preflight: clean inline boundary class "

260 f"{class_name} also carries the {TP_FORM_MARKER} marker; "

261 "regenerate the artifact"

262 )

263 continue

264 if emitted == FORM_IDENTITY:

265 if has_impl or has_marker:

266 raise RuntimeError(

267 f"codegen preflight: class {class_name} was pruned as "

268 f"identity but the source carries a rewritten forward "

269 f"(marker={has_marker}, _forward_impl={has_impl}); "

270 "regenerate the artifact"

271 )

272 continue

273 if not has_impl:

274 raise RuntimeError(

275 f"codegen preflight: class {class_name} should carry an "

276 f"extracted _forward_impl for form {emitted!r} but does not; "

277 "regenerate the artifact"

278 )

279 if emitted == FORM_TP_COLLECTIVE:

280 if not (has_marker or has_toggle_marker):

281 raise RuntimeError(

282 f"codegen preflight: class {class_name} should carry the "

283 f"{TP_FORM_MARKER} marker but does not; regenerate the "

284 "artifact"

285 )

286 body = source_text[func.body_start:func.body_end]

287 # The static template's own channel is ``self._hp_tp``; a

288 # class whose frozen entry declares D-22 deferred bias params

289 # additionally calls ``self._hp_deferred_bias`` at the exit

290 # (bound at install time). Anything else is drift.

291 allowed = "tp|deferred_bias" if deferred_bias else "tp"

292 stray = re.search(rf"self\._hp_(?!({allowed})\b)", body)

293 if stray is not None:

294 raise RuntimeError(

295 f"codegen preflight: static tp_collective forward on "

296 f"class {class_name} references {stray.group(0)}* "

297 "attributes outside the static template; regenerate the "

298 "artifact"

299 )

300 elif has_marker:

301 raise RuntimeError(

302 f"codegen preflight: class {class_name} carries the "

303 f"{TP_FORM_MARKER} marker but its emitted form is "

304 f"{emitted!r}; regenerate the artifact"

305 )

306

307

308def verify_param_plan(meta: Any, model: Any) -> None:

309 """Verify the frozen param plan covers the model's trainable parameters.

310

311 Two directions, both against ``model`` (F4b semantics at preflight time):

312

313 1. Every key recorded in ``meta.param_plan`` (boundary + relative param

314 name, e.g. ``model.layers.0.mlp.experts.gate_up_proj``) must resolve

315 to a parameter on ``model``.

316 2. Every trainable parameter of ``model`` must be covered by a plan key

317 — either exactly, or as a leaf under a grouped key (the planner stacks

318 expert params under ``experts.gate_up_proj``, which owns both

319 ``.weight`` and ``.bias``). A parameter is NOT covered by a bare

320 ancestor *boundary* (``model.norm`` alone never covers

321 ``model.norm.bias``); only an explicit param entry counts. This is

322 deliberately per-boundary rather than ``frozen_sharded_params``-only:

323 replicated parameters are still planned (their placement is recorded

324 in ``param_plan``) and must be listed there too.

325

326 An *empty* plan fails because generation always freezes a derived plan.

327 An all-``Replicate`` topology still records every parameter, so

328 ``param_plan == {}`` means derivation failed or was

329 skipped — the artifact is incomplete and must not pass. A model-side

330 change that the signature did not catch (e.g. a different checkpoint

331 variant) fails fast instead of silently running with a stale sharding

332 contract. Degrades to a warning when ``model`` is not given (preflight

333 without a model handle can still check the plan's internal consistency).

334 """

335 param_plan = getattr(meta, "param_plan", None) or {}

336 if not param_plan:

337 raise RuntimeError(

338 "codegen preflight: param plan is empty — generation did not "

339 "freeze a sharding plan (derivation failed or was skipped); "

340 "check generation logs and regenerate the artifact"

341 )

342

343 if model is None:

344 logger.warning(

345 "codegen preflight: no model handle — param-plan coverage "

346 "skipped (only internal consistency checked)"

347 )

348 return

349

350 # Every planned parameter key must resolve to a model parameter.

351 planned: set[str] = set()

352 missing: list[str] = []

353 for boundary, entry in param_plan.items():

354 params = entry.get("params", {}) if isinstance(entry, dict) else {}

355 for name in params:

356 fqn = f"{boundary}.{name}" if name else boundary

357 planned.add(fqn)

358 if _get_parameter(model, fqn) is None:

359 missing.append(fqn)

360 if missing:

361 raise RuntimeError(

362 f"codegen preflight: param plan references parameters missing "

363 f"from the model: {missing}"

364 )

365

366 # Every trainable model parameter must be covered by an explicit plan

367 # entry (exact FQN or a leaf under a grouped key).

368 uncovered = [

369 fqn

370 for fqn in _iter_trainable_params(model)

371 if not any(fqn == key or fqn.startswith(f"{key}.") for key in planned)

372 ]

373 if uncovered:

374 raise RuntimeError(

375 f"codegen preflight: {len(uncovered)} trainable parameters are "

376 f"not covered by the frozen plan: {uncovered[:20]}"

377 )

378

379

380# ---------------------------------------------------------------------------

381# helpers

382# ---------------------------------------------------------------------------

383

384

385def meta_to_dict(meta: Any) -> dict[str, Any]:

386 """Serialize meta to dict, or pass through a dict already."""

387 if isinstance(meta, dict):

388 return meta

389 from hyper_parallel.codegen.meta import meta_to_dict as _meta_to_dict

390

391 return _meta_to_dict(meta)

392

393

394def _iter_trainable_params(model: Any):

395 """Yield ``fqn`` for every trainable parameter of ``model`` (or its config)."""

396 try:

397 import torch # pylint: disable=unused-import

398 except ImportError:

399 return

400 seen: set[int] = set()

401 for module_name, module in model.named_modules():

402 for param_name, param in module.named_parameters(recurse=False):

403 if not param.requires_grad:

404 continue

405 fqn = f"{module_name}.{param_name}" if module_name else param_name

406 if id(param) in seen:

407 continue

408 seen.add(id(param))

409 yield fqn

410

411

412def _get_parameter(model: Any, fqn: str) -> Any:

413 """Look up a parameter by dotted FQN, tolerating an absent attribute."""

414 obj = model

415 for part in fqn.split("."):

416 try:

417 obj = getattr(obj, part)

418 except AttributeError:

419 return None

420 return obj if hasattr(obj, "requires_grad") else None

421

422

423__all__ = [

424 "verify_boundary_forms",

425 "verify_generated_import",

426 "verify_meta_required_fields",

427 "verify_output_hashes",

428 "verify_param_plan",

429 "verify_signature",

430 "warn_skipped_overrides",

431]

Coverage for / home / jenkins / .local / lib / python3.10 / site-packages / hyper_parallel / codegen / check / preflight.py: 12%

139 statements