Coverage for / home / jenkins / .local / lib / python3.10 / site-packages / hyper_parallel / tools / model_integration / cli.py: 60%
710 statements
« prev ^ index » next coverage.py v7.13.1, created at 2026-10-25 05:45 +0800
1# Copyright 2026 Huawei Technologies Co., Ltd
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14# ============================================================================
15"""Auditable CLI for model inspection, model integration, parity, and precision validation."""
17from __future__ import annotations
19import argparse
20import json
21import os
22import subprocess
23import sys
24from pathlib import Path
25from typing import Any, Iterable, Mapping, Optional
27import yaml
29from hyper_parallel.components.checkpoint import STEP_PREFIX
30from hyper_parallel.tools.model_integration.evidence_store import (
31 EvidenceStore,
32 find_repository_root,
33)
34from hyper_parallel.tools.model_integration.structure_inventory import (
35 ModelInventory,
36 build_model_inventory,
37 diff_inventories,
38 inspect_local_model_assets,
39)
40from hyper_parallel.tools.model_integration.manifest import (
41 ManifestError,
42 ValidationManifest,
43 load_manifest,
44)
45from hyper_parallel.tools.model_integration.module_parity import (
46 ParityContext,
47 run_module_parity,
48)
49from hyper_parallel.tools.model_integration.case_compare import (
50 compare_checkpoint_layouts,
51 compare_parameter_probes,
52 compare_preflight_configs,
53 compare_scalar_curves,
54 load_jsonl,
55 summarize_performance,
56)
57from hyper_parallel.tools.model_integration.case_matrix import (
58 generate_validation_cases,
59)
60from hyper_parallel.tools.model_integration.schemas import (
61 ModelIntegrationReport,
62 FindingSeverity,
63 IntegrationState,
64 ModelIntegrationFinding,
65 SCHEMA_VERSION,
66)
67from hyper_parallel.tools.model_integration.integration_validator import (
68 validate_final_model,
69)
70from hyper_parallel.models.registry import get_model_adapter
71from hyper_parallel.models.adapter_spec import RecomputePolicy
72from hyper_parallel.models.validation_spec import ModelValidationSpec
75_REPOSITORY_ROOT = find_repository_root()
76_SHARED_INITIAL_CHECKPOINT_STEP = 1
77_HCCL_PORT_STRIDE = 256
80def _manifest_from_args(args: argparse.Namespace) -> ValidationManifest:
81 manifest = load_manifest(args.manifest, output_dir=args.output_dir)
82 manifest.validate_for(args.command)
83 return manifest
86def _store(manifest: ValidationManifest) -> EvidenceStore:
87 store = EvidenceStore(manifest.output_dir)
88 store.write_yaml("manifest.resolved.yaml", manifest.to_dict())
89 store.capture_environment()
90 return store
93def _ensure_discovered(store: EvidenceStore) -> None:
94 """Initialize the first validation gate for an existing adapter."""
95 if store.current_state() is None:
96 store.update_state(IntegrationState.DISCOVERED, ("manifest.resolved.yaml",))
99def _require_state(store: EvidenceStore, expected: IntegrationState, command: str) -> None:
100 """Require the previous evidence-backed gate before a dependent command."""
101 actual = store.current_state()
102 if actual is not expected:
103 actual_name = "none" if actual is None else actual.value
104 raise ManifestError(
105 f"{command} requires integration state {expected.value}, got {actual_name}; "
106 "run the preceding model-integration gates in the same output directory"
107 )
110def _require_precision_handoff(
111 store: EvidenceStore,
112 manifest: ValidationManifest,
113) -> None:
114 """Accept same-run parity state or import an explicit passing handoff."""
115 current_state = store.current_state()
116 if current_state is IntegrationState.MODULE_PARITY_PASSED:
117 return
118 if current_state in (IntegrationState.FAILED, IntegrationState.BLOCKED):
119 raise ManifestError(
120 f"validate cannot import a handoff into terminal integration state "
121 f"{current_state.value}; use a new output directory"
122 )
123 handoff_path = manifest.integration_handoff_path
124 if handoff_path is None:
125 _require_state(store, IntegrationState.MODULE_PARITY_PASSED, "validate")
126 return
127 handoff = yaml.safe_load(handoff_path.read_text(encoding="utf-8"))
128 store.write_yaml("integration_handoff.yaml", handoff)
129 imported_evidence = []
130 for handoff_key, default_path, destination in (
131 ("structure_findings", "check/findings.json", "check/findings.json"),
132 ("module_parity", "module_parity/comparison.json", "module_parity/comparison.json"),
133 ("checkpoint_coverage", "checkpoint/coverage.json", "checkpoint/coverage.json"),
134 ):
135 source = Path(handoff.get(handoff_key, default_path))
136 if not source.is_absolute():
137 source = handoff_path.parent / source
138 if not source.is_file():
139 raise ManifestError(
140 f"integration_handoff {handoff_key} evidence does not exist: {source}"
141 )
142 try:
143 payload = json.loads(source.read_text(encoding="utf-8"))
144 except (OSError, json.JSONDecodeError) as error:
145 raise ManifestError(
146 f"could not read integration_handoff {handoff_key} evidence: {error}"
147 ) from error
148 store.write_json(destination, payload)
149 imported_evidence.append(destination)
150 for state in (
151 IntegrationState.DISCOVERED,
152 IntegrationState.STRUCTURE_VALIDATED,
153 IntegrationState.MODULE_PARITY_PASSED,
154 ):
155 store.advance_if_before(
156 state,
157 ("integration_handoff.yaml", *imported_evidence),
158 reason="imported passing integration handoff",
159 )
162def _build_declared_model(manifest: ValidationManifest, section_name: str) -> Any:
163 """Build the model declared by one manifest section."""
164 section = getattr(manifest, section_name)
165 builder = section.get("builder")
166 if not isinstance(builder, Mapping):
167 raise ManifestError(
168 f"{section_name}.builder is required and must contain _target_ for this command"
169 )
170 target = builder.get("_target_")
171 callable_value = manifest.import_callable(target)
172 kwargs = {name: value for name, value in builder.items() if name != "_target_"}
173 result = callable_value(**kwargs)
174 if not hasattr(result, "named_modules"):
175 raise ManifestError(
176 f"{section_name}.builder returned {type(result).__name__}, expected a module"
177 )
178 return result
181def _static_inventory(inspection: Mapping[str, Any], source: Mapping[str, Any]) -> ModelInventory:
182 return ModelInventory(
183 source=dict(source),
184 config=dict(inspection.get("config", {})),
185 checkpoint=dict(inspection.get("checkpoint", {})),
186 )
189def _inspect_side(
190 manifest: ValidationManifest,
191 section_name: str,
192) -> ModelInventory:
193 """Collect the module inventory for one manifest section."""
194 section = getattr(manifest, section_name)
195 if isinstance(section.get("builder"), Mapping):
196 return build_model_inventory(
197 _build_declared_model(manifest, section_name),
198 source={
199 "section": section_name,
200 "revision": section.get("revision"),
201 },
202 )
203 path = manifest.model_path if section_name == "model" else manifest.reference_path
204 if path is None:
205 raise ManifestError(
206 f"{section_name} requires either a local path or an explicit builder"
207 )
208 return _static_inventory(
209 inspect_local_model_assets(path),
210 {"section": section_name, "path": str(path), "revision": section.get("revision")},
211 )
214def command_inspect(args: argparse.Namespace) -> int:
215 """Create reference/candidate inventories and a classified diff."""
216 manifest = _manifest_from_args(args)
217 store = _store(manifest)
218 candidate = _inspect_side(manifest, "model")
219 store.write_json("inventory/candidate.json", candidate.to_dict())
220 differences = []
221 if manifest.reference:
222 reference = _inspect_side(manifest, "reference")
223 store.write_json("inventory/reference.json", reference.to_dict())
224 differences = [
225 difference.to_dict()
226 for difference in diff_inventories(reference, candidate)
227 ]
228 store.write_json(
229 "inventory/structure_diff.json",
230 {"status": "REVIEW_REQUIRED" if differences else "PASS", "differences": differences},
231 )
232 store.write_json("source_fingerprints.json", manifest.fingerprint_paths())
233 store.advance_if_before(
234 IntegrationState.DISCOVERED,
235 (
236 "environment.json",
237 "manifest.resolved.yaml",
238 "inventory/candidate.json",
239 "inventory/structure_diff.json",
240 ),
241 )
242 print(store.root)
243 return 0
246_LICENSE = """# Copyright 2026 Huawei Technologies Co., Ltd
247#
248# Licensed under the Apache License, Version 2.0 (the \"License\");
249# you may not use this file except in compliance with the License.
250# You may obtain a copy of the License at
251#
252# http://www.apache.org/licenses/LICENSE-2.0
253#
254# Unless required by applicable law or agreed to in writing, software
255# distributed under the License is distributed on an \"AS IS\" BASIS,
256# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
257# See the License for the specific language governing permissions and
258# limitations under the License.
259# ============================================================================
260"""
263def command_scaffold(args: argparse.Namespace) -> int:
264 """Generate a minimal adapter/validation skeleton in a new directory."""
265 manifest = _manifest_from_args(args)
266 destination = Path(args.scaffold_dir).expanduser().resolve()
267 if destination.exists() and any(destination.iterdir()):
268 raise ManifestError(f"scaffold destination must be new or empty: {destination}")
269 adapter_dir = destination / "adapter"
270 adapter_dir.mkdir(parents=True, exist_ok=True)
271 family = manifest.family
272 files = {
273 destination / "__init__.py": _LICENSE + f'"""{family} model integration package."""\n',
274 adapter_dir / "__init__.py": _LICENSE + f'"""{family} adapter providers."""\n',
275 adapter_dir / "registration.py": _LICENSE
276 + '"""Register model-owned integration providers."""\n\n'
277 + "from hyper_parallel.models.adapter_spec import ModelAdapterSpec\n"
278 + "from hyper_parallel.models.registry import register_model_adapter\n\n\n"
279 + "def _load_validation():\n"
280 + ' """Load validation-only declarations lazily."""\n'
281 + " from .validation import get_validation_spec # pylint: disable=C0415\n\n"
282 + " return get_validation_spec()\n\n\n"
283 + "MODEL_ADAPTER_SPEC = ModelAdapterSpec(\n"
284 + f" architecture={manifest.model.get('architecture', family)!r},\n"
285 + f" model_type={manifest.model.get('model_type', family)!r},\n"
286 + " validation=_load_validation,\n"
287 + ")\n"
288 + "register_model_adapter(MODEL_ADAPTER_SPEC)\n",
289 adapter_dir / "validation.py": _LICENSE
290 + '"""Declare authoritative module and self-consistency validation."""\n\n'
291 + "from hyper_parallel.models.validation_spec import ModelValidationSpec\n\n\n"
292 + "def get_validation_spec() -> ModelValidationSpec:\n"
293 + ' """Return model-owned validation declarations."""\n'
294 + " return ModelValidationSpec()\n",
295 destination / "validation.yaml": (
296 "schema_version: 2\n"
297 f"model:\n adapter: {family}\n"
298 "launcher:\n"
299 " module: your.training.module\n"
300 " config: path/to/trainer.yaml\n"
301 "matrix:\n"
302 " devices: 1\n"
303 " steps: 10\n"
304 ' baseline: {tp: 1, cp: 1, ep: 1, fsdp: 1, recompute: "off"}\n'
305 "acceptance:\n same_topology: {loss_max_abs: 0.0, norm_max_abs: 0.0}\n"
306 ),
307 destination / "test_registration.py": _LICENSE
308 + '"""Registration smoke test for the generated model adapter."""\n\n'
309 + "import unittest\n\n"
310 + "from hyper_parallel.models.registry import get_model_adapter\n\n\n"
311 + "from .adapter import registration # noqa: F401\n\n\n"
312 + "class TestRegistration(unittest.TestCase):\n"
313 + ' """Verify model-family lookup after registration import."""\n\n'
314 + " def test_registration(self) -> None:\n"
315 + ' """Require the generated adapter to be discoverable."""\n'
316 + f" self.assertIsNotNone(get_model_adapter({family!r}))\n",
317 }
318 for path, content in files.items():
319 path.write_text(content, encoding="utf-8")
320 store = _store(manifest)
321 store.write_json(
322 "scaffold.json",
323 {"files": tuple(str(path.relative_to(destination)) for path in files)},
324 )
325 print(destination)
326 return 0
329def command_check(args: argparse.Namespace) -> int:
330 """Run strict final-tree/materialization/FSDP ownership diagnostics."""
331 manifest = _manifest_from_args(args)
332 store = _store(manifest)
333 try:
334 model = _build_declared_model(manifest, "model")
335 except ManifestError:
336 raise
337 except (ImportError, OSError, RuntimeError, TypeError, ValueError) as error:
338 finding = ModelIntegrationFinding(
339 phase="D2",
340 owner_fqn="<model_builder>",
341 severity=FindingSeverity.ERROR,
342 message="final model construction failed before model integration could inspect the tree",
343 facts={"error_type": type(error).__name__, "error": str(error)},
344 why_unsafe="a partially built or partially planned model cannot enter training",
345 remediation="follow the replacement or planner error and rebuild the final model",
346 related_config=("model.builder", "plan_overrides"),
347 )
348 report = ModelIntegrationReport(findings=[finding])
349 store.write_json("check/findings.json", report.to_dict())
350 _ensure_discovered(store)
351 store.update_state(
352 IntegrationState.FAILED,
353 ("check/findings.json",),
354 reason="model construction failed",
355 )
356 print(finding.format_human(), file=sys.stderr)
357 return 1
358 report, ownership, fsdp_units = validate_final_model(
359 model,
360 context={"manifest": manifest.to_dict()},
361 )
362 store.write_json("check/findings.json", report.to_dict())
363 store.write_json("check/fsdp_units.json", fsdp_units)
364 store.write_json(
365 "checkpoint/coverage.json",
366 report.metadata.get("checkpoint_coverage", {}),
367 )
368 store.write_csv(
369 "check/parameter_ownership.csv",
370 (entry.to_dict() for entry in ownership),
371 fieldnames=(
372 "parameter_fqn",
373 "owner_fqn",
374 "gradient_domain",
375 "source_mesh",
376 "source_placements",
377 "global_shape",
378 "local_shape",
379 ),
380 )
381 if report.errors:
382 store.update_state(
383 IntegrationState.FAILED,
384 ("check/findings.json", "check/parameter_ownership.csv"),
385 reason="structure model integration reported ERROR findings",
386 )
387 print("\n\n".join(finding.format_human() for finding in report.errors), file=sys.stderr)
388 return 1
389 _ensure_discovered(store)
390 store.update_state(
391 IntegrationState.STRUCTURE_VALIDATED,
392 ("check/findings.json", "check/parameter_ownership.csv", "check/fsdp_units.json"),
393 )
394 print(store.root)
395 return 0
398def _validation_spec(manifest: ValidationManifest) -> ModelValidationSpec:
399 """Resolve the model-owned validation spec from the adapter."""
400 adapter = get_model_adapter(manifest.family)
401 if adapter is None:
402 raise ManifestError(f"no model adapter registered for {manifest.family!r}")
403 provider = adapter.validation
404 if provider is None or not callable(provider):
405 raise ManifestError(
406 f"model adapter {manifest.family!r} does not declare a validation provider"
407 )
408 validation_spec = provider()
409 if not isinstance(validation_spec, ModelValidationSpec):
410 raise ManifestError("adapter validation provider returned the wrong type")
411 return validation_spec
414def _recompute_policy(manifest: ValidationManifest) -> Optional[RecomputePolicy]:
415 """Resolve normal-training recompute policy for precision case generation."""
416 adapter = get_model_adapter(manifest.family)
417 provider = None if adapter is None else adapter.recompute
418 if provider is None:
419 return None
420 recompute_policy = provider()
421 if not isinstance(recompute_policy, RecomputePolicy):
422 raise ManifestError("adapter recompute provider returned the wrong type")
423 return recompute_policy
426def _parameter_tolerance(
427 acceptance: Mapping[str, Any],
428 profile: str,
429) -> dict[str, Any]:
430 """Resolve strict base parameter limits plus an optional named override."""
431 parameters = acceptance.get("parameters", {})
432 if not isinstance(parameters, Mapping):
433 raise ManifestError("acceptance.parameters must be a mapping")
434 profile_names = ("module_parity", "same_topology", "cross_topology")
435 tolerance = {
436 name: value
437 for name, value in parameters.items()
438 if name not in profile_names
439 }
440 override = parameters.get(profile)
441 if override is None:
442 return tolerance
443 if not isinstance(override, Mapping):
444 raise ManifestError(f"acceptance.parameters.{profile} must be a mapping")
445 override = dict(override)
446 if "summary" in override:
447 base_summary = tolerance.get("summary", {})
448 override_summary = override["summary"]
449 if not isinstance(base_summary, Mapping) or not isinstance(override_summary, Mapping):
450 raise ManifestError(
451 f"acceptance.parameters.{profile}.summary must be a mapping"
452 )
453 override["summary"] = {**base_summary, **override_summary}
454 tolerance.update(override)
455 return tolerance
458def command_parity(args: argparse.Namespace) -> int:
459 """Run final-module authoritative parity cases from the model provider."""
460 manifest = _manifest_from_args(args)
461 store = _store(manifest)
462 _require_state(store, IntegrationState.STRUCTURE_VALIDATED, "parity")
463 validation_spec = _validation_spec(manifest)
464 tolerance = _parameter_tolerance(manifest.acceptance, "module_parity")
465 context = ParityContext(
466 manifest=manifest,
467 dtype=args.dtype,
468 device=args.device,
469 evidence_store=store,
470 options={"tolerance": tolerance},
471 )
472 report = run_module_parity(validation_spec, context)
473 if report["status"] == "PASS":
474 store.write_yaml(
475 "integration_handoff.yaml",
476 {
477 "schema_version": SCHEMA_VERSION,
478 "family": manifest.family,
479 "manifest": "manifest.resolved.yaml",
480 "structure_findings": "check/findings.json",
481 "module_parity": "module_parity/comparison.json",
482 "checkpoint_coverage": "checkpoint/coverage.json",
483 "status": "PASS",
484 },
485 )
486 store.update_state(
487 IntegrationState.MODULE_PARITY_PASSED,
488 ("module_parity/comparison.json", "integration_handoff.yaml"),
489 )
490 return 0
491 terminal = IntegrationState.FAILED if report["status"] == "FAIL" else IntegrationState.BLOCKED
492 store.update_state(terminal, ("module_parity/comparison.json",), reason=report["status"])
493 return 1 if report["status"] == "FAIL" else 2
496def _phase_train_iters(phase: str, steps: int, split_step: int) -> int:
497 """Map the launch phase to the number of executed training steps."""
498 if phase == "initialize":
499 return _SHARED_INITIAL_CHECKPOINT_STEP
500 if phase == "prepare":
501 return split_step
502 return steps
505def _shared_checkpoint_overrides(
506 manifest: ValidationManifest,
507 phase: str,
508) -> dict[str, Any]:
509 """Replay every measured case from the same warm-start checkpoint."""
510 if not bool(manifest.matrix.get("shared_initial_checkpoint")):
511 return {}
512 if phase in ("initialize", "restore"):
513 return {}
514 initial_checkpoint_dir = (
515 manifest.output_dir / "cases" / "baseline" / "initial_checkpoint"
516 )
517 # The warm-start step exists to align model/optimizer/RNG state. Every
518 # measured case must replay from the same configured data start; a
519 # stateful Online cursor may also be impossible to restore after TP/CP
520 # changes the DP world size. True resume phases keep the default and
521 # restore the cursor saved at resume_split_step.
522 return {
523 "checkpoint.restore_from": str(
524 initial_checkpoint_dir
525 / f"{STEP_PREFIX}{_SHARED_INITIAL_CHECKPOINT_STEP}"
526 ),
527 "checkpoint.restore_dataloader_state": False,
528 }
531def _phase_checkpoint_overrides(
532 manifest: ValidationManifest,
533 case: Any,
534 phase: str,
535 evidence_dir: Path,
536 resume_dir: Path,
537 split_step: int,
538) -> dict[str, Any]:
539 """Resolve checkpoint save/restore overrides for one launch phase."""
540 initial_checkpoint_dir = (
541 manifest.output_dir / "cases" / "baseline" / "initial_checkpoint"
542 )
543 if phase == "initialize":
544 return {
545 "checkpoint.save_ckpt": True,
546 "checkpoint.save_steps": 1,
547 "checkpoint.save_epochs": 0,
548 "checkpoint.checkpoint_dir": str(initial_checkpoint_dir),
549 }
550 if phase == "prepare":
551 return {
552 "checkpoint.save_ckpt": True,
553 "checkpoint.save_steps": split_step,
554 "checkpoint.save_epochs": 0,
555 "checkpoint.checkpoint_dir": str(resume_dir),
556 }
557 if phase == "restore":
558 return {
559 "checkpoint.save_ckpt": False,
560 "checkpoint.restore_from": str(resume_dir / f"{STEP_PREFIX}{split_step}"),
561 "checkpoint.checkpoint_dir": str(resume_dir),
562 }
563 checkpoint_enabled = case.name == "baseline" and bool(
564 manifest.matrix.get("same_topology_resume")
565 or manifest.matrix.get("cross_topology_resume")
566 )
567 overrides: dict[str, Any] = {"checkpoint.save_ckpt": checkpoint_enabled}
568 if checkpoint_enabled:
569 overrides.update(
570 {
571 "checkpoint.save_steps": split_step,
572 "checkpoint.save_epochs": 0,
573 "checkpoint.checkpoint_dir": str(evidence_dir / "dcp"),
574 }
575 )
576 return overrides
579def _trainer_launch_argv(
580 manifest: ValidationManifest,
581 case: Any,
582 phase: str,
583 evidence_dir: Path,
584 resume_dir: Path,
585 topology: Optional[Mapping[str, Any]] = None,
586) -> list[str]:
587 """Build the standard Trainer command from one concise launcher declaration."""
588 launcher = manifest.launcher
589 module = launcher["module"]
590 config = manifest.launcher_config_path
591 if config is None:
592 raise ManifestError("launcher.config is required for the standard Trainer launcher")
593 topology = case.topology if topology is None else topology
594 steps = int(manifest.matrix["steps"])
595 split_step = int(manifest.matrix.get("resume_split_step", steps))
596 tokens = [
597 "torchrun",
598 "--standalone",
599 f"--nproc_per_node={manifest.matrix['devices']}",
600 "--module",
601 module,
602 str(config),
603 ]
604 controlled_overrides = {
605 "training.train_iters": _phase_train_iters(phase, steps, split_step),
606 # Split-run preparation stops early but must retain the full-run LR
607 # curve so its checkpoint is numerically comparable with the baseline.
608 "training.lr_scheduler_iters": steps,
609 "accelerator.tp_size": topology.get("tp", 1),
610 "accelerator.cp_size": topology.get("cp", 1),
611 "accelerator.ep_size": topology.get("ep", 1),
612 "accelerator.sequence_parallel": topology.get("sequence_parallel", False),
613 "fsdp_config.dp_shard_size": topology.get("fsdp", 1),
614 "activation_checkpoint.mode": "selective",
615 "activation_checkpoint.selection": {
616 "source": "model_adapter_safe_regions",
617 **topology.get("recompute", {"layer_count": 0}),
618 },
619 "model_integration.mode": "runtime",
620 }
621 if "validate_placement" in topology:
622 controlled_overrides["model.validate_placement"] = topology["validate_placement"]
623 controlled_overrides.update(_shared_checkpoint_overrides(manifest, phase))
624 controlled_overrides.update(
625 _phase_checkpoint_overrides(
626 manifest,
627 case,
628 phase,
629 evidence_dir,
630 resume_dir,
631 split_step,
632 )
633 )
634 tokens.extend(
635 f"--{name}={value}"
636 for name, value in controlled_overrides.items()
637 )
638 return tokens
641def _resolve_case_launches(
642 manifest: ValidationManifest,
643 case: Any,
644 case_dir: Path,
645) -> list[tuple[str, list[str], Path]]:
646 """Resolve train or prepare/restore commands without executing them."""
647 resume_dir = case_dir / "resume_checkpoint"
648 if case.kind != "resume":
649 launches = []
650 if case.name == "baseline" and manifest.matrix.get("shared_initial_checkpoint"):
651 initialize_dir = case_dir / "initialize"
652 launches.append(
653 (
654 "initialize",
655 _trainer_launch_argv(
656 manifest,
657 case,
658 "initialize",
659 initialize_dir,
660 resume_dir,
661 ),
662 initialize_dir,
663 )
664 )
665 launches.append(
666 (
667 "train",
668 _trainer_launch_argv(
669 manifest, case, "train", case_dir, resume_dir
670 ),
671 case_dir,
672 )
673 )
674 return launches
675 prepare_dir = case_dir / "prepare"
676 prepare_topology = case.metadata.get("prepare_topology", case.topology)
677 return [
678 (
679 "prepare",
680 _trainer_launch_argv(
681 manifest,
682 case,
683 "prepare",
684 prepare_dir,
685 resume_dir,
686 topology=prepare_topology,
687 ),
688 prepare_dir,
689 ),
690 (
691 "restore",
692 _trainer_launch_argv(
693 manifest, case, "restore", case_dir, resume_dir
694 ),
695 case_dir,
696 ),
697 ]
700def _case_environment(
701 case: Any,
702 phase: str,
703 evidence_dir: Path,
704 resume_dir: Path,
705 *,
706 port_slot: int = 0,
707 port_stride: int = 1,
708) -> dict[str, str]:
709 """Add case identity and isolate sequential distributed launch ports."""
710 environment = os.environ.copy()
711 environment.update(
712 {
713 "HP_MODEL_INTEGRATION_CASE": case.name,
714 "HP_MODEL_INTEGRATION_KIND": case.kind,
715 "HP_MODEL_INTEGRATION_PHASE": phase,
716 "HP_MODEL_INTEGRATION_COMPARE_TO": case.compare_to,
717 "HP_MODEL_INTEGRATION_EVIDENCE_DIR": str(evidence_dir),
718 "HP_MODEL_INTEGRATION_RESUME_DIR": str(resume_dir),
719 "HYPER_PARALLEL_MODEL_INTEGRATION_OUTPUT_DIR": str(evidence_dir),
720 }
721 )
722 hccl_base_port = environment.get("HCCL_IF_BASE_PORT")
723 if hccl_base_port is not None:
724 try:
725 isolated_port = int(hccl_base_port) + port_slot * port_stride
726 except ValueError as error:
727 raise ManifestError(
728 "HCCL_IF_BASE_PORT must be an integer when model-integration "
729 f"validation isolates case ports, got {hccl_base_port!r}"
730 ) from error
731 if not 0 < isolated_port <= 65535:
732 raise ManifestError(
733 "model-integration HCCL port isolation exceeds the valid port "
734 f"range: base={hccl_base_port}, slot={port_slot}, "
735 f"stride={port_stride}"
736 )
737 environment["HCCL_IF_BASE_PORT"] = str(isolated_port)
738 return environment
741def _execute_case_launch(
742 manifest: ValidationManifest,
743 case: Any,
744 launch: Mapping[str, Any],
745 resume_dir: Path,
746) -> tuple[Optional[int], Optional[str]]:
747 """Execute one resolved process and persist its stdout and stderr."""
748 phase = launch["phase"]
749 evidence_dir = Path(launch["evidence_dir"])
750 evidence_dir.mkdir(parents=True, exist_ok=True)
751 try:
752 result = subprocess.run(
753 launch["argv"],
754 check=False,
755 capture_output=True,
756 text=True,
757 env=_case_environment(
758 case,
759 phase,
760 evidence_dir,
761 resume_dir,
762 port_slot=int(launch.get("port_slot", 0)),
763 port_stride=max(
764 _HCCL_PORT_STRIDE,
765 int(manifest.matrix.get("devices", 1)),
766 ),
767 ),
768 cwd=_REPOSITORY_ROOT,
769 timeout=manifest.launcher.get("timeout_seconds"),
770 )
771 except OSError as error:
772 return None, f"{phase} launcher could not start: {error}"
773 except subprocess.TimeoutExpired as error:
774 evidence_dir.joinpath("stdout.log").write_text(error.stdout or "", encoding="utf-8")
775 evidence_dir.joinpath("stderr.log").write_text(error.stderr or "", encoding="utf-8")
776 timeout = manifest.launcher.get("timeout_seconds")
777 return None, f"{phase} launcher exceeded timeout_seconds={timeout}"
778 evidence_dir.joinpath("stdout.log").write_text(result.stdout, encoding="utf-8")
779 evidence_dir.joinpath("stderr.log").write_text(result.stderr, encoding="utf-8")
780 return result.returncode, None
783def _run_validation_case(
784 manifest: ValidationManifest,
785 store: EvidenceStore,
786 case: Any,
787 case_index: int,
788) -> dict[str, Any]:
789 """Run one resolved validation case and return its report payload."""
790 case_dir = store.path(f"cases/{case.name}")
791 case_dir.mkdir(parents=True, exist_ok=True)
792 resume_dir = case_dir / "resume_checkpoint"
793 launches = _resolve_case_launches(manifest, case, case_dir)
794 commands = [
795 {
796 "phase": phase,
797 "argv": phase_argv,
798 "evidence_dir": str(phase_evidence_dir),
799 "port_slot": case_index * 2 + launch_index,
800 }
801 for launch_index, (phase, phase_argv, phase_evidence_dir) in enumerate(launches)
802 ]
803 store.write_json(
804 f"cases/{case.name}/preflight.json",
805 {"case": case.to_dict(), "launches": commands},
806 )
807 returncodes = []
808 for launch in commands:
809 returncode, blocked_reason = _execute_case_launch(
810 manifest,
811 case,
812 launch,
813 resume_dir,
814 )
815 if blocked_reason is not None:
816 return {"case": case.name, "status": "BLOCKED", "reason": blocked_reason}
817 returncodes.append(returncode)
818 if returncode:
819 break
820 return {
821 "case": case.name,
822 "status": "PASS" if not any(returncodes) else "FAIL",
823 "returncodes": returncodes,
824 "metrics": f"cases/{case.name}/metrics.jsonl",
825 }
828def _case_artifact(case_dir: Path, relative_path: str) -> Path:
829 """Resolve a launcher-owned artifact or the standard Trainer model integration location."""
830 direct = case_dir / relative_path
831 if direct.exists():
832 return direct
833 runtime = case_dir / "cases" / "runtime" / relative_path
834 return runtime
837def _performance_summary(case_dir: Path, warmup_steps: int) -> dict[str, Any]:
838 """Summarize rank-zero timing and maximum allocator usage over all ranks."""
839 runtime_dir = case_dir / "cases" / "runtime"
840 rank_files = tuple(sorted(runtime_dir.glob("performance_rank*.jsonl")))
841 memory_rows = [row for path in rank_files for row in load_jsonl(path)]
842 return summarize_performance(
843 load_jsonl(_case_artifact(case_dir, "performance.jsonl")),
844 warmup_steps=warmup_steps,
845 memory_rows=memory_rows,
846 )
849def _aggregate_statuses(statuses: Iterable[str]) -> str:
850 """Reduce phase statuses using FAIL > BLOCKED > PASS precedence."""
851 statuses = tuple(statuses)
852 if "FAIL" in statuses:
853 return "FAIL"
854 if "BLOCKED" in statuses:
855 return "BLOCKED"
856 return "PASS"
859def _compare_validation_case(
860 manifest: ValidationManifest,
861 case: Any,
862 baseline_dir: Path,
863 candidate_dir: Path,
864 launch_statuses: tuple[str, str],
865) -> dict[str, Any]:
866 """Compare one candidate against the baseline using declared contracts."""
867 try:
868 tolerance_name = case.metadata["acceptance"]
869 except (AttributeError, KeyError) as error:
870 raise ManifestError(
871 f"validation case {case.name!r} has no acceptance classification; "
872 "regenerate the case matrix with generate_validation_cases()"
873 ) from error
874 preflight = compare_preflight_configs(
875 baseline_dir,
876 candidate_dir,
877 )
878 scalar = compare_scalar_curves(
879 load_jsonl(_case_artifact(baseline_dir, "metrics.jsonl")),
880 load_jsonl(_case_artifact(candidate_dir, "metrics.jsonl")),
881 manifest.acceptance.get(tolerance_name, {}),
882 allow_candidate_subset=case.kind == "resume",
883 )
884 parameter = compare_parameter_probes(
885 baseline_dir,
886 candidate_dir,
887 _parameter_tolerance(manifest.acceptance, tolerance_name),
888 allow_candidate_subset=case.kind == "resume",
889 compare_optimizer_states=tolerance_name == "same_topology",
890 )
891 checkpoint_layout = None
892 statuses = (*launch_statuses, preflight["status"], scalar["status"], parameter["status"])
893 if case.kind == "resume":
894 checkpoint_layout = compare_checkpoint_layouts(
895 baseline_dir,
896 candidate_dir,
897 same_topology=case.metadata.get("resume") == "same_topology",
898 )
899 statuses = (*statuses, checkpoint_layout["status"])
900 return {
901 "status": _aggregate_statuses(statuses),
902 "preflight": preflight,
903 "scalar": scalar,
904 "parameter_probes": parameter,
905 "checkpoint_layout": checkpoint_layout,
906 }
909def _finalize_validation(
910 store: EvidenceStore,
911 results: list[dict[str, Any]],
912 comparisons: Mapping[str, Mapping[str, Any]],
913 performance: Mapping[str, Mapping[str, Any]],
914) -> int:
915 """Persist the matrix result and advance the evidence state machine."""
916 status = _aggregate_statuses(
917 [result["status"] for result in results]
918 + [comparison["status"] for comparison in comparisons.values()]
919 )
920 store.write_json(
921 "comparison.json",
922 {
923 "status": status,
924 "cases": results,
925 "comparisons": comparisons,
926 "performance": performance,
927 },
928 )
929 if status == "PASS":
930 store.advance_if_before(IntegrationState.MATRIX_PASSED, ("comparison.json",))
931 return 0
932 terminal = IntegrationState.FAILED if status == "FAIL" else IntegrationState.BLOCKED
933 store.update_state(terminal, ("comparison.json",), reason=status)
934 return 1 if status == "FAIL" else 2
937def command_validate(args: argparse.Namespace) -> int:
938 """Generate and optionally execute the minimum precision matrix."""
939 manifest = _manifest_from_args(args)
940 store = _store(manifest)
941 _require_precision_handoff(store, manifest)
942 validation_spec = _validation_spec(manifest)
943 try:
944 cases = generate_validation_cases(
945 manifest.matrix,
946 _recompute_policy(manifest),
947 validation_spec.topology_constraints,
948 )
949 except (TypeError, ValueError) as error:
950 raise ManifestError(f"invalid validation matrix: {error}") from error
951 store.write_json("cases/resolved_cases.json", [case.to_dict() for case in cases])
952 if args.generate_only:
953 print(store.path("cases/resolved_cases.json"))
954 return 0
955 results = [
956 _run_validation_case(manifest, store, case, case_index)
957 for case_index, case in enumerate(cases)
958 ]
959 results_by_name = {result["case"]: result for result in results}
960 baseline_dir = store.path("cases/baseline")
961 warmup_steps = int(manifest.matrix.get("performance_warmup_steps", 1))
962 comparisons = {}
963 performance = {"baseline": _performance_summary(baseline_dir, warmup_steps)}
964 for case in cases:
965 if case.name == "baseline":
966 continue
967 candidate_dir = store.path(f"cases/{case.name}")
968 launch_statuses = (
969 results_by_name["baseline"]["status"],
970 results_by_name[case.name]["status"],
971 )
972 comparisons[case.name] = _compare_validation_case(
973 manifest,
974 case,
975 baseline_dir,
976 candidate_dir,
977 launch_statuses,
978 )
979 performance[case.name] = _performance_summary(candidate_dir, warmup_steps)
980 return _finalize_validation(store, results, comparisons, performance)
983def _read_json(path: Path) -> Optional[dict[str, Any]]:
984 """Read an optional JSON evidence file."""
985 if not path.exists():
986 return None
987 return json.loads(path.read_text(encoding="utf-8"))
990def _report_entries(
991 findings: Optional[Mapping[str, Any]],
992 parity: Optional[Mapping[str, Any]],
993 comparison: Optional[Mapping[str, Any]],
994) -> tuple[list[str], list[str], list[str], list[str]]:
995 """Render compact structure, parity, precision, and performance entries."""
996 structure_entries = ["not executed"]
997 if findings is not None:
998 structure_entries = [
999 f"{finding['severity']}: {finding['message']} "
1000 f"(fix: {finding.get('remediation') or 'n/a'})"
1001 for finding in findings.get("findings", ())
1002 ] or ["PASS: no ERROR findings"]
1003 parity_entries = ["not executed"]
1004 if parity is not None:
1005 parity_entries = [
1006 f"{case.get('case', '<unnamed>')}: {case.get('status')}"
1007 + (f" ({case.get('reason')})" if case.get("reason") else "")
1008 for case in parity.get("cases", ())
1009 ] or [f"{parity.get('status')}: no executed cases"]
1010 precision_entries = ["not executed"]
1011 performance_entries = ["not executed"]
1012 if comparison is not None:
1013 precision_entries = [
1014 f"{name}: {value.get('status')}"
1015 for name, value in comparison.get("comparisons", {}).items()
1016 ] or [f"{comparison.get('status')}: baseline only"]
1017 performance_entries = [
1018 f"{name}: step_p50={value.get('step_time_p50_seconds')}, "
1019 f"tokens/s={value.get('tokens_per_second_p50')}, "
1020 f"peak_allocated={value.get('peak_memory_allocated_bytes')} bytes"
1021 for name, value in comparison.get("performance", {}).items()
1022 ] or ["not measured"]
1023 return structure_entries, parity_entries, precision_entries, performance_entries
1026def _resolved_report_cases(store: EvidenceStore) -> list[dict[str, Any]]:
1027 """Load resolved matrix cases for report rendering."""
1028 cases_path = store.path("cases/resolved_cases.json")
1029 if not cases_path.is_file():
1030 return []
1031 payload = json.loads(cases_path.read_text(encoding="utf-8"))
1032 return payload if isinstance(payload, list) else []
1035def _report_manifest(store: EvidenceStore) -> dict[str, Any]:
1036 """Load the resolved manifest without requiring the original input path."""
1037 manifest_path = store.path("manifest.resolved.yaml")
1038 if not manifest_path.is_file():
1039 return {}
1040 payload = yaml.safe_load(manifest_path.read_text(encoding="utf-8")) or {}
1041 return payload if isinstance(payload, dict) else {}
1044def _case_category(case: Mapping[str, Any]) -> str:
1045 """Classify one case without inflating strategy coverage with mechanics."""
1046 name = str(case.get("name", ""))
1047 if name == "baseline":
1048 return "baseline"
1049 if name.startswith(("axis-", "combined-")):
1050 return "strategy"
1051 if name.startswith("recompute_"):
1052 return "recompute"
1053 if case.get("kind") == "resume":
1054 return "resume"
1055 if case.get("kind") == "production_validate":
1056 return "production_validate"
1057 return "other"
1060def _coverage_counts(resolved_cases: Iterable[Mapping[str, Any]]) -> dict[str, int]:
1061 """Count independent evidence categories."""
1062 counts = {
1063 "baseline": 0,
1064 "strategy": 0,
1065 "recompute": 0,
1066 "resume": 0,
1067 "production_validate": 0,
1068 "other": 0,
1069 }
1070 for case in resolved_cases:
1071 counts[_case_category(case)] += 1
1072 return counts
1075def _coverage_notes(manifest: Mapping[str, Any], language: str) -> list[str]:
1076 """Return localized exclusions, falling back to the canonical notes."""
1077 matrix = manifest.get("matrix", {})
1078 if not isinstance(matrix, Mapping):
1079 return []
1080 key = "coverage_notes_zh" if language == "zh" else "coverage_notes"
1081 notes = matrix.get(key, matrix.get("coverage_notes", ()))
1082 if not isinstance(notes, (list, tuple)):
1083 return []
1084 return [note for note in notes if isinstance(note, str)]
1087def _precision_coverage_entries(store: EvidenceStore, language: str = "en") -> list[str]:
1088 """Summarize real matrix coverage separately from resume mechanics."""
1089 resolved_cases = _resolved_report_cases(store)
1090 counts = _coverage_counts(resolved_cases)
1091 if language == "zh":
1092 entries = [
1093 f"基线={counts['baseline']}，策略泛化={counts['strategy']}，"
1094 f"重计算={counts['recompute']}，断点续训={counts['resume']}，"
1095 f"生产/Validate 对照={counts['production_validate']}"
1096 ]
1097 else:
1098 entries = [
1099 f"baseline={counts['baseline']}, strategy_generalization={counts['strategy']}, "
1100 f"recompute={counts['recompute']}, resume={counts['resume']}, "
1101 f"production_validate={counts['production_validate']}"
1102 ]
1104 manifest = _report_manifest(store)
1105 exclusion_prefix = "排除项" if language == "zh" else "excluded"
1106 entries.extend(
1107 f"{exclusion_prefix}: {note}"
1108 for note in _coverage_notes(manifest, language)
1109 )
1110 return entries
1113def _format_report_number(value: Any) -> str:
1114 """Format optional numerical evidence compactly and deterministically."""
1115 if value is None:
1116 return "n/a"
1117 if isinstance(value, float):
1118 return f"{value:.6g}"
1119 return str(value)
1122def _format_topology(topology: Mapping[str, Any]) -> str:
1123 """Render the stable topology dimensions in one line."""
1124 keys = ("tp", "cp", "ep", "fsdp", "sequence_parallel", "recompute")
1125 return ", ".join(f"{key}={topology.get(key)}" for key in keys)
1128def _scalar_case_summary(comparison: Mapping[str, Any]) -> dict[str, Any]:
1129 """Collapse per-step scalar evidence while preserving identity gates."""
1130 steps = comparison.get("scalar", {}).get("steps", ())
1132 def maximum(name: str) -> Optional[float]:
1133 """Return the largest observed value without converting absent evidence."""
1134 values = [float(row[name]) for row in steps if row.get(name) is not None]
1135 return max(values, default=None)
1137 return {
1138 "steps": len(steps),
1139 "max_loss": maximum("loss_max_abs"),
1140 "max_loss_rel": maximum("loss_max_rel"),
1141 "max_norm": maximum("norm_max_abs"),
1142 "max_norm_rel": maximum("norm_max_rel"),
1143 "max_post_clip_norm": maximum("post_clip_norm_max_abs"),
1144 "max_post_clip_norm_rel": maximum("post_clip_norm_max_rel"),
1145 "input_identity": all(bool(row.get("input_identity")) for row in steps) if steps else False,
1146 "learning_rate_identity": (
1147 all(bool(row.get("learning_rate_equal")) for row in steps) if steps else False
1148 ),
1149 "finite": all(bool(row.get("finite")) for row in steps) if steps else False,
1150 }
1153def _parameter_acceptance_entry(
1154 tolerance: Mapping[str, Any],
1155 label: str,
1156 language: str,
1157) -> str:
1158 """Render exact-value and summary-only parameter limits without conflating them."""
1159 summary = tolerance.get("summary", {})
1160 if not isinstance(summary, Mapping):
1161 summary = {}
1162 if language == "zh":
1163 return (
1164 f"{label}：完整值 max_abs <= {_format_report_number(tolerance.get('max_abs'))}，"
1165 f"relative_l2 <= {_format_report_number(tolerance.get('relative_l2'))}，"
1166 f"组合规则={tolerance.get('combination', 'all')}；仅摘要不声明逐元素误差，"
1167 "L2 范数相对误差 <= "
1168 f"{_format_report_number(summary.get('l2_norm_relative'))}"
1169 )
1170 return (
1171 f"{label}: exact-value max_abs <= {_format_report_number(tolerance.get('max_abs'))}, "
1172 f"relative_l2 <= {_format_report_number(tolerance.get('relative_l2'))}, "
1173 f"combination={tolerance.get('combination', 'all')}; summary-only probes make no "
1174 "elementwise claim, L2-norm relative error <= "
1175 f"{_format_report_number(summary.get('l2_norm_relative'))}"
1176 )
1179def _acceptance_entries(manifest: Mapping[str, Any], language: str) -> list[str]:
1180 """Render every scalar and parameter threshold used by the matrix."""
1181 acceptance = manifest.get("acceptance", {})
1182 if not isinstance(acceptance, Mapping):
1183 acceptance = {}
1184 same = acceptance.get("same_topology", {})
1185 cross = acceptance.get("cross_topology", {})
1186 parity_parameters = _parameter_tolerance(acceptance, "module_parity")
1187 same_parameters = _parameter_tolerance(acceptance, "same_topology")
1188 cross_parameters = _parameter_tolerance(acceptance, "cross_topology")
1189 if language == "zh":
1190 return [
1191 "同拓扑："
1192 f"loss 最大绝对误差 <= {_format_report_number(same.get('loss_max_abs'))}，"
1193 f"loss 最大相对误差 <= {_format_report_number(same.get('loss_max_rel'))}，"
1194 f"梯度范数最大绝对误差 <= {_format_report_number(same.get('norm_max_abs'))}，"
1195 f"梯度范数最大相对误差 <= {_format_report_number(same.get('norm_max_rel'))}，"
1196 f"组合规则={same.get('combination', 'all')}",
1197 "跨拓扑："
1198 f"loss 最大绝对误差 <= {_format_report_number(cross.get('loss_max_abs'))}，"
1199 f"loss 最大相对误差 <= {_format_report_number(cross.get('loss_max_rel'))}，"
1200 f"梯度范数最大绝对误差 <= {_format_report_number(cross.get('norm_max_abs'))}，"
1201 f"梯度范数最大相对误差 <= {_format_report_number(cross.get('norm_max_rel'))}，"
1202 f"组合规则={cross.get('combination', 'all')}",
1203 _parameter_acceptance_entry(parity_parameters, "模块对拍参数", "zh"),
1204 _parameter_acceptance_entry(same_parameters, "同拓扑参数", "zh"),
1205 _parameter_acceptance_entry(cross_parameters, "跨拓扑参数", "zh"),
1206 ]
1207 return [
1208 "same topology: "
1209 f"loss max abs <= {_format_report_number(same.get('loss_max_abs'))}, "
1210 f"loss max rel <= {_format_report_number(same.get('loss_max_rel'))}, "
1211 f"gradient-norm max abs <= {_format_report_number(same.get('norm_max_abs'))}, "
1212 f"gradient-norm max rel <= {_format_report_number(same.get('norm_max_rel'))}, "
1213 f"combination={same.get('combination', 'all')}",
1214 "cross topology: "
1215 f"loss max abs <= {_format_report_number(cross.get('loss_max_abs'))}, "
1216 f"loss max rel <= {_format_report_number(cross.get('loss_max_rel'))}, "
1217 f"gradient-norm max abs <= {_format_report_number(cross.get('norm_max_abs'))}, "
1218 f"gradient-norm max rel <= {_format_report_number(cross.get('norm_max_rel'))}, "
1219 f"combination={cross.get('combination', 'all')}",
1220 _parameter_acceptance_entry(parity_parameters, "module parity parameters", "en"),
1221 _parameter_acceptance_entry(same_parameters, "same-topology parameters", "en"),
1222 _parameter_acceptance_entry(cross_parameters, "cross-topology parameters", "en"),
1223 ]
1226def _baseline_detail_entry(
1227 category: str,
1228 name: str,
1229 topology: str,
1230 default_steps: Any,
1231 language: str,
1232) -> str:
1233 """Render the reference-curve entry for the baseline case."""
1234 if language == "zh":
1235 return (
1236 f"[{category}] `{name}`：{topology}；正式步数={default_steps}；作为参考曲线。"
1237 )
1238 return (
1239 f"[{category}] `{name}`: {topology}; measured_steps={default_steps}; reference curve."
1240 )
1243def _checkpoint_layout_status(
1244 case: Mapping[str, Any],
1245 case_comparison: Mapping[str, Any],
1246 language: str,
1247) -> str:
1248 """Render the DCP layout verdict or explain why none was recorded."""
1249 checkpoint = case_comparison.get("checkpoint_layout")
1250 if isinstance(checkpoint, Mapping):
1251 return checkpoint.get("status", "n/a")
1252 if case.get("kind") != "resume":
1253 return (
1254 "not applicable (non-resume case)"
1255 if language == "en"
1256 else "不适用（非断点续训用例）"
1257 )
1258 return "not recorded" if language == "en" else "未记录"
1261def _case_detail_entry(
1262 case: Mapping[str, Any],
1263 name: str,
1264 category: str,
1265 topology: str,
1266 comparisons: Mapping[str, Any],
1267 acceptance: Any,
1268 language: str,
1269) -> str:
1270 """Render thresholds, scalar gates, probes, and DCP for one case."""
1271 case_comparison = comparisons.get(name, {})
1272 scalar = _scalar_case_summary(case_comparison)
1273 metadata = case.get("metadata", {})
1274 profile = metadata.get("acceptance", "unknown")
1275 threshold = acceptance.get(profile, {}) if isinstance(acceptance, Mapping) else {}
1276 parameter = case_comparison.get("parameter_probes", {})
1277 checkpoint_status = _checkpoint_layout_status(case, case_comparison, language)
1278 identity_status = "PASS" if scalar["input_identity"] else "FAIL"
1279 lr_status = "PASS" if scalar["learning_rate_identity"] else "FAIL"
1280 finite_status = "PASS" if scalar["finite"] else "FAIL"
1281 if language == "zh":
1282 return (
1283 f"[{category}] `{name}`：{topology}；阈值档={profile} "
1284 f"(loss<={_format_report_number(threshold.get('loss_max_abs'))}, "
1285 f"loss_rel<={_format_report_number(threshold.get('loss_max_rel'))}, "
1286 f"norm<={_format_report_number(threshold.get('norm_max_abs'))}, "
1287 f"norm_rel<={_format_report_number(threshold.get('norm_max_rel'))}, "
1288 f"组合={threshold.get('combination', 'all')})；"
1289 f"对比步数={scalar['steps']}；max_loss_abs={_format_report_number(scalar['max_loss'])}；"
1290 f"max_loss_rel={_format_report_number(scalar['max_loss_rel'])}；"
1291 f"max_norm_abs={_format_report_number(scalar['max_norm'])}；"
1292 f"max_norm_rel={_format_report_number(scalar['max_norm_rel'])}；"
1293 f"max_post_clip_norm_abs={_format_report_number(scalar['max_post_clip_norm'])}；"
1294 f"max_post_clip_norm_rel={_format_report_number(scalar['max_post_clip_norm_rel'])}；"
1295 f"输入身份={identity_status}；LR 身份={lr_status}；有限性={finite_status}；"
1296 f"参数探针={parameter.get('status', 'n/a')} "
1297 f"(optimizer 数值={parameter.get('optimizer_state_numeric_comparison', 'n/a')}, "
1298 f"跳过状态数={parameter.get('skipped_optimizer_states', 0)})；"
1299 f"checkpoint layout={checkpoint_status}；总状态={case_comparison.get('status', 'n/a')}。"
1300 )
1301 return (
1302 f"[{category}] `{name}`: {topology}; tolerance_profile={profile} "
1303 f"(loss<={_format_report_number(threshold.get('loss_max_abs'))}, "
1304 f"loss_rel<={_format_report_number(threshold.get('loss_max_rel'))}, "
1305 f"norm<={_format_report_number(threshold.get('norm_max_abs'))}, "
1306 f"norm_rel<={_format_report_number(threshold.get('norm_max_rel'))}, "
1307 f"combination={threshold.get('combination', 'all')}); "
1308 f"compared_steps={scalar['steps']}; max_loss_abs={_format_report_number(scalar['max_loss'])}; "
1309 f"max_loss_rel={_format_report_number(scalar['max_loss_rel'])}; "
1310 f"max_norm_abs={_format_report_number(scalar['max_norm'])}; "
1311 f"max_norm_rel={_format_report_number(scalar['max_norm_rel'])}; "
1312 f"max_post_clip_norm_abs={_format_report_number(scalar['max_post_clip_norm'])}; "
1313 f"max_post_clip_norm_rel={_format_report_number(scalar['max_post_clip_norm_rel'])}; "
1314 f"input_identity={identity_status}; LR_identity={lr_status}; finite={finite_status}; "
1315 f"parameter_probes={parameter.get('status', 'n/a')} "
1316 f"(optimizer_numeric={parameter.get('optimizer_state_numeric_comparison', 'n/a')}, "
1317 f"skipped_states={parameter.get('skipped_optimizer_states', 0)}); "
1318 f"checkpoint_layout={checkpoint_status}; overall={case_comparison.get('status', 'n/a')}."
1319 )
1322def _precision_detail_entries(
1323 resolved_cases: Iterable[Mapping[str, Any]],
1324 comparison: Optional[Mapping[str, Any]],
1325 manifest: Mapping[str, Any],
1326 language: str,
1327) -> list[str]:
1328 """Render topology, thresholds, scalar gates, probes, and DCP per case."""
1329 comparisons = {} if comparison is None else comparison.get("comparisons", {})
1330 acceptance = manifest.get("acceptance", {})
1331 matrix = manifest.get("matrix", {})
1332 default_steps = matrix.get("steps", "n/a") if isinstance(matrix, Mapping) else "n/a"
1333 entries = []
1334 category_names = {
1335 "en": {
1336 "baseline": "baseline",
1337 "strategy": "strategy generalization",
1338 "recompute": "recompute",
1339 "resume": "resume",
1340 "production_validate": "Production/Validate",
1341 "other": "other",
1342 },
1343 "zh": {
1344 "baseline": "基线",
1345 "strategy": "策略泛化",
1346 "recompute": "重计算",
1347 "resume": "断点续训",
1348 "production_validate": "生产/Validate 对照",
1349 "other": "其他",
1350 },
1351 }
1352 for case in resolved_cases:
1353 name = str(case.get("name", "<unnamed>"))
1354 category = category_names[language][_case_category(case)]
1355 topology = _format_topology(case.get("topology", {}))
1356 if name == "baseline":
1357 entries.append(
1358 _baseline_detail_entry(category, name, topology, default_steps, language)
1359 )
1360 continue
1361 entries.append(
1362 _case_detail_entry(
1363 case,
1364 name,
1365 category,
1366 topology,
1367 comparisons,
1368 acceptance,
1369 language,
1370 )
1371 )
1372 return entries or (["未执行"] if language == "zh" else ["not executed"])
1375def _worst_case_entries(
1376 comparison: Optional[Mapping[str, Any]],
1377 language: str,
1378) -> list[str]:
1379 """Identify the worst observed scalar deltas with case and step."""
1380 worst_loss = (-1.0, "n/a", "n/a")
1381 worst_norm = (-1.0, "n/a", "n/a")
1382 for name, case in ({} if comparison is None else comparison.get("comparisons", {})).items():
1383 for row in case.get("scalar", {}).get("steps", ()):
1384 loss = float(row.get("loss_max_abs", 0.0))
1385 norm = float(row.get("norm_max_abs", 0.0))
1386 if loss > worst_loss[0]:
1387 worst_loss = (loss, name, row.get("step"))
1388 if norm > worst_norm[0]:
1389 worst_norm = (norm, name, row.get("step"))
1390 if worst_loss[0] < 0:
1391 return ["未测量"] if language == "zh" else ["not measured"]
1392 if language == "zh":
1393 return [
1394 f"全矩阵最大 loss 绝对误差={_format_report_number(worst_loss[0])}，"
1395 f"用例=`{worst_loss[1]}`，step={worst_loss[2]}",
1396 f"全矩阵最大梯度范数绝对误差={_format_report_number(worst_norm[0])}，"
1397 f"用例=`{worst_norm[1]}`，step={worst_norm[2]}",
1398 ]
1399 return [
1400 f"matrix maximum loss absolute error={_format_report_number(worst_loss[0])}, "
1401 f"case=`{worst_loss[1]}`, step={worst_loss[2]}",
1402 f"matrix maximum gradient-norm absolute error={_format_report_number(worst_norm[0])}, "
1403 f"case=`{worst_norm[1]}`, step={worst_norm[2]}",
1404 ]
1407def _detailed_parity_entries(
1408 parity: Optional[Mapping[str, Any]],
1409 language: str,
1410) -> list[str]:
1411 """Render each native module parity observation and compatibility note."""
1412 if parity is None:
1413 return ["未执行"] if language == "zh" else ["not executed"]
1414 entries = []
1415 for case in parity.get("cases", ()):
1416 case_name = case.get("case", "<unnamed>")
1417 entries.append(
1418 f"`{case_name}`：{case.get('status')}。" if language == "zh"
1419 else f"`{case_name}`: {case.get('status')}."
1420 )
1421 metric_rows = case.get("metrics", ())
1422 for metric in metric_rows:
1423 maximum = metric.get("max_abs")
1424 relative_l2 = metric.get("relative_l2")
1425 if language == "zh":
1426 entries.append(
1427 f"模块 `{metric.get('name', '<unnamed>')}`：{metric.get('status')}；"
1428 f"最大绝对误差={_format_report_number(maximum)}；"
1429 f"相对 L2 误差={_format_report_number(relative_l2)}。"
1430 )
1431 else:
1432 entries.append(
1433 f"module `{metric.get('name', '<unnamed>')}`: {metric.get('status')}; "
1434 f"maximum absolute error={_format_report_number(maximum)}; "
1435 f"relative L2 error={_format_report_number(relative_l2)}."
1436 )
1437 for observation in case.get("observations", ()):
1438 metrics = observation.get("metrics")
1439 errors = observation.get("max_abs_errors", {})
1440 maximum = (
1441 metrics.get("max_abs")
1442 if isinstance(metrics, Mapping)
1443 else max((float(value) for value in errors.values()), default=None)
1444 )
1445 relative_l2 = metrics.get("relative_l2") if isinstance(metrics, Mapping) else None
1446 if language == "zh":
1447 detail = (
1448 f"最大绝对误差={_format_report_number(maximum)}；"
1449 f"相对 L2 误差={_format_report_number(relative_l2)}"
1450 if maximum is not None
1451 else f"精确一致性={observation.get('status')}"
1452 )
1453 entries.append(
1454 f"模块 `{observation.get('name', '<unnamed>')}`：{observation.get('status')}；"
1455 f"{detail}。"
1456 )
1457 else:
1458 detail = (
1459 f"maximum absolute error={_format_report_number(maximum)}; "
1460 f"relative L2 error={_format_report_number(relative_l2)}"
1461 if maximum is not None
1462 else f"exact equality={observation.get('status')}"
1463 )
1464 entries.append(
1465 f"module `{observation.get('name', '<unnamed>')}`: {observation.get('status')}; "
1466 f"{detail}."
1467 )
1468 for note in case.get("compatibility", ()):
1469 prefix = "兼容性说明（原文）" if language == "zh" else "compatibility note"
1470 entries.append(f"{prefix}: {note}")
1471 return entries or (["无已执行用例"] if language == "zh" else ["no executed cases"])
1474def _detailed_performance_entries(
1475 comparison: Optional[Mapping[str, Any]],
1476 language: str,
1477) -> list[str]:
1478 """Render warm performance and memory without making them precision gates."""
1479 performance = {} if comparison is None else comparison.get("performance", {})
1480 entries = []
1481 for name, value in performance.items():
1482 allocated = value.get("peak_memory_allocated_bytes")
1483 reserved = value.get("peak_memory_reserved_bytes")
1484 allocated_mib = None if allocated is None else float(allocated) / (1024 ** 2)
1485 reserved_mib = None if reserved is None else float(reserved) / (1024 ** 2)
1486 if language == "zh":
1487 entries.append(
1488 f"`{name}`：稳态步数={value.get('steady_steps', 'n/a')}；"
1489 f"step p50={_format_report_number(value.get('step_time_p50_seconds'))}s；"
1490 f"p90={_format_report_number(value.get('step_time_p90_seconds'))}s；"
1491 f"tokens/s p50={_format_report_number(value.get('tokens_per_second_p50'))}；"
1492 f"samples/s p50={_format_report_number(value.get('samples_per_second_p50'))}；"
1493 f"峰值 allocated={_format_report_number(allocated_mib)} MiB；"
1494 f"reserved={_format_report_number(reserved_mib)} MiB。"
1495 )
1496 else:
1497 entries.append(
1498 f"`{name}`: steady_steps={value.get('steady_steps', 'n/a')}; "
1499 f"step p50={_format_report_number(value.get('step_time_p50_seconds'))}s; "
1500 f"p90={_format_report_number(value.get('step_time_p90_seconds'))}s; "
1501 f"tokens/s p50={_format_report_number(value.get('tokens_per_second_p50'))}; "
1502 f"samples/s p50={_format_report_number(value.get('samples_per_second_p50'))}; "
1503 f"peak allocated={_format_report_number(allocated_mib)} MiB; "
1504 f"reserved={_format_report_number(reserved_mib)} MiB."
1505 )
1506 return entries or (["未测量"] if language == "zh" else ["not measured"])
1509def _reproducibility_entries(
1510 store: EvidenceStore,
1511 environment: Optional[Mapping[str, Any]],
1512 manifest: Mapping[str, Any],
1513 language: str,
1514) -> list[str]:
1515 """Render revisions and import paths needed to replay the evidence."""
1516 environment = environment or {}
1517 torch_info = environment.get("torch") or {}
1518 transformers_info = environment.get("transformers") or {}
1519 framework_info = environment.get("hyper_parallel") or {}
1520 reference = manifest.get("reference", {})
1521 labels = {
1522 "en": ("resolved manifest", "environment", "repository revision", "framework import", "reference source"),
1523 "zh": ("已解析清单", "运行环境", "代码仓 revision", "框架导入路径", "参考源码"),
1524 }[language]
1525 return [
1526 f"{labels[0]}: `{store.path('manifest.resolved.yaml').name}`",
1527 f"{labels[1]}: `{store.path('environment.json').name}`",
1528 f"{labels[2]}: `{environment.get('git_revision', 'n/a')}`",
1529 f"{labels[3]}: `{framework_info.get('import_path', 'n/a')}`",
1530 f"PyTorch: `{torch_info.get('version', 'n/a')}`; Transformers: "
1531 f"`{transformers_info.get('version', 'n/a')}`",
1532 f"{labels[4]}: `{reference.get('source_path', 'n/a')}`",
1533 ]
1536def _structure_entries(
1537 findings: Optional[Mapping[str, Any]],
1538 language: str,
1539) -> list[str]:
1540 """Render structure status, inventory size, and actionable findings."""
1541 if findings is None:
1542 return ["未执行"] if language == "zh" else ["not executed"]
1543 metadata = findings.get("metadata", {})
1544 counts = metadata.get("inventory_counts", {})
1545 if language == "zh":
1546 entries = [
1547 f"状态={findings.get('status')}；model_type={metadata.get('model_type', 'n/a')}；"
1548 f"模块={counts.get('modules', 'n/a')}；参数={counts.get('parameters', 'n/a')}；"
1549 f"buffer={counts.get('buffers', 'n/a')}。"
1550 ]
1551 else:
1552 entries = [
1553 f"status={findings.get('status')}; model_type={metadata.get('model_type', 'n/a')}; "
1554 f"modules={counts.get('modules', 'n/a')}; parameters={counts.get('parameters', 'n/a')}; "
1555 f"buffers={counts.get('buffers', 'n/a')}."
1556 ]
1557 for finding in findings.get("findings", ()):
1558 entries.append(
1559 f"{finding.get('code')} {finding.get('severity')}: {finding.get('message')} "
1560 f"(fix: {finding.get('remediation') or 'n/a'})"
1561 )
1562 return entries
1565def _localized_value(value: Mapping[str, Any], key: str, language: str) -> Any:
1566 """Read a localized evidence field with an English fallback."""
1567 localized_key = f"{key}_zh" if language == "zh" else key
1568 return value.get(localized_key, value.get(key, "n/a"))
1571def _localized_sentence(value: Mapping[str, Any], key: str, language: str) -> str:
1572 """Return localized prose without duplicate terminal punctuation."""
1573 return str(_localized_value(value, key, language)).rstrip(".。")
1576def _verdict_entries(
1577 store: EvidenceStore,
1578 findings: Optional[Mapping[str, Any]],
1579 parity: Optional[Mapping[str, Any]],
1580 comparison: Optional[Mapping[str, Any]],
1581 checkpoint: Optional[Mapping[str, Any]],
1582 language: str,
1583) -> list[str]:
1584 """State exactly what the aggregate PASS covers and does not cover."""
1585 operator_audit = _read_json(store.path("analysis/operator_support.json")) or {}
1586 statuses = (
1587 (findings or {}).get("status", "NOT_EXECUTED"),
1588 (parity or {}).get("status", "NOT_EXECUTED"),
1589 (comparison or {}).get("status", "NOT_EXECUTED"),
1590 )
1591 strict_validate = operator_audit.get("strict_validate_status", "NOT_RECORDED")
1592 checkpoint_status = (checkpoint or {}).get("status", "NOT_EXECUTED")
1593 if language == "zh":
1594 return [
1595 f"已执行门禁：结构={statuses[0]}；原生模块对齐={statuses[1]}；"
1596 f"已声明精度矩阵={statuses[2]}。",
1597 "首行 PASS 只适用于 resolved cases 中实际执行的范围；未执行或明确排除的能力不会因该 PASS "
1598 f"自动视为已覆盖。严格 Validate={strict_validate}；预训练 checkpoint={checkpoint_status}。",
1599 ]
1600 return [
1601 f"executed gates: structure={statuses[0]}; native module parity={statuses[1]}; "
1602 f"declared precision matrix={statuses[2]}.",
1603 "The first-line PASS applies only to the executed resolved cases; omitted or explicitly excluded "
1604 f"capabilities are not covered by that PASS. strict Validate={strict_validate}; "
1605 f"pretrained checkpoint={checkpoint_status}.",
1606 ]
1609def _operator_support_entries(store: EvidenceStore, language: str) -> list[str]:
1610 """Render the exact-callable distributed-operator audit when recorded."""
1611 audit = _read_json(store.path("analysis/operator_support.json"))
1612 if audit is None:
1613 return ["未记录"] if language == "zh" else ["not recorded"]
1614 if language == "zh":
1615 entries = [
1616 f"审计状态={audit.get('status', 'n/a')}；"
1617 f"严格 Validate 状态={audit.get('strict_validate_status', 'n/a')}。"
1618 ]
1619 for operator in audit.get("operators", ()):
1620 consumer = _localized_sentence(operator, "consumer", language)
1621 reason = _localized_sentence(operator, "reason", language)
1622 entries.append(
1623 f"`{operator.get('callable', 'n/a')}`：状态={operator.get('status', 'n/a')}；"
1624 f"registry key=`{operator.get('registry_key', 'n/a')}`；"
1625 f"实现={operator.get('implementation', 'n/a')}；"
1626 f"注册={operator.get('registered', False)}；"
1627 f"使用方={consumer}；结论={reason}。"
1628 )
1629 return entries
1630 entries = [
1631 f"audit status={audit.get('status', 'n/a')}; "
1632 f"strict Validate status={audit.get('strict_validate_status', 'n/a')}."
1633 ]
1634 for operator in audit.get("operators", ()):
1635 consumer = _localized_sentence(operator, "consumer", language)
1636 reason = _localized_sentence(operator, "reason", language)
1637 entries.append(
1638 f"`{operator.get('callable', 'n/a')}`: status={operator.get('status', 'n/a')}; "
1639 f"registry_key=`{operator.get('registry_key', 'n/a')}`; "
1640 f"implementation={operator.get('implementation', 'n/a')}; "
1641 f"registered={operator.get('registered', False)}; "
1642 f"consumer={consumer}; finding={reason}."
1643 )
1644 return entries
1647def _parallel_ownership_entries(store: EvidenceStore, language: str) -> list[str]:
1648 """Render actual parameter placements instead of inferring them from launch size."""
1649 audit = _read_json(store.path("analysis/parallel_ownership.json"))
1650 if audit is None:
1651 return ["未记录"] if language == "zh" else ["not recorded"]
1652 if language == "zh":
1653 entries = [
1654 f"审计状态={audit.get('status', 'n/a')}；证据拓扑="
1655 f"{_format_topology(audit.get('topology', {}))}。"
1656 ]
1657 for subtree in audit.get("subtrees", ()):
1658 examples = "；".join(
1659 subtree.get("placement_evidence_zh", subtree.get("placement_evidence", ()))
1660 ) or "n/a"
1661 conclusion = _localized_sentence(subtree, "conclusion", language)
1662 entries.append(
1663 f"{_localized_value(subtree, 'name', language)}："
1664 f"TP 状态={subtree.get('tp_status', 'n/a')}；证据={examples}；"
1665 f"结论={conclusion}。"
1666 )
1667 conclusion = _localized_sentence(audit, "conclusion", language)
1668 entries.append(f"总体结论：{conclusion}。")
1669 return entries
1670 entries = [
1671 f"audit status={audit.get('status', 'n/a')}; evidence topology="
1672 f"{_format_topology(audit.get('topology', {}))}."
1673 ]
1674 for subtree in audit.get("subtrees", ()):
1675 examples = "; ".join(subtree.get("placement_evidence", ())) or "n/a"
1676 conclusion = _localized_sentence(subtree, "conclusion", language)
1677 entries.append(
1678 f"{_localized_value(subtree, 'name', language)}: "
1679 f"TP status={subtree.get('tp_status', 'n/a')}; evidence={examples}; "
1680 f"conclusion={conclusion}."
1681 )
1682 conclusion = _localized_sentence(audit, "conclusion", language)
1683 entries.append(f"overall conclusion: {conclusion}.")
1684 return entries
1687def _issue_ledger_entries(store: EvidenceStore, language: str) -> list[str]:
1688 """Render resolved and open integration issues with stable identifiers."""
1689 ledger = _read_json(store.path("analysis/issues.json"))
1690 if ledger is None:
1691 return ["未记录"] if language == "zh" else ["not recorded"]
1692 entries = []
1693 for issue in ledger.get("issues", ()):
1694 title = _localized_value(issue, "title", language)
1695 detail = _localized_value(issue, "detail", language)
1696 if language == "zh":
1697 entries.append(
1698 f"`{issue.get('id', 'n/a')}` [{issue.get('status', 'n/a')}/"
1699 f"{issue.get('severity', 'n/a')}] {title}：{detail}"
1700 )
1701 else:
1702 entries.append(
1703 f"`{issue.get('id', 'n/a')}` [{issue.get('status', 'n/a')}/"
1704 f"{issue.get('severity', 'n/a')}] {title}: {detail}"
1705 )
1706 return entries or (["无记录"] if language == "zh" else ["no entries"])
1709def _checkpoint_entries(
1710 checkpoint: Optional[Mapping[str, Any]],
1711 language: str,
1712) -> list[str]:
1713 """Explain whether checkpoint tensors were loaded or scratch-initialized."""
1714 if checkpoint is None:
1715 return ["未执行"] if language == "zh" else ["not executed"]
1716 if language == "zh":
1717 reason = checkpoint.get("reason", "n/a")
1718 if reason == "model was initialized from scratch or no load report was retained":
1719 reason = "模型从随机状态初始化，或没有保留权重加载报告"
1720 return [
1721 f"状态={checkpoint.get('status')}；loaded={checkpoint.get('loaded')}；"
1722 f"missing={checkpoint.get('missing')}；unexpected={checkpoint.get('unexpected')}；"
1723 f"原因={reason}。"
1724 ]
1725 return [
1726 f"status={checkpoint.get('status')}; loaded={checkpoint.get('loaded')}; "
1727 f"missing={checkpoint.get('missing')}; unexpected={checkpoint.get('unexpected')}; "
1728 f"reason={checkpoint.get('reason', 'n/a')}."
1729 ]
1732def _report_sections(
1733 store: EvidenceStore,
1734 findings: Optional[Mapping[str, Any]],
1735 parity: Optional[Mapping[str, Any]],
1736 comparison: Optional[Mapping[str, Any]],
1737 checkpoint: Optional[Mapping[str, Any]],
1738 environment: Optional[Mapping[str, Any]],
1739 language: str,
1740) -> list[tuple[str, list[str]]]:
1741 """Build the detailed English or Chinese report sections."""
1742 manifest = _report_manifest(store)
1743 cases = _resolved_report_cases(store)
1744 if language == "zh":
1745 headings = (
1746 "可复现性",
1747 "结论与适用范围",
1748 "结构验证",
1749 "Checkpoint 覆盖",
1750 "分布式算子审计",
1751 "真实并行归属",
1752 "问题台账",
1753 "原生模块精度对齐",
1754 "精度覆盖范围",
1755 "验收阈值",
1756 "精度矩阵详情",
1757 "全局最差误差",
1758 "性能与内存（不作为精度门禁）",
1759 "证据索引",
1760 )
1761 evidence_entries = [
1762 "完整机器可读比较：`comparison.json`",
1763 "展开后的用例：`cases/resolved_cases.json`",
1764 "分布式算子审计：`analysis/operator_support.json`",
1765 "并行归属证据：`analysis/parallel_ownership.json`",
1766 "问题台账：`analysis/issues.json`",
1767 "每个用例的日志、metrics、参数探针和 checkpoint 证据：`cases/<case>/`",
1768 ]
1769 else:
1770 headings = (
1771 "Reproducibility",
1772 "Verdict and scope",
1773 "Structure validation",
1774 "Checkpoint coverage",
1775 "Distributed operator audit",
1776 "Actual parallel ownership",
1777 "Issue ledger",
1778 "Native module parity",
1779 "Precision coverage",
1780 "Acceptance thresholds",
1781 "Precision matrix details",
1782 "Worst observed errors",
1783 "Performance and memory (non-gating)",
1784 "Evidence index",
1785 )
1786 evidence_entries = [
1787 "complete machine-readable comparison: `comparison.json`",
1788 "resolved cases: `cases/resolved_cases.json`",
1789 "distributed operator audit: `analysis/operator_support.json`",
1790 "parallel ownership evidence: `analysis/parallel_ownership.json`",
1791 "issue ledger: `analysis/issues.json`",
1792 "per-case logs, metrics, parameter probes, and checkpoint evidence: `cases/<case>/`",
1793 ]
1794 return [
1795 (headings[0], _reproducibility_entries(store, environment, manifest, language)),
1796 (
1797 headings[1],
1798 _verdict_entries(store, findings, parity, comparison, checkpoint, language),
1799 ),
1800 (headings[2], _structure_entries(findings, language)),
1801 (headings[3], _checkpoint_entries(checkpoint, language)),
1802 (headings[4], _operator_support_entries(store, language)),
1803 (headings[5], _parallel_ownership_entries(store, language)),
1804 (headings[6], _issue_ledger_entries(store, language)),
1805 (headings[7], _detailed_parity_entries(parity, language)),
1806 (headings[8], _precision_coverage_entries(store, language)),
1807 (headings[9], _acceptance_entries(manifest, language)),
1808 (headings[10], _precision_detail_entries(cases, comparison, manifest, language)),
1809 (headings[11], _worst_case_entries(comparison, language)),
1810 (headings[12], _detailed_performance_entries(comparison, language)),
1811 (headings[13], evidence_entries),
1812 ]
1815def command_report(args: argparse.Namespace) -> int:
1816 """Render detailed English and Chinese PASS/FAIL/BLOCKED summaries."""
1817 store = EvidenceStore(args.output_dir)
1818 findings = _read_json(store.path("check/findings.json"))
1819 parity = _read_json(store.path("module_parity/comparison.json"))
1820 comparison = _read_json(store.path("comparison.json"))
1821 checkpoint = _read_json(store.path("checkpoint/coverage.json"))
1822 environment = _read_json(store.path("environment.json"))
1823 required_reports = (findings, parity, comparison)
1824 status = _aggregate_statuses(
1825 [value.get("status") for value in required_reports if value is not None]
1826 + (["BLOCKED"] if any(value is None for value in required_reports) else [])
1827 )
1828 english_summary = store.render_summary(
1829 status,
1830 "Model integration validation",
1831 _report_sections(
1832 store,
1833 findings,
1834 parity,
1835 comparison,
1836 checkpoint,
1837 environment,
1838 "en",
1839 ),
1840 )
1841 chinese_summary = store.render_summary(
1842 status,
1843 "模型接入验证报告",
1844 _report_sections(
1845 store,
1846 findings,
1847 parity,
1848 comparison,
1849 checkpoint,
1850 environment,
1851 "zh",
1852 ),
1853 relative_path="summary.zh-CN.md",
1854 )
1855 print(f"{status}: {english_summary}; {chinese_summary}")
1856 return 0 if status == "PASS" else 1 if status == "FAIL" else 2
1859def _parser() -> argparse.ArgumentParser:
1860 """Build the command-line argument parser."""
1861 parser = argparse.ArgumentParser(
1862 prog="python -m hyper_parallel.tools.model_integration",
1863 description=__doc__,
1864 epilog="exit codes: 0=PASS, 1=FAIL or execution error, 2=BLOCKED or manifest error",
1865 )
1866 subparsers = parser.add_subparsers(dest="command", required=True)
1867 for name, handler, help_text in (
1868 ("inspect", command_inspect, "inspect source and final model structure"),
1869 ("scaffold", command_scaffold, "generate a minimal model adapter scaffold"),
1870 ("check", command_check, "validate the final built model"),
1871 ("parity", command_parity, "run model-owned module parity cases"),
1872 ("validate", command_validate, "run the precision validation matrix"),
1873 ):
1874 subparser = subparsers.add_parser(name, help=help_text)
1875 subparser.add_argument("--manifest", required=True)
1876 subparser.add_argument("--output-dir")
1877 subparser.set_defaults(handler=handler)
1878 if name == "scaffold":
1879 subparser.add_argument("--scaffold-dir", required=True)
1880 elif name == "parity":
1881 subparser.add_argument("--device", default="cpu")
1882 subparser.add_argument("--dtype", default="float32")
1883 elif name == "validate":
1884 subparser.add_argument("--generate-only", action="store_true")
1885 report_parser = subparsers.add_parser("report", help="render an evidence summary")
1886 report_parser.add_argument("--output-dir", required=True)
1887 report_parser.set_defaults(handler=command_report)
1888 return parser
1891def main(argv: Optional[list[str]] = None) -> int:
1892 """Execute the selected model-integration command."""
1893 args = _parser().parse_args(argv)
1894 try:
1895 return int(args.handler(args))
1896 except ManifestError as error:
1897 print(f"manifest error: {error}", file=sys.stderr)
1898 return 2
1901if __name__ == "__main__":
1902 raise SystemExit(main())