Coverage for / home / jenkins / .local / lib / python3.10 / site-packages / hyper_parallel / codegen / emit / bundle.py: 66%
106 statements
« prev ^ index » next coverage.py v7.13.1, created at 2026-11-15 07:18 +0800
1# Copyright 2026 Huawei Technologies Co., Ltd
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14# ============================================================================
15"""Bundle emission: the entry the codegen manager calls to produce an artifact.
17``emit_bundle`` is what the manager calls: it is invoked as
18``emit_bundle(layout, meta)`` after the native plan has
19been derived and frozen into ``meta``. So this layer never derives anything —
20it renders the already-frozen contract into a ``files`` dict
21(filename -> content), and the manager stages + atomically swaps the whole
22bundle in one directory switch.
24The whole bundle is written as one atomic unit: ``emit_bundle``
25returns the modeling file, its diff, and ``__init__.py`` as a dict, and
26``write_bundle_atomic`` stages them in a temp sibling then
27swaps the directory in. A half-written bundle can never be imported.
29The in-memory context carries the frozen plan, resolved source, and parallel dimensions, so
30passing separate copies would create a second source of truth.
31"""
33from __future__ import annotations
35import logging
36import os
37import re
38from typing import Any
40from hyper_parallel.codegen.artifact import ArtifactLayout
41from hyper_parallel.codegen.emit.patches import emission_patches_from_meta
42from hyper_parallel.codegen.emit.recorded import materialize_recorded_forwards
43from hyper_parallel.codegen.emit.validation import verify_generated_source_structure
44from hyper_parallel.codegen.emit.diff import unified_diff
45from hyper_parallel.codegen.emit.modeling import (
46 copy_original_modeling,
47 emit_modeling_file,
48 verify_recorded_patch_bundle,
49)
50from hyper_parallel.codegen.inline.pipeline import render_inline_strategies_for_source_unit
51from hyper_parallel.codegen.source.resolver import resolve_remote_siblings
53logger = logging.getLogger(__name__)
55_NO_PATCH_DIFF = "# No codegen patches were required for this configuration.\n"
56_GENERATED_SOURCE_HEADER = (
57 "# GENERATED CODE. DO NOT EDIT MANUALLY.\n"
58 "# pylint: skip-file\n"
59 "# codespell:ignore-begin\n"
60)
61_GENERATED_SOURCE_FOOTER = "# codespell:ignore-end\n"
62_APACHE_LICENSE_HEADER = """# Copyright 2026 Huawei Technologies Co., Ltd
63#
64# Licensed under the Apache License, Version 2.0 (the "License");
65# you may not use this file except in compliance with the License.
66# You may obtain a copy of the License at
67#
68# http://www.apache.org/licenses/LICENSE-2.0
69#
70# Unless required by applicable law or agreed to in writing, software
71# distributed under the License is distributed on an "AS IS" BASIS,
72# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
73# See the License for the specific language governing permissions and
74# limitations under the License.
75# ============================================================================
76"""
77_COPIED_SOURCE_SPELLING_FIXES = {
78 "Ple" "se": "Please",
79 "suppo" "erted": "supported",
80}
83def emit_bundle(
84 layout: ArtifactLayout,
85 meta: Any,
86 *,
87 patch_plan: Any = None,
88) -> dict[str, str]:
89 """Render the artifact bundle from one frozen in-memory context.
91 Manager-facing entry point. Returns a ``filename -> content`` dict
92 holding the generated modeling file, the unified diff against the original
93 source, and ``__init__.py``; the caller (manager) stages and atomically
94 swaps the complete bundle directory.
96 """
97 if getattr(meta, "status", None) == "not_required":
98 logger.info("codegen: rendered audit-only bundle %s", layout.artifact_dir)
99 modeling_name = os.path.basename(layout.modeling_path)
100 return {
101 modeling_name: _prepare_generated_python(
102 copy_original_modeling(meta.source)
103 ),
104 os.path.basename(layout.diff_path): _NO_PATCH_DIFF,
105 "__init__.py": _strip_trailing_whitespace(emit_init_file(layout)),
106 }
107 if patch_plan is None:
108 patch_plan = emission_patches_from_meta(meta)
109 original_text = copy_original_modeling(meta.source)
110 generated_text = _prepare_generated_python(
111 emit_modeling_file(meta, patch_plan=patch_plan)
112 )
114 modeling_name = os.path.basename(layout.modeling_path)
115 diff_text = _strip_trailing_whitespace(
116 emit_diff(original_text, generated_text, layout)
117 )
119 logger.info(
120 "codegen: rendered bundle %s (modeling %s, %d plan boundaries)",
121 layout.artifact_dir,
122 modeling_name,
123 len(getattr(meta, "param_plan", None) or {}),
124 )
125 files = {
126 # Filename keys (not absolute paths): write_bundle_atomic stages them
127 # in a temp sibling and swaps the whole directory in.
128 modeling_name: generated_text,
129 os.path.basename(layout.diff_path): diff_text,
130 "__init__.py": _strip_trailing_whitespace(emit_init_file(layout)),
131 }
132 # A ``<remote>`` source is
133 # shipped inside the checkpoint and uses single-dot relative imports
134 # (``from .configuration_deepseek import ...``) that resolve against its
135 # own package. The loader compiles the generated file as the synthetic
136 # package ``hyper_parallel_generated.<digest>``, so those siblings must be
137 # IN the bundle and registered under that package. Copy the transitive
138 # closure of the source's relative imports here using their original
139 # basenames, so the generated bundle remains self-contained.
140 resolved = resolve_remote_siblings(meta.source)
141 if resolved:
142 siblings, sibling_diffs = _emit_remote_siblings(resolved, meta, patch_plan)
143 import_map = {
144 os.path.basename(path): name
145 for path, name in zip(resolved.values(), siblings)
146 }
147 files[modeling_name] = _rewrite_sibling_imports(
148 files[modeling_name], import_map
149 )
150 files[os.path.basename(layout.diff_path)] = _strip_trailing_whitespace(
151 emit_diff(original_text, files[modeling_name], layout)
152 )
153 files.update(siblings)
154 files.update(sibling_diffs)
155 verify_generated_source_structure(files)
156 verify_recorded_patch_bundle(files, patch_plan=patch_plan)
157 return files
160def _emit_remote_siblings(
161 resolved: dict[str, str],
162 meta: Any,
163 patch_plan: Any,
164) -> tuple[dict[str, str], dict[str, str]]:
165 """Materialize remote-code siblings and their visible diffs."""
166 siblings, originals, import_map = {}, {}, {}
167 for path in resolved.values():
168 basename = os.path.basename(path)
169 with open(path, "r", encoding="utf-8", errors="replace") as handle:
170 original = handle.read()
171 materialized = _strip_trailing_whitespace(
172 _materialize_sibling(original, meta, patch_plan, source_path=path)
173 )
174 rendered = _prepare_generated_python(materialized)
175 output_name = (
176 _generated_sibling_name(basename)
177 if materialized != _strip_trailing_whitespace(original)
178 else basename
179 )
180 import_map[basename] = output_name
181 siblings[output_name] = rendered
182 originals[output_name] = (basename, original)
183 siblings = {
184 name: _rewrite_sibling_imports(text, import_map)
185 for name, text in siblings.items()
186 }
187 diffs = _remote_sibling_diffs(siblings, originals)
188 return siblings, diffs
191def _remote_sibling_diffs(
192 siblings: dict[str, str],
193 originals: dict[str, tuple[str, str]],
194) -> dict[str, str]:
195 """Render diffs only for remote siblings whose source changed."""
196 diffs = {}
197 for output_name, rendered in siblings.items():
198 basename, original = originals[output_name]
199 if rendered == _strip_trailing_whitespace(original):
200 continue
201 diffs[f"{output_name}.diff"] = _strip_trailing_whitespace(
202 unified_diff(
203 _normalize_copied_source(original),
204 rendered,
205 fromfile=f"a/{basename}",
206 tofile=f"b/{output_name}",
207 )
208 )
209 return diffs
212def _prepare_generated_python(source_text: str) -> str:
213 """Mark copied source as generated and normalize comment-only URLs.
215 Generated bundles deliberately preserve complete upstream and remote-code
216 source units. Repository lint rules for hand-maintained code therefore do
217 not apply to those copied units. The markers are comments and do not alter
218 imports, executable statements, or runtime behavior.
219 """
220 text = _normalize_copied_source(source_text)
221 text = re.sub(
222 r"(^[ \t]*\#.*https?://\S*[A-Za-z0-9])$",
223 r"\1 (link).",
224 text,
225 flags=re.MULTILINE,
226 )
227 if text.startswith(_GENERATED_SOURCE_HEADER):
228 return text
229 return (
230 _GENERATED_SOURCE_HEADER
231 + text.rstrip()
232 + "\n"
233 + _GENERATED_SOURCE_FOOTER
234 )
237def _normalize_copied_source(source_text: str) -> str:
238 """Normalize repository-gate-only defects in copied upstream source."""
239 text = _strip_trailing_whitespace(source_text)
240 for typo, correction in _COPIED_SOURCE_SPELLING_FIXES.items():
241 text = text.replace(typo, correction)
242 return text
245def _materialize_sibling(
246 source_text: str,
247 meta: Any,
248 patch_plan: Any,
249 *,
250 source_path: str,
251) -> str:
252 """Apply source-owned strategy and recorded patches to one sibling."""
253 source = getattr(meta, "source", None)
254 architecture = (
255 source.get("architecture")
256 if isinstance(source, dict)
257 else getattr(source, "architecture", None)
258 )
259 rendered = render_inline_strategies_for_source_unit(
260 source_text,
261 meta,
262 model_type=architecture,
263 patch_plan=patch_plan,
264 )
265 return materialize_recorded_forwards(
266 rendered,
267 meta,
268 patch_plan=patch_plan,
269 source_path=source_path,
270 )
273def _generated_sibling_name(basename: str) -> str:
274 stem, extension = os.path.splitext(basename)
275 return f"{stem}_gen_npu{extension}"
278def _rewrite_sibling_imports(
279 source_text: str,
280 bundled_siblings: dict[str, str],
281) -> str:
282 """Retarget sibling imports to source units shipped in the bundle."""
283 rendered = source_text
284 for original, generated in sorted(bundled_siblings.items()):
285 original_module = os.path.splitext(original)[0]
286 generated_module = os.path.splitext(generated)[0]
287 rendered = re.sub(
288 rf"(?m)^(\s*from\s+\.+){re.escape(original_module)}(\s+import\s+)",
289 rf"\g<1>{generated_module}\g<2>",
290 rendered,
291 )
292 rendered = re.sub(
293 rf"(?m)^(\s*from\s+)"
294 rf"transformers_modules(?:\.[A-Za-z_][A-Za-z0-9_]*)+\."
295 rf"{re.escape(original_module)}(\s+import\s+)",
296 rf"\g<1>.{generated_module}\g<2>",
297 rendered,
298 )
299 return rendered
302def _strip_trailing_whitespace(text: str) -> str:
303 """Keep generated artifacts clean without changing their line structure."""
304 return "".join(f"{line.rstrip()}\n" for line in text.splitlines())
307def emit_init_file(layout: ArtifactLayout) -> str:
308 """Render the bundle's ``__init__.py`` text.
310 The bundle is an audit/export artifact, so this file carries no re-exports.
311 """
312 del layout
313 return (
314 _APACHE_LICENSE_HEADER
315 + '"""Generated codegen bundle — DO NOT EDIT MANUALLY.\n'
316 "\n"
317 "Regenerate with ``python -m hyper_parallel.codegen generate``; the\n"
318 "the bundle is rebuilt from the current training configuration.\n"
319 '"""\n'
320 )
323def emit_diff(original_text: str, generated_text: str, layout: ArtifactLayout) -> str:
324 """Render the unified diff from the original source to the generated file.
326 Returned rather than written so ``emit_bundle`` can write the whole bundle
327 in one atomic call.
328 """
329 return unified_diff(
330 _normalize_copied_source(original_text),
331 generated_text,
332 fromfile=f"a/{_source_basename(layout)}",
333 tofile=f"b/{os.path.basename(layout.modeling_path)}",
334 )
337def _source_basename(layout: ArtifactLayout) -> str:
338 """Diff ``fromfile`` label: the original modeling filename."""
339 name = os.path.basename(layout.modeling_path)
340 if name.endswith("_gen_npu.py"):
341 return name[: -len("_gen_npu.py")] + ".py"
342 return name
345__all__ = [
346 "emit_bundle",
347 "emit_diff",
348 "emit_init_file",
349 "emit_modeling_file",
350]