Coverage for / home / jenkins / .local / lib / python3.10 / site-packages / hyper_multicore / _build / prepare_dependencies.py: 0%
130 statements
« prev ^ index » next coverage.py v7.13.1, created at 2026-11-15 05:03 +0800
1# Copyright 2026 Huawei Technologies Co., Ltd.
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14# ============================================================================
15"""Explicitly acquire or verify pinned native build dependencies."""
17from __future__ import annotations
19import argparse
20import hashlib
21import json
22from pathlib import Path
23import subprocess
24import sys
25from typing import Any
28_REPO_ROOT = Path(__file__).resolve().parents[2]
29_LOCK_PATH = Path(__file__).with_name("dependencies.lock.json")
30_DEFAULT_CACHE = _REPO_ROOT / "build" / "native" / "deps"
33def _parse_args(dependencies: dict[str, Any]) -> argparse.Namespace:
34 """Parse the explicit dependency acquisition contract."""
35 parser = argparse.ArgumentParser(description=__doc__)
36 parser.add_argument(
37 "--dependency",
38 action="append",
39 required=True,
40 choices=tuple(dependencies),
41 help="Dependency to prepare. Repeat the option to prepare several dependencies.",
42 )
43 parser.add_argument("--cache-dir", default=str(_DEFAULT_CACHE))
44 parser.add_argument("--source-dir")
45 parser.add_argument("--verify-only", action="store_true")
46 return parser.parse_args()
49def main() -> int:
50 """Acquire requested dependencies, then verify their locked identities."""
51 dependencies = load_dependency_locks()
52 args = _parse_args(dependencies)
53 cache_dir = Path(args.cache_dir)
54 if not cache_dir.is_absolute():
55 cache_dir = _REPO_ROOT / cache_dir
56 if args.source_dir and len(args.dependency) != 1:
57 raise ValueError("--source-dir can only be used with one --dependency value.")
58 prepared: dict[str, Any] = {}
59 for dependency in dict.fromkeys(args.dependency):
60 source_dir = Path(args.source_dir) if args.source_dir else cache_dir / dependency / "src"
61 if not source_dir.is_absolute():
62 source_dir = _REPO_ROOT / source_dir
63 dependency_lock = dependencies[dependency]
64 revisions = dependency_revisions(dependency_lock)
65 external_source = bool(args.source_dir and source_dir.exists())
66 if not source_dir.exists():
67 if args.verify_only:
68 raise ValueError(f"Dependency source directory does not exist: {source_dir}")
69 _clone_dependency(next(iter(revisions.values())), source_dir)
70 verified = {}
71 for revision_name, revision in revisions.items():
72 _ensure_commit(revision, source_dir, allow_fetch=not (args.verify_only or external_source))
73 verified[revision_name] = verify_git_dependency(
74 revision, source_dir, dependency_name=f"{dependency}.{revision_name}"
75 if "revisions" in dependency_lock else dependency,
76 )
77 prepared[dependency] = (
78 {"repository": dependency_lock["repository"], "source_dir": str(source_dir.resolve()),
79 "revisions": verified} if "revisions" in dependency_lock else verified["default"]
80 )
81 print(json.dumps(prepared, sort_keys=True))
82 return 0
85def verify_git_dependency(
86 dependency_lock: dict[str, Any],
87 source_dir: str | Path,
88 dependency_name: str | None = None,
89) -> dict[str, Any]:
90 """Verify a locked commit independently of the shared worktree's current HEAD.
92 Args:
93 dependency_lock: Dependency entry from ``dependencies.lock.json``.
94 source_dir: Existing Git worktree to verify.
95 dependency_name: Stable lock name; inferred from the repository when omitted.
97 Returns:
98 Verified source identity suitable for a run record.
100 Raises:
101 ValueError: If the source is missing, modified, or does not match the lock.
102 """
103 source_dir = Path(source_dir).resolve()
104 if not (source_dir / ".git").exists():
105 raise ValueError(f"Dependency source is not a Git worktree: {source_dir}")
106 commit = dependency_lock["commit"]
107 actual_commit = _git_output(source_dir, "rev-parse", f"{commit}^{{commit}}")
108 actual_tree = _git_output(source_dir, "rev-parse", f"{commit}^{{tree}}")
109 tracked_changes = _git_output(source_dir, "status", "--porcelain", "--untracked-files=no")
110 if actual_commit != dependency_lock["commit"]:
111 raise ValueError(f"Dependency commit mismatch: expected={dependency_lock['commit']}, actual={actual_commit}")
112 if actual_tree != dependency_lock["tree"]:
113 raise ValueError(f"Dependency tree mismatch: expected={dependency_lock['tree']}, actual={actual_tree}")
114 if tracked_changes:
115 raise ValueError("Dependency source contains tracked modifications; upstream patching is not allowed.")
116 repository_name = Path(dependency_lock["repository"]).stem
117 archive_hash = _git_archive_sha256(
118 source_dir,
119 dependency_lock["commit"],
120 f"{repository_name}-{dependency_lock['version']}/",
121 )
122 expected_archive_hashes = {
123 dependency_lock["git_archive_tar_sha256"],
124 *dependency_lock.get("git_archive_tar_sha256_compat", []),
125 }
126 if archive_hash not in expected_archive_hashes:
127 raise ValueError(
128 "Dependency archive hash mismatch: "
129 f"expected one of {sorted(expected_archive_hashes)}, actual={archive_hash}"
130 )
131 return {
132 "dependency": dependency_name or repository_name.replace("-", "_"),
133 "version": dependency_lock["version"],
134 "repository": dependency_lock["repository"],
135 "source_dir": str(source_dir),
136 "commit": actual_commit,
137 "tree": actual_tree,
138 "git_archive_tar_sha256": archive_hash,
139 "patched": False,
140 }
143def load_dependency_locks(lock_path: Path | None = None) -> dict[str, Any]:
144 """Read schema 2 and reject repeated repository declarations before acquisition."""
145 lock = json.loads((lock_path or _LOCK_PATH).read_text(encoding="utf-8"))
146 if lock.get("schema_version") != 2:
147 raise ValueError("Native dependency lock must use schema_version 2")
148 dependencies = lock["components"]["multicore"]
149 repositories = {}
150 for name, dependency in dependencies.items():
151 repository = dependency["repository"]
152 if repository in repositories:
153 raise ValueError(
154 f"Repository {repository} is declared by both {repositories[repository]} and {name}; "
155 "declare additional versions under revisions instead."
156 )
157 repositories[repository] = name
158 dependency_revisions(dependency)
159 return dependencies
162def dependency_revisions(dependency_lock: dict[str, Any]) -> dict[str, dict[str, Any]]:
163 """Resolve single-version or named revisions into complete verification entries."""
164 if "revisions" not in dependency_lock:
165 return {"default": dependency_lock}
166 revisions = dependency_lock["revisions"]
167 if not isinstance(revisions, dict) or not revisions:
168 raise ValueError("Dependency revisions must be a nonempty mapping")
169 if any(not isinstance(revision, dict) or "repository" in revision for revision in revisions.values()):
170 raise ValueError("Each revision must be an object; repository belongs to the parent entry")
171 return {name: {**revision, "repository": dependency_lock["repository"]}
172 for name, revision in revisions.items()}
175def _clone_dependency(dependency_lock: dict[str, Any], source_dir: Path) -> None:
176 """Initialize one ordinary repository, then check out its first locked revision."""
177 source_dir.mkdir(parents=True)
178 _run_git(source_dir, "init")
179 _run_git(source_dir, "remote", "add", "origin", dependency_lock["repository"])
180 _ensure_commit(dependency_lock, source_dir, allow_fetch=True)
181 _run_git(source_dir, "checkout", "--detach", dependency_lock["commit"])
184def _ensure_commit(dependency_lock: dict[str, Any], source_dir: Path, allow_fetch: bool) -> None:
185 """Fetch only missing commits and retain them without switching the shared HEAD."""
186 if not (source_dir / ".git").exists():
187 raise ValueError(f"Dependency source is not a Git worktree: {source_dir}")
188 commit = dependency_lock["commit"]
189 result = subprocess.run(["git", "cat-file", "-e", f"{commit}^{{commit}}"], cwd=source_dir,
190 check=False, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
191 if result.returncode != 0:
192 if not allow_fetch:
193 raise ValueError(f"Locked commit {commit} is missing from {source_dir}")
194 _run_git(source_dir, "fetch", "--depth", "1", "origin",
195 dependency_lock.get("fetch_ref") or dependency_lock["version"])
196 if allow_fetch:
197 _run_git(source_dir, "update-ref", f"refs/hyper_multicore/{commit}", commit)
200def _run_git(source_dir: Path, *args: str) -> None:
201 """Execute acquisition commands with diagnostics on stderr, leaving stdout as JSON."""
202 result = subprocess.run(["git", *args], cwd=source_dir, check=False, stdout=sys.stderr)
203 if result.returncode != 0:
204 raise RuntimeError(f"Git dependency command failed ({' '.join(args)}) in {source_dir}")
207def _git_output(source_dir: Path, *args: str) -> str:
208 """Run one read-only Git inspection and return stripped stdout."""
209 result = subprocess.run(
210 ["git", *args],
211 cwd=source_dir,
212 check=False,
213 stdout=subprocess.PIPE,
214 stderr=subprocess.PIPE,
215 text=True,
216 )
217 if result.returncode != 0:
218 raise ValueError(f"Git inspection failed ({' '.join(args)}): {result.stderr.strip()}")
219 return result.stdout.strip()
222def _git_archive_sha256(source_dir: Path, commit: str, archive_prefix: str) -> str:
223 """Hash a deterministic uncompressed Git archive without creating a tarball."""
224 with subprocess.Popen(
225 ["git", "archive", "--format=tar", f"--prefix={archive_prefix}", commit],
226 cwd=source_dir,
227 stdout=subprocess.PIPE,
228 stderr=subprocess.PIPE,
229 ) as process:
230 if process.stdout is None:
231 raise RuntimeError("Cannot read git archive output.")
232 digest = hashlib.sha256()
233 for chunk in iter(lambda: process.stdout.read(1024 * 1024), b""):
234 digest.update(chunk)
235 _, stderr = process.communicate()
236 if process.returncode != 0:
237 raise ValueError(f"Git archive failed: {stderr.decode(errors='replace').strip()}")
238 return digest.hexdigest()
241if __name__ == "__main__":
242 try:
243 raise SystemExit(main())
244 except (RuntimeError, ValueError) as error:
245 sys.stderr.write(f"[HP-NATIVE-DEPENDENCY-ERROR] {error}\n")
246 raise SystemExit(1) from error