Coverage for  / home / jenkins / .local / lib / python3.10 / site-packages / hyper_multicore / _build / build_publication.py: 0%

1# Copyright 2026 Huawei Technologies Co., Ltd 

2# 

3# Licensed under the Apache License, Version 2.0 (the "License"); 

4# you may not use this file except in compliance with the License. 

5# You may obtain a copy of the License at 

6# 

7# http://www.apache.org/licenses/LICENSE-2.0 

8# 

9# Unless required by applicable law or agreed to in writing, software 

10# distributed under the License is distributed on an "AS IS" BASIS, 

11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 

12# See the License for the specific language governing permissions and 

13# limitations under the License. 

14# ============================================================================ 

15"""Validate staged artifacts and atomically expose complete filesystem outputs.""" 

16 

17from __future__ import annotations 

18import hashlib 

19import re 

20import base64 

21import csv 

22import ctypes 

23from email.parser import BytesParser 

24import io 

25import os 

26from pathlib import Path 

27import shutil 

28import tempfile 

29import zipfile 

30from packaging.requirements import Requirement 

31from .artifacts import _shmem_cpp_test_sources, _source_identity 

32from .command_runner import Commands 

33from ._bootstrap import load_module 

34 

35MANIFEST = load_module("native_manifest") 

36 

37 

38def publish_wheel(source: Path, destination: Path) -> None: 

39 """Expose a complete wheel atomically, preserving any prior successful artifact.""" 

40 with tempfile.NamedTemporaryFile( 

41 dir=destination.parent, prefix=f".{destination.name}.", suffix=".tmp", delete=False 

42 ) as file: 

43 temporary = Path(file.name) 

44 try: 

45 shutil.copy2(source, temporary) 

46 with temporary.open("rb") as file: 

47 os.fsync(file.fileno()) 

48 os.replace(temporary, destination) 

49 finally: 

50 temporary.unlink(missing_ok=True) 

51 

52 

53def exchange_directories(first: Path, second: Path) -> None: 

54 """Exchange Linux directory entries without a missing-payload interval.""" 

55 library = ctypes.CDLL(None, use_errno=True) 

56 rename = getattr(library, "renameat2", None) 

57 if rename is None: 

58 raise RuntimeError("Atomic native payload publication requires Linux renameat2") 

59 rename.argtypes = [ctypes.c_int, ctypes.c_char_p, ctypes.c_int, ctypes.c_char_p, ctypes.c_uint] 

60 rename.restype = ctypes.c_int 

61 if rename(-100, os.fsencode(first), -100, os.fsencode(second), 2) != 0: # AT_FDCWD, RENAME_EXCHANGE 

62 error = ctypes.get_errno() 

63 raise OSError(error, os.strerror(error), str(second)) 

64 

65 

66def publish_payload(candidate: Path, selected: Path) -> None: 

67 """Keep the visible payload complete even if interrupted immediately after publication.""" 

68 try: 

69 if selected.is_symlink(): 

70 raise ValueError(f"Refusing to replace an external payload symlink: {selected}") 

71 if selected.exists() and not selected.is_dir(): 

72 raise ValueError(f"Native payload destination is not a directory: {selected}") 

73 if selected.exists(): 

74 exchange_directories(candidate, selected) 

75 else: 

76 os.replace(candidate, selected) 

77 finally: 

78 if candidate.exists(): 

79 shutil.rmtree(candidate) 

80 

81 

82def validate_elf(path: Path, arch: str, run: Commands, env: dict) -> None: 

83 """Keep target architecture, RPATH, private dependencies and ELF hardening explicit.""" 

84 data = run(["readelf", "-hW", "-lW", "-dW", str(path)], label=f"elf-{path.name}", env=env, capture=True) 

85 expected = "AArch64" if arch == "aarch64" else "Advanced Micro Devices X86-64" 

86 if expected not in data or "GNU_RELRO" not in data or "BIND_NOW" not in data: 

87 raise ValueError(f"Missing architecture or ELF hardening: {path}") 

88 if re.search(r"GNU_STACK[^\n]*RWE", data): 

89 raise ValueError(f"Executable stack is forbidden: {path}") 

90 if "(RPATH)" in data or "(RUNPATH)" in data or "Shared library: [libcust_opapi.so]" in data: 

91 raise ValueError(f"Forbidden adapter runtime dependency/path: {path}") 

92 if "Shared library: [libshmem" in data: 

93 raise ValueError(f"Generic SHMEM dependency: {path}") 

94 linked = run(["ldd", str(path)], label=f"dependencies-{path.name}", env=env, capture=True) 

95 missing = [line.strip().split()[0] for line in linked.splitlines() if "not found" in line] 

96 if any(not name.startswith(("libtorch", "libc10")) for name in missing): 

97 raise ValueError(f"Missing adapter dependencies: {missing}") 

98 

99 

100def validate_wheel(wheel: Path, payload: Path, info: dict) -> None: 

101 """Verify archive membership, RECORD, identities and native payload before publication.""" 

102 with zipfile.ZipFile(wheel) as archive: 

103 names, record_path, metadata_path = wheel_identity_members(archive) 

104 validate_wheel_record(archive, record_path, names) 

105 validate_wheel_metadata(archive, metadata_path, info) 

106 validate_wheel_payload(archive, names, payload, info) 

107 

108 

109def wheel_identity_members(archive): 

110 """Require unique safe entries and exactly one distribution identity.""" 

111 names = archive.namelist() 

112 if len(names) != len(set(names)) or any(Path(n).is_absolute() or ".." in Path(n).parts for n in names): 

113 raise ValueError("Wheel contains duplicate or unsafe entries") 

114 records = [n for n in names if len(Path(n).parts) == 2 and n.endswith(".dist-info/RECORD")] 

115 metas = [n for n in names if len(Path(n).parts) == 2 and n.endswith(".dist-info/METADATA")] 

116 if len(records) != 1 or len(metas) != 1: 

117 raise ValueError("Wheel requires unique RECORD and METADATA") 

118 return names, records[0], metas[0] 

119 

120 

121def validate_wheel_record(archive, record_path, names): 

122 """Require RECORD to cover every archive member with its correct size and digest.""" 

123 entries = list(csv.reader(io.StringIO(archive.read(record_path).decode()))) 

124 if len(entries) != len({row[0] for row in entries}) or {row[0] for row in entries} != set(names): 

125 raise ValueError("Wheel RECORD membership mismatch") 

126 for name, checksum, size in entries: 

127 if name == record_path: 

128 continue 

129 contents = archive.read(name) 

130 expected = "sha256=" + base64.urlsafe_b64encode(hashlib.sha256(contents).digest()).rstrip(b"=").decode() 

131 if checksum != expected or size != str(len(contents)): 

132 raise ValueError(f"Wheel RECORD mismatch: {name}") 

133 

134 

135def validate_wheel_metadata(archive, metadata_path, info): 

136 """Verify the component identity and unconstrained framework dependency.""" 

137 metadata = BytesParser().parsebytes(archive.read(metadata_path)) 

138 if metadata["Name"].replace("_", "-") != "hyper-multicore" or metadata["Version"] != info["artifact_version"]: 

139 raise ValueError("Wheel distribution identity mismatch") 

140 torch = [Requirement(v) for v in metadata.get_all("Requires-Dist", []) if Requirement(v).name.lower() == "torch"] 

141 if len(torch) != 1 or str(torch[0].specifier) or torch[0].url: 

142 raise ValueError("Wheel must declare Torch without a version constraint or direct URL") 

143 

144 

145def validate_wheel_payload(archive, names, payload, info): 

146 """Check complete native membership, resource hashes and assembled identity bytes.""" 

147 expected = {"hyper_multicore/" + n for n in info["payload_sha256"]} 

148 actual = { 

149 n 

150 for n in names 

151 if any(n.startswith("hyper_multicore/" + prefix + "/") for prefix in ("lib", "shmem/lib", "adapters")) 

152 } 

153 if actual != expected: 

154 raise ValueError("Wheel native payload membership mismatch") 

155 for name, checksum in info["payload_sha256"].items(): 

156 if hashlib.sha256(archive.read("hyper_multicore/" + name)).hexdigest() != checksum: 

157 raise ValueError(f"Wheel payload mismatch: {name}") 

158 for name in ("build_info.json", "adapters/manifest.json"): 

159 if archive.read("hyper_multicore/" + name) != (payload / name).read_bytes(): 

160 raise ValueError(f"Wheel identity mismatch: {name}") 

161 

162 

163class SourceGuard: 

164 """Reject mixed repository sources before and after formal publication.""" 

165 

166 def __init__(self, root: Path): 

167 self.root = root 

168 self.frozen = self.observe() 

169 

170 def observe(self): 

171 """Record content and file generations, including edit-and-restore changes.""" 

172 paths = [] 

173 for name in ("hyper_multicore", "scripts"): 

174 paths.extend(p for p in (self.root / name).rglob("*") 

175 if p.is_file() and not {"lib", "__pycache__"}.intersection(p.relative_to(self.root).parts)) 

176 paths.extend(self.root / name for name in 

177 ("setup.py", "MANIFEST.in", "build.sh", "requirements.txt")) 

178 paths.extend(_shmem_cpp_test_sources(self.root)) 

179 generations = {} 

180 for path in paths: 

181 if path.is_file(): 

182 stat = path.stat() 

183 generations[str(path)] = (stat.st_ino, stat.st_size, stat.st_mtime_ns, stat.st_ctime_ns) 

184 return _source_identity(self.root), generations 

185 

186 def check_source(self): 

187 if self.observe() != self.frozen: 

188 raise RuntimeError("Source inputs changed during build; refusing mixed-source payload")