139 lines
7.2 KiB
Python
139 lines
7.2 KiB
Python
"""Reproducible native builds and a checked, model-specific executable cache."""
|
|
from __future__ import annotations
|
|
|
|
from dataclasses import dataclass
|
|
from hashlib import sha256
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import time
|
|
|
|
from .compiler import NativeProgram
|
|
|
|
ROOT = Path(__file__).resolve().parents[3]
|
|
NATIVE = ROOT / "native"
|
|
CACHE = ROOT / "app/data/native-builds"
|
|
LIBRARIES = ("cvode", "core", "nvecserial", "sunmatrixdense", "sunlinsoldense")
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class NativeBuild:
|
|
executable: Path
|
|
manifest: dict
|
|
cache_hit: bool
|
|
seconds: float
|
|
|
|
|
|
def _hash(path: Path) -> str:
|
|
return sha256(path.read_bytes()).hexdigest()
|
|
|
|
|
|
def toolchain() -> tuple[str, Path, str]:
|
|
if os.name != "nt" and not sys.platform.startswith("linux"):
|
|
raise RuntimeError("Native builds support Windows and Linux; this platform is not supported.")
|
|
compiler = os.environ.get("SIMULATION_NATIVE_CC") or shutil.which("gcc")
|
|
if not compiler:
|
|
raise RuntimeError("C compiler not found; set SIMULATION_NATIVE_CC to gcc.")
|
|
configured = os.environ.get("SUNDIALS_ROOT")
|
|
candidates = ([Path(configured)] if configured else
|
|
[Path(sys.base_prefix) / "Library"] if os.name == "nt" else
|
|
[Path(sys.prefix) / "native/sundials-7.4.0", Path("/usr/local"), Path("/usr")])
|
|
base = next((path for path in candidates if (path / "include/cvode/cvode.h").is_file()), candidates[0])
|
|
if not (base / "include/cvode/cvode.h").is_file():
|
|
raise RuntimeError("SUNDIALS C development files not found; set SUNDIALS_ROOT.")
|
|
version = subprocess.run([compiler, "--version"], capture_output=True, text=True, check=True, timeout=15).stdout.splitlines()[0]
|
|
return compiler, base, version
|
|
|
|
|
|
def build_native(program: NativeProgram, *, cache_dir: Path | None = None) -> NativeBuild:
|
|
start = time.perf_counter()
|
|
compiler, sundials, compiler_version = toolchain()
|
|
runtime = sorted(NATIVE.rglob("*.c")) + sorted((NATIVE / "include").glob("*.h"))
|
|
flags = ["-std=c11", "-O3", "-Wall", "-Wextra", "-Werror", "-ffp-contract=off", "-fno-fast-math"]
|
|
executable_name = "model.exe" if os.name == "nt" else "model"
|
|
if os.name == "nt":
|
|
flags += ["-D__USE_MINGW_ANSI_STDIO=1", "-static-libgcc"]
|
|
libraries = [sundials / "lib" / f"sundials_{name}.lib" for name in LIBRARIES]
|
|
dlls = [sundials / "bin" / f"sundials_{name}.dll" for name in LIBRARIES]
|
|
vc_runtime = sundials / "bin/vcruntime140.dll"
|
|
if vc_runtime.is_file():
|
|
dlls.append(vc_runtime)
|
|
elif sys.platform.startswith("linux"):
|
|
flags += ["-D_POSIX_C_SOURCE=200809L"]
|
|
library_dirs = [sundials / "lib", sundials / "lib64", sundials / "lib/x86_64-linux-gnu"]
|
|
library_dir = next((path for path in library_dirs if all(
|
|
(path / f"libsundials_{name}.a").is_file() for name in LIBRARIES)), None)
|
|
if library_dir is None:
|
|
raise RuntimeError("SUNDIALS static libraries not found; build with BUILD_STATIC_LIBS=ON and set SUNDIALS_ROOT.")
|
|
libraries = [library_dir / f"libsundials_{name}.a" for name in LIBRARIES]
|
|
dlls = []
|
|
else:
|
|
raise RuntimeError("Native builds support Windows and Linux; this platform is not supported.")
|
|
link_libraries = list(map(str, libraries))
|
|
if sys.platform.startswith("linux"):
|
|
link_libraries = ["-Wl,--start-group", *link_libraries, "-Wl,--end-group"]
|
|
sources = {str(p.relative_to(ROOT)): _hash(p) for p in runtime}
|
|
sources["native/THIRD_PARTY_NOTICES.txt"] = _hash(NATIVE / "THIRD_PARTY_NOTICES.txt")
|
|
dependencies = {str(p.name): _hash(p) for p in libraries + dlls}
|
|
# Header hashes include precision/index ABI settings as well as library APIs.
|
|
for directory in ("sundials", "cvode", "nvector", "sunmatrix", "sunlinsol"):
|
|
for path in sorted((sundials / "include" / directory).glob("*.h")):
|
|
dependencies[f"{directory}/{path.name}"] = _hash(path)
|
|
identity = dict(source=program.source, header=program.header, contract=program.manifest(), sources=sources,
|
|
dependencies=dependencies, compiler=compiler_version, flags=flags,
|
|
platform=sys.platform, abi=1)
|
|
signature = sha256(json.dumps(identity, sort_keys=True).encode()).hexdigest()
|
|
cache = (cache_dir or CACHE).resolve()
|
|
target = cache / signature
|
|
manifest_path = target / "manifest.json"
|
|
if manifest_path.is_file():
|
|
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
|
|
if all((target / name).is_file() and _hash(target / name) == digest
|
|
for name, digest in manifest["artifacts"].items()):
|
|
return NativeBuild(target / executable_name, manifest, True, time.perf_counter()-start)
|
|
raise RuntimeError(f"Native cache integrity check failed: {target}")
|
|
cache.mkdir(parents=True, exist_ok=True)
|
|
stage = Path(tempfile.mkdtemp(prefix="building-", dir=cache))
|
|
(stage / "model.c").write_text(program.source, encoding="utf-8")
|
|
(stage / "model.h").write_text(program.header, encoding="utf-8")
|
|
command = [compiler, *flags, "-I", str(stage), "-I", str(NATIVE / "include"),
|
|
"-I", str(sundials / "include"), str(stage / "model.c"),
|
|
*[str(p) for p in runtime if p.suffix == ".c"],
|
|
*link_libraries, "-lm", "-o", str(stage / executable_name)]
|
|
result = subprocess.run(command, capture_output=True, text=True, timeout=120)
|
|
(stage / "build.log").write_text(result.stdout + result.stderr, encoding="utf-8")
|
|
if result.returncode:
|
|
raise RuntimeError(f"Native compilation failed; see {stage / 'build.log'}: {result.stderr[-3000:]}")
|
|
for library in dlls:
|
|
shutil.copy2(library, stage / library.name)
|
|
shutil.copy2(NATIVE / "THIRD_PARTY_NOTICES.txt", stage / "THIRD_PARTY_NOTICES.txt")
|
|
manifest = {
|
|
**program.manifest(), "buildKey": signature, "compiler": compiler_version,
|
|
"compilerFlags": flags, "sourceHashes": sources, "dependencyHashes": dependencies,
|
|
"artifacts": {p.name: _hash(p) for p in stage.iterdir() if p.name != "build.log"},
|
|
}
|
|
(stage / "manifest.json").write_text(json.dumps(manifest, ensure_ascii=False, indent=2), encoding="utf-8")
|
|
try:
|
|
stage.rename(target)
|
|
except OSError:
|
|
# A concurrent compiler may have published the identical cache first.
|
|
if not manifest_path.is_file():
|
|
raise
|
|
published = json.loads(manifest_path.read_text(encoding="utf-8"))
|
|
# PE linker timestamps can differ between concurrent equivalent builds.
|
|
# Validate the winning build against its own hashes and our identity.
|
|
if published.get("buildKey") != signature or not all(
|
|
(target / name).is_file() and _hash(target / name) == digest
|
|
for name, digest in published["artifacts"].items()
|
|
):
|
|
raise RuntimeError("Concurrent native build did not produce the expected artifacts.")
|
|
manifest = published
|
|
if stage.resolve().parent != cache:
|
|
raise RuntimeError("Unexpected native build staging directory.")
|
|
shutil.rmtree(stage)
|
|
return NativeBuild(target / executable_name, manifest, False, time.perf_counter()-start)
|