C内核按库功能拆解,编译结果缓存区构建,编译过程与已有缓存结果对照功能实现

This commit is contained in:
lujingze committed 2026-09-12 05:24:48 +00:00
1 parent aa4951b14e
commit 151e6e4b97
30 files changed
+4849 -597

No files matched your search

+7 -2
View File
@@ -69,6 +69,11 @@ SimulationProgressEmitter = Callable[
] ]
SIMULATION_PHASE_MESSAGES = { SIMULATION_PHASE_MESSAGES = {
"initializing": "正在初始化状态与方程", "initializing": "正在初始化状态与方程",
"native-generation": "正在生成原生模型代码",
"native-cache-check": "正在检查已编译模型缓存",
"native-cache-hit": "已复用模型缓存,正在启动求解器",
"native-compilation": "正在编译模型与缺失的原生模块",
"native-linking": "正在链接模型可执行文件",
"integrating": "正在进行时间积分与压力流量求解", "integrating": "正在进行时间积分与压力流量求解",
"postprocessing": "正在整理采样结果", "postprocessing": "正在整理采样结果",
"cancelled": "正在整理已终止仿真的部分结果", "cancelled": "正在整理已终止仿真的部分结果",
@@ -746,9 +751,9 @@ def _run_system_xml_simulation_profiled(
emit(0, "validation", "正在校验 System XML") emit(0, "validation", "正在校验 System XML")
report = validate_system_xml_document(xml_bytes) report = validate_system_xml_document(xml_bytes)
document = _validated_xml_document_or_422(report) document = _validated_xml_document_or_422(report)
emit(0, "compilation", "正在编译组件与连接关系") emit(0, "compilation", "正在解析组件与连接关系")
network = _compile_xml_document_or_422(document) network = _compile_xml_document_or_422(document)
emit(0, "initialization", "模型编译完成,正在准备求解器") emit(0, "initialization", "连接关系已解析,正在准备原生模型")
def report_system_progress(progress: float, phase: str) -> None: def report_system_progress(progress: float, phase: str) -> None:
bounded_progress = min(1.0, max(0.0, progress)) bounded_progress = min(1.0, max(0.0, progress))
+287 -76
View File
@@ -1,23 +1,31 @@
"""Reproducible native builds and a checked, model-specific executable cache.""" """On-demand native modules, checked object reuse and bounded executable caches."""
from __future__ import annotations from __future__ import annotations
from dataclasses import dataclass from concurrent.futures import ThreadPoolExecutor
from dataclasses import dataclass, field
from hashlib import sha256 from hashlib import sha256
import json import json
import os import os
from pathlib import Path from pathlib import Path
import re
import shutil import shutil
import subprocess import subprocess
import sys import sys
import tempfile import tempfile
import time import time
from typing import Any, Callable
from .compiler import NativeProgram from .compiler import NativeProgram
from .modules import component_modules
from .cache_storage import acquire_cache_lease, touch_cache_entry, prune_cache
ROOT = Path(__file__).resolve().parents[3] ROOT = Path(__file__).resolve().parents[3]
NATIVE = ROOT / "native" NATIVE = ROOT / "native"
CACHE = ROOT / "app/data/native-builds" CACHE = ROOT / "app/data/native-builds"
LIBRARIES = ("cvode", "core", "nvecserial", "sunmatrixdense", "sunlinsoldense") LIBRARIES = ("cvode", "core", "nvecserial", "sunmatrixdense", "sunlinsoldense")
COMPILER_FLAGS = ("-std=c11", "-O3", "-Wall", "-Wextra", "-Werror", "-ffp-contract=off", "-fno-fast-math")
RUNTIME_SOURCES = ("runtime/main.c", "runtime/common.c", "runtime/rk45.c",
"runtime/cvode_solver.c", "runtime/json_numbers.c", "encoding/ryu/d2s.c")
@dataclass(frozen=True) @dataclass(frozen=True)
@@ -26,12 +34,115 @@ class NativeBuild:
manifest: dict manifest: dict
cache_hit: bool cache_hit: bool
seconds: float seconds: float
details: dict = field(default_factory=dict)
_lease: Any = field(default=None, repr=False, compare=False)
_cache: Path | None = field(default=None, repr=False, compare=False)
def close(self) -> None:
"""Release this caller's executable pin after its last use."""
if self._lease is not None:
self._lease.close()
if self._cache is not None:
_prune(self._cache)
def _hash(path: Path) -> str: def _hash(path: Path) -> str:
return sha256(path.read_bytes()).hexdigest() return sha256(path.read_bytes()).hexdigest()
def _key(value: object) -> str:
return sha256(json.dumps(value, sort_keys=True, separators=(",", ":")).encode()).hexdigest()
def _runtime_sources(program: NativeProgram) -> list[Path]:
return [NATIVE / name for name in RUNTIME_SOURCES] + [
NATIVE / "components/modules" / f"{name}.c"
for name in component_modules(program.source + "\n" + program.header)
]
def _budget(name: str, default: int) -> int:
value = os.environ.get(name, str(default))
if not re.fullmatch(r"[0-9]+", value):
raise ValueError(f"{name} must be a nonnegative integer in MiB.")
return int(value) * 1024**2
def _prune(cache: Path) -> dict:
return prune_cache(cache,
model_limit_bytes=_budget("SIMULATION_NATIVE_MODEL_CACHE_MB", 256),
object_limit_bytes=_budget("SIMULATION_NATIVE_OBJECT_CACHE_MB", 128))
def _checked_manifest(target: Path, key_name: str, key: str, version: int, required: set[str]) -> dict | None:
if not target.exists() and not target.is_symlink():
return None
message = f"Native cache integrity check failed: {target}"
if target.is_symlink() or not target.is_dir():
raise RuntimeError(message)
try:
path = target / "manifest.json"
if path.is_symlink():
raise ValueError("linked manifest")
manifest = json.loads(path.read_text(encoding="utf-8"))
if not isinstance(manifest, dict) or manifest.get(key_name) != key or manifest.get("cacheVersion") != version:
raise ValueError("identity mismatch")
artifacts = manifest.get("artifacts")
if not isinstance(artifacts, dict) or set(artifacts) != required:
raise ValueError("incomplete artifacts")
if key_name == "objectKey":
recipe = {name: manifest[name] for name in ("cacheVersion", "sourceName", "preprocessedSha256", "compiler")}
if _key(recipe) != key:
raise ValueError("object identity mismatch")
else:
recipe = manifest.get("buildIdentity")
if not isinstance(recipe, dict) or _key(recipe) != key or manifest.get("objectKeys") != recipe["objectKeys"]:
raise ValueError("model identity mismatch")
if artifacts["model.c"] != recipe["sourceSha256"] or artifacts["model.h"] != recipe["headerSha256"]:
raise ValueError("model source identity mismatch")
if _key({name: manifest[name] for name in recipe["contractKeys"]}) != recipe["contractSha256"]:
raise ValueError("model contract mismatch")
if artifacts["THIRD_PARTY_NOTICES.txt"] != recipe["notice"]:
raise ValueError("license artifact mismatch")
for name in required:
if name.lower().endswith(".dll") and artifacts[name] != recipe["dependencies"][name]:
raise ValueError("runtime dependency artifact mismatch")
for name, digest in artifacts.items():
if not re.fullmatch(r"[A-Za-z0-9_.-]+", name) or name in (".", ".."):
raise ValueError("invalid artifact name")
path = target / name
if path.is_symlink() or not path.is_file() or not isinstance(digest, str) or _hash(path) != digest:
raise ValueError("artifact mismatch")
return manifest
except (OSError, ValueError, TypeError, KeyError) as exc:
raise RuntimeError(message) from exc
def _publish(stage: Path, target: Path, key_name: str, key: str, version: int, required: set[str]) -> dict:
try:
stage.rename(target)
except OSError:
# Another reader/build may publish the same content while we compile.
# Its artifacts need not be byte-identical (e.g. PE timestamps), but
# must satisfy the same complete identity and its own recorded hashes.
winner = _checked_manifest(target, key_name, key, version, required)
if winner is None:
raise
shutil.rmtree(stage)
return winner
manifest = _checked_manifest(target, key_name, key, version, required)
assert manifest is not None
return manifest
def _command(command: list[str], *, log: list[str], timeout: float = 120) -> bytes:
result = subprocess.run(command, capture_output=True, timeout=timeout)
log.append(" ".join(command) + "\n" + result.stderr.decode("utf-8", errors="replace"))
if result.returncode:
raise RuntimeError("Native compilation failed: " + result.stderr.decode("utf-8", errors="replace")[-3000:])
return result.stdout
def toolchain() -> tuple[str, Path, str]: def toolchain() -> tuple[str, Path, str]:
if os.name != "nt" and not sys.platform.startswith("linux"): if os.name != "nt" and not sys.platform.startswith("linux"):
raise RuntimeError("Native builds support Windows and Linux; this platform is not supported.") raise RuntimeError("Native builds support Windows and Linux; this platform is not supported.")
@@ -49,90 +160,190 @@ def toolchain() -> tuple[str, Path, str]:
return compiler, base, version return compiler, base, version
def build_native(program: NativeProgram, *, cache_dir: Path | None = None) -> NativeBuild: def build_native(program: NativeProgram, *, cache_dir: Path | None = None,
start = time.perf_counter() progress_callback: Callable[[str], None] | None = None) -> NativeBuild:
started = time.perf_counter()
if progress_callback:
progress_callback("native-cache-check")
compiler, sundials, compiler_version = toolchain() compiler, sundials, compiler_version = toolchain()
runtime = sorted(NATIVE.rglob("*.c")) + sorted((NATIVE / "include").rglob("*.h")) flags = list(COMPILER_FLAGS)
flags = ["-std=c11", "-O3", "-Wall", "-Wextra", "-Werror", "-ffp-contract=off", "-fno-fast-math"]
executable_name = "model.exe" if os.name == "nt" else "model" executable_name = "model.exe" if os.name == "nt" else "model"
if os.name == "nt": if os.name == "nt":
flags += ["-D__USE_MINGW_ANSI_STDIO=1", "-static-libgcc"] flags += ["-D__USE_MINGW_ANSI_STDIO=1", "-static-libgcc"]
libraries = [sundials / "lib" / f"sundials_{name}.lib" for name in LIBRARIES] libraries = [sundials / "lib" / f"sundials_{name}.lib" for name in LIBRARIES]
dlls = [sundials / "bin" / f"sundials_{name}.dll" for name in LIBRARIES] dlls = [sundials / "bin" / f"sundials_{name}.dll" for name in LIBRARIES]
vc_runtime = sundials / "bin/vcruntime140.dll" if (sundials / "bin/vcruntime140.dll").is_file():
if vc_runtime.is_file(): dlls.append(sundials / "bin/vcruntime140.dll")
dlls.append(vc_runtime)
elif sys.platform.startswith("linux"):
flags += ["-D_POSIX_C_SOURCE=200809L"]
library_dirs = [sundials / "lib", sundials / "lib64", sundials / "lib/x86_64-linux-gnu"]
library_dir = next((path for path in library_dirs if all(
(path / f"libsundials_{name}.a").is_file() for name in LIBRARIES)), None)
if library_dir is None:
raise RuntimeError("SUNDIALS static libraries not found; build with BUILD_STATIC_LIBS=ON and set SUNDIALS_ROOT.")
libraries = [library_dir / f"libsundials_{name}.a" for name in LIBRARIES]
dlls = []
else: else:
raise RuntimeError("Native builds support Windows and Linux; this platform is not supported.") flags += ["-D_POSIX_C_SOURCE=200809L"]
candidates = [sundials / "lib", sundials / "lib64", sundials / "lib/x86_64-linux-gnu"]
directory = next((p for p in candidates if all((p / f"libsundials_{name}.a").is_file() for name in LIBRARIES)), None)
if directory is None:
raise RuntimeError("SUNDIALS static libraries not found; set SUNDIALS_ROOT.")
libraries = [directory / f"libsundials_{name}.a" for name in LIBRARIES]
dlls = []
cache = (cache_dir or CACHE).resolve()
cache.mkdir(parents=True, exist_ok=True)
for kind in ("models", "objects"):
path = cache / kind
if path.is_symlink():
raise RuntimeError(f"Native cache directory must not be a symbolic link: {path}")
path.mkdir(exist_ok=True)
_budget("SIMULATION_NATIVE_MODEL_CACHE_MB", 256)
_budget("SIMULATION_NATIVE_OBJECT_CACHE_MB", 128)
work_key = sha256(os.urandom(32)).hexdigest()
work_lease = acquire_cache_lease(cache, "models", work_key)
try:
stage = Path(tempfile.mkdtemp(prefix=f"building-{work_key}-", dir=cache))
except BaseException:
work_lease.close()
raise
logs: list[str] = []
model_lease = None
try:
(stage / "model.c").write_text(program.source, encoding="utf-8", newline="\n")
(stage / "model.h").write_text(program.header, encoding="utf-8", newline="\n")
compiler_path = Path(shutil.which(compiler) or compiler).resolve()
compiler_identity = {"version": compiler_version, "binarySha256": _hash(compiler_path),
"target": _command([compiler, "-dumpmachine"], log=logs).decode().strip(),
"flags": flags, "platform": sys.platform}
runtime = _runtime_sources(program)
sources = [stage / "model.c", *runtime]
native_headers = {p: _hash(p) for p in (NATIVE / "include").rglob("*.h")}
dependency_headers = {p: _hash(p) for directory in ("sundials", "cvode", "nvector", "sunmatrix", "sunlinsol")
for p in (sundials / "include" / directory).glob("*.h")}
preprocessing_start = time.perf_counter()
# Preprocessed bytes include the actual transitive headers and all
# effective macros/ABI switches. Compile precisely these bytes, so a
# header edit cannot race the cache identity and the compiler input.
def preprocess(source: Path) -> dict:
local_log: list[str] = []
source_hash = _hash(source)
data = _command([compiler, *flags, "-E", "-P", f"-fmacro-prefix-map={stage}=/generated",
"-I", str(stage), "-I", str(NATIVE / "include"), "-I", str(sundials / "include"),
str(source)], log=local_log)
name = "model.c" if source == stage / "model.c" else "native/" + source.relative_to(NATIVE).as_posix()
if _hash(source) != source_hash:
raise RuntimeError(f"Native source changed during preprocessing: {source}")
digest = sha256(data).hexdigest()
identity = {"cacheVersion": 1, "sourceName": name, "preprocessedSha256": digest, "compiler": compiler_identity}
return {"sourceName": name, "sourceSha256": source_hash, "data": data,
"objectKey": _key(identity), "identity": identity, "log": local_log}
with ThreadPoolExecutor(max_workers=min(4, len(sources))) as pool:
units = list(pool.map(preprocess, sources))
for unit in units:
logs.extend(unit.pop("log"))
preprocessing_seconds = time.perf_counter() - preprocessing_start
if any(_hash(path) != digest for path, digest in {**native_headers, **dependency_headers}.items()):
raise RuntimeError("Native headers changed during preprocessing; retry with stable sources.")
dependencies = {p.name: _hash(p) for p in libraries + dlls}
notice = (NATIVE / "THIRD_PARTY_NOTICES.txt").read_bytes()
identity = {"cacheVersion": 2, "sourceSha256": sha256(program.source.encode()).hexdigest(),
"headerSha256": sha256(program.header.encode()).hexdigest(),
"contractSha256": _key(program.manifest()), "contractKeys": sorted(program.manifest()),
"objectKeys": [unit["objectKey"] for unit in units],
"compiler": compiler_identity, "dependencies": dependencies, "notice": sha256(notice).hexdigest()}
signature = _key(identity)
target = cache / "models" / signature
required = {"model.c", "model.h", executable_name, "THIRD_PARTY_NOTICES.txt", *(p.name for p in dlls)}
model_lease = acquire_cache_lease(cache, "models", signature)
manifest = _checked_manifest(target, "buildKey", signature, 2, required)
details = {"selectedModules": list(component_modules(program.source + "\n" + program.header)),
"unitCount": len(units), "objectCacheHits": 0, "objectCompilations": 0,
"preprocessSeconds": preprocessing_seconds, "compileSeconds": 0.0,
"compileWallSeconds": 0.0, "linkSeconds": 0.0, "modelCacheHit": manifest is not None}
if manifest is not None:
if progress_callback:
progress_callback("native-cache-hit")
touch_cache_entry(cache, "models", signature)
details["pruning"] = _prune(cache)
result = NativeBuild(target / executable_name, manifest, True, time.perf_counter() - started,
details, model_lease, cache)
model_lease = None
return result
if progress_callback:
progress_callback("native-compilation")
compile_start = time.perf_counter()
def compile_unit(index_unit: tuple[int, dict]) -> dict:
index, unit = index_unit
key = unit["objectKey"]
destination = cache / "objects" / key
with acquire_cache_lease(cache, "objects", key):
object_manifest = _checked_manifest(destination, "objectKey", key, 1, {"unit.o"})
hit = object_manifest is not None
elapsed = 0.0
local_log: list[str] = []
if not hit:
temporary = Path(tempfile.mkdtemp(prefix=f"building-{key}-", dir=cache / "objects"))
try:
(temporary / "unit.i").write_bytes(unit["data"])
before = time.perf_counter()
_command([compiler, *flags, "-x", "cpp-output", "-c", str(temporary / "unit.i"),
"-o", str(temporary / "unit.o")], log=local_log)
elapsed = time.perf_counter() - before
(temporary / "unit.i").unlink()
object_manifest = {**unit["identity"], "objectKey": key, "artifacts": {"unit.o": _hash(temporary / "unit.o")}}
(temporary / "manifest.json").write_text(json.dumps(object_manifest, indent=2) + "\n")
_publish(temporary, destination, "objectKey", key, 1, {"unit.o"})
finally:
if temporary.exists():
shutil.rmtree(temporary)
# Linking uses private copies, so another model's LRU pass can
# safely reclaim the shared object after this lease is released.
local_object = stage / f"object-{index}.o"
shutil.copyfile(destination / "unit.o", local_object)
touch_cache_entry(cache, "objects", key)
return {"path": local_object, "hit": hit, "seconds": elapsed, "log": local_log}
with ThreadPoolExecutor(max_workers=min(4, len(units))) as pool:
objects = list(pool.map(compile_unit, enumerate(units)))
details["compileWallSeconds"] = time.perf_counter() - compile_start
details["compileSeconds"] = sum(item["seconds"] for item in objects)
details["objectCacheHits"] = sum(item["hit"] for item in objects)
details["objectCompilations"] = len(objects) - details["objectCacheHits"]
for item in objects:
logs.extend(item["log"])
link_libraries = list(map(str, libraries)) link_libraries = list(map(str, libraries))
if sys.platform.startswith("linux"): if sys.platform.startswith("linux"):
link_libraries = ["-Wl,--start-group", *link_libraries, "-Wl,--end-group"] link_libraries = ["-Wl,--start-group", *link_libraries, "-Wl,--end-group"]
sources = {str(p.relative_to(ROOT)): _hash(p) for p in runtime} if progress_callback:
sources["native/THIRD_PARTY_NOTICES.txt"] = _hash(NATIVE / "THIRD_PARTY_NOTICES.txt") progress_callback("native-linking")
dependencies = {str(p.name): _hash(p) for p in libraries + dlls} before_link = time.perf_counter()
# Header hashes include precision/index ABI settings as well as library APIs. _command([compiler, *flags, *[str(item["path"]) for item in objects], *link_libraries,
for directory in ("sundials", "cvode", "nvector", "sunmatrix", "sunlinsol"): "-lm", "-o", str(stage / executable_name)], log=logs)
for path in sorted((sundials / "include" / directory).glob("*.h")): details["linkSeconds"] = time.perf_counter() - before_link
dependencies[f"{directory}/{path.name}"] = _hash(path) if {p.name: _hash(p) for p in libraries + dlls} != dependencies:
identity = dict(source=program.source, header=program.header, contract=program.manifest(), sources=sources, raise RuntimeError("Native dependencies changed during linking; retry with a stable toolchain.")
dependencies=dependencies, compiler=compiler_version, flags=flags, for item in objects:
platform=sys.platform, abi=1) item["path"].unlink()
signature = sha256(json.dumps(identity, sort_keys=True).encode()).hexdigest()
cache = (cache_dir or CACHE).resolve()
target = cache / signature
manifest_path = target / "manifest.json"
if manifest_path.is_file():
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
if all((target / name).is_file() and _hash(target / name) == digest
for name, digest in manifest["artifacts"].items()):
return NativeBuild(target / executable_name, manifest, True, time.perf_counter()-start)
raise RuntimeError(f"Native cache integrity check failed: {target}")
cache.mkdir(parents=True, exist_ok=True)
stage = Path(tempfile.mkdtemp(prefix="building-", dir=cache))
(stage / "model.c").write_text(program.source, encoding="utf-8")
(stage / "model.h").write_text(program.header, encoding="utf-8")
command = [compiler, *flags, "-I", str(stage), "-I", str(NATIVE / "include"),
"-I", str(sundials / "include"), str(stage / "model.c"),
*[str(p) for p in runtime if p.suffix == ".c"],
*link_libraries, "-lm", "-o", str(stage / executable_name)]
result = subprocess.run(command, capture_output=True, text=True, timeout=120)
(stage / "build.log").write_text(result.stdout + result.stderr, encoding="utf-8")
if result.returncode:
raise RuntimeError(f"Native compilation failed; see {stage / 'build.log'}: {result.stderr[-3000:]}")
for library in dlls: for library in dlls:
shutil.copy2(library, stage / library.name) shutil.copyfile(library, stage / library.name)
shutil.copy2(NATIVE / "THIRD_PARTY_NOTICES.txt", stage / "THIRD_PARTY_NOTICES.txt") if _hash(stage / library.name) != dependencies[library.name]:
manifest = { raise RuntimeError("Native runtime dependency changed during copying; retry with a stable toolchain.")
**program.manifest(), "buildKey": signature, "compiler": compiler_version, (stage / "THIRD_PARTY_NOTICES.txt").write_bytes(notice)
"compilerFlags": flags, "sourceHashes": sources, "dependencyHashes": dependencies, (stage / "build.log").write_text("\n".join(logs), encoding="utf-8")
"artifacts": {p.name: _hash(p) for p in stage.iterdir() if p.name != "build.log"}, manifest = {**program.manifest(), "cacheVersion": 2, "buildKey": signature, "buildIdentity": identity,
} "compiler": compiler_version, "compilerFlags": flags,
(stage / "manifest.json").write_text(json.dumps(manifest, ensure_ascii=False, indent=2), encoding="utf-8") "sourceHashes": {unit["sourceName"]: unit["sourceSha256"] for unit in units},
"nativeHeaderHashes": {"native/" + p.relative_to(NATIVE).as_posix(): digest
for p, digest in native_headers.items()},
"dependencyHashes": dependencies, "selectedModules": details["selectedModules"],
"objectKeys": [unit["objectKey"] for unit in units],
"artifacts": {name: _hash(stage / name) for name in sorted(required)}}
(stage / "manifest.json").write_text(json.dumps(manifest, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
manifest = _publish(stage, target, "buildKey", signature, 2, required)
touch_cache_entry(cache, "models", signature)
details["pruning"] = _prune(cache)
result = NativeBuild(target / executable_name, manifest, False, time.perf_counter() - started,
details, model_lease, cache)
model_lease = None
return result
finally:
if model_lease is not None:
model_lease.close()
# Failed compilations never publish partial entries or retain unlimited
# model sources/preprocessed files. Error stderr is included in the exception.
try: try:
stage.rename(target) if stage.exists():
except OSError:
# A concurrent compiler may have published the identical cache first.
if not manifest_path.is_file():
raise
published = json.loads(manifest_path.read_text(encoding="utf-8"))
# PE linker timestamps can differ between concurrent equivalent builds.
# Validate the winning build against its own hashes and our identity.
if published.get("buildKey") != signature or not all(
(target / name).is_file() and _hash(target / name) == digest
for name, digest in published["artifacts"].items()
):
raise RuntimeError("Concurrent native build did not produce the expected artifacts.")
manifest = published
if stage.resolve().parent != cache:
raise RuntimeError("Unexpected native build staging directory.")
shutil.rmtree(stage) shutil.rmtree(stage)
return NativeBuild(target / executable_name, manifest, False, time.perf_counter()-start) finally:
work_lease.close()
@@ -0,0 +1,453 @@
"""Leases and bounded LRU eviction for immutable native cache entries.
Builds publish and validate their own artifacts. Readers hold a shared lease
before checking an entry and until they finish using it. Eviction only removes
an entry while holding the corresponding nonblocking exclusive lease.
"""
from __future__ import annotations
from contextlib import suppress
from dataclasses import dataclass
import errno
import os
from pathlib import Path
import re
import shutil
import stat
import weakref
MODEL_LIMIT_BYTES = 256 * 1024**2
OBJECT_LIMIT_BYTES = 128 * 1024**2
LOCK_SHARDS = 128
_KINDS = frozenset(("models", "objects"))
_KEY = re.compile(r"[0-9a-f]{64}\Z")
# tempfile.mkdtemp uses eight lowercase letters, digits or underscores. Older
# stages lacking a lease key are deliberately outside automatic cleanup.
_STAGE = re.compile(r"building-([0-9a-f]{64})-[a-z0-9_]{8}\Z")
if os.name == "nt":
import ctypes
from ctypes import wintypes
import msvcrt
class _Offset(ctypes.Structure):
_fields_ = [("Offset", wintypes.DWORD), ("OffsetHigh", wintypes.DWORD)]
class _OffsetUnion(ctypes.Union):
_fields_ = [("offset", _Offset), ("Pointer", ctypes.c_void_p)]
class _Overlapped(ctypes.Structure):
_fields_ = [
("Internal", ctypes.c_size_t), ("InternalHigh", ctypes.c_size_t),
("offset", _OffsetUnion), ("hEvent", wintypes.HANDLE),
]
_kernel32 = ctypes.WinDLL("kernel32", use_last_error=True)
_lock_file = _kernel32.LockFileEx
_lock_file.argtypes = [
wintypes.HANDLE, wintypes.DWORD, wintypes.DWORD,
wintypes.DWORD, wintypes.DWORD, ctypes.POINTER(_Overlapped),
]
_lock_file.restype = wintypes.BOOL
_unlock_file = _kernel32.UnlockFileEx
_unlock_file.argtypes = [
wintypes.HANDLE, wintypes.DWORD, wintypes.DWORD,
wintypes.DWORD, ctypes.POINTER(_Overlapped),
]
_unlock_file.restype = wintypes.BOOL
else:
import fcntl
def _validate(kind: str, key: str) -> None:
if kind not in _KINDS or not isinstance(key, str) or _KEY.fullmatch(key) is None:
raise ValueError("Native cache leases require models/objects and a lowercase SHA-256 key.")
def _reparse(info: os.stat_result) -> bool:
# Junctions as well as symbolic links are excluded on Windows.
return stat.S_ISLNK(info.st_mode) or bool(
getattr(info, "st_file_attributes", 0)
& getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0x400)
)
def _directory(path: Path) -> os.stat_result | None:
try:
info = path.lstat()
except FileNotFoundError:
return None
if _reparse(info) or not stat.S_ISDIR(info.st_mode):
return None
return info
def _require_directory(path: Path, *, create: bool = False) -> None:
if create:
path.mkdir(parents=True, exist_ok=True)
if _directory(path) is None:
raise RuntimeError(f"Native cache directory must be a real directory: {path}")
def _lock(fd: int, *, exclusive: bool, blocking: bool) -> bool:
if os.name == "nt":
# os.open supplies a synchronous handle. LockFileEx therefore really
# waits without FAIL_IMMEDIATELY; unlike msvcrt.locking it also supports
# overlapping shared leases, including multiple builds in one process.
# The one-byte lock may extend beyond EOF, so lock files remain empty.
overlapped = _Overlapped()
flags = (2 if exclusive else 0) | (0 if blocking else 1)
if _lock_file(msvcrt.get_osfhandle(fd), flags, 0, 1, 0, ctypes.byref(overlapped)):
return True
code = ctypes.get_last_error()
if not blocking and code == 33: # ERROR_LOCK_VIOLATION
return False
raise ctypes.WinError(code)
operation = fcntl.LOCK_EX if exclusive else fcntl.LOCK_SH
if not blocking:
operation |= fcntl.LOCK_NB
while True:
try:
fcntl.flock(fd, operation)
return True
except InterruptedError:
continue
except OSError as exc:
if not blocking and exc.errno in (errno.EACCES, errno.EAGAIN):
return False
raise
def _release(fd: int, owner_pid: int) -> None:
# A forked child's finalizer must not explicitly unlock its parent's flock.
# Closing the inherited descriptor alone leaves the parent's lease intact.
with suppress(OSError):
if os.getpid() == owner_pid:
if os.name == "nt":
overlapped = _Overlapped()
_unlock_file(msvcrt.get_osfhandle(fd), 0, 1, 0, ctypes.byref(overlapped))
else:
fcntl.flock(fd, fcntl.LOCK_UN)
with suppress(OSError):
os.close(fd)
class CacheLease:
"""An acquired cache-use lock; close explicitly or use as a context manager."""
__slots__ = ("path", "kind", "key", "exclusive", "_finalizer", "__weakref__")
def __init__(self, fd: int, path: Path, kind: str, key: str, exclusive: bool):
self.path = path
self.kind = kind
self.key = key
self.exclusive = exclusive
self._finalizer = weakref.finalize(self, _release, fd, os.getpid())
@property
def closed(self) -> bool:
return not self._finalizer.alive
def close(self) -> None:
self._finalizer()
def __enter__(self) -> CacheLease:
if self.closed:
raise RuntimeError("Cannot reuse a closed native cache lease.")
return self
def __exit__(self, *_: object) -> None:
self.close()
def acquire_cache_lease(
cache_dir: Path, kind: str, key: str, *, exclusive: bool = False,
blocking: bool = True,
) -> CacheLease | None:
"""Acquire a shared-use/exclusive-eviction lease; return None only if busy.
Locks use 128 stable shards per namespace, bounding metadata as model keys
accumulate. A collision can defer eviction but cannot admit unsafe deletion.
Lock files must never be unlinked while the application is running: waiters
may already hold the old inode. There are at most 256 empty lock files.
"""
_validate(kind, key)
shard = int(key[:8], 16) % LOCK_SHARDS
return _acquire_lock_file(
Path(cache_dir).absolute(), kind, key, f"{kind}-{shard:03d}.lock",
exclusive=exclusive, blocking=blocking,
)
def _acquire_lock_file(
cache: Path, kind: str, key: str, lock_name: str, *, exclusive: bool,
blocking: bool,
) -> CacheLease | None:
_require_directory(cache, create=True)
lock_dir = cache / ".locks"
_require_directory(lock_dir, create=True)
lock_path = lock_dir / lock_name
try:
info = lock_path.lstat()
except FileNotFoundError:
info = None
if info is not None and (_reparse(info) or not stat.S_ISREG(info.st_mode)):
raise RuntimeError(f"Native cache lock must be a regular file: {lock_path}")
flags = os.O_RDWR | os.O_CREAT | getattr(os, "O_NOFOLLOW", 0) | getattr(os, "O_BINARY", 0)
fd = os.open(lock_path, flags, 0o600)
try:
if not stat.S_ISREG(os.fstat(fd).st_mode):
raise RuntimeError(f"Native cache lock must be a regular file: {lock_path}")
os.set_inheritable(fd, False)
if not _lock(fd, exclusive=exclusive, blocking=blocking):
os.close(fd)
return None
return CacheLease(fd, cache / kind / key, kind, key, exclusive)
except BaseException:
os.close(fd)
raise
def touch_cache_entry(cache_dir: Path, kind: str, key: str) -> None:
"""Mark an immutable entry as recently used while its caller holds a lease."""
_validate(kind, key)
cache = Path(cache_dir).absolute()
_require_directory(cache)
_require_directory(cache / kind)
entry = cache / kind / key
_require_directory(entry)
# Directory mtime is separate from the checksummed artifact/manifest bytes.
os.utime(entry, None, follow_symlinks=False)
@dataclass(frozen=True)
class _Entry:
path: Path
size: int
mtime_ns: int
device: int
inode: int
def _tree_size(path: Path) -> int | None:
"""Count regular file bytes without reading files or following any links."""
size = 0
try:
with os.scandir(path) as children:
for child in children:
info = child.stat(follow_symlinks=False)
if _reparse(info):
return None
if stat.S_ISREG(info.st_mode):
size += info.st_size
elif stat.S_ISDIR(info.st_mode):
nested = _tree_size(Path(child.path))
if nested is None:
return None
size += nested
else:
return None
except FileNotFoundError:
return None # A concurrent sweep may already have removed this entry.
return size
def _scan(namespace: Path) -> tuple[list[_Entry], int]:
entries = []
skipped = 0
if _directory(namespace) is None:
try:
namespace.lstat()
except FileNotFoundError:
return entries, 0
return entries, 1
with os.scandir(namespace) as children:
for child in children:
if _KEY.fullmatch(child.name) is None:
skipped += 1
continue
path = Path(child.path)
info = _directory(path)
if info is None:
skipped += 1
continue
size = _tree_size(path)
if size is None:
skipped += 1
continue
entries.append(_Entry(path, size, info.st_mtime_ns, info.st_dev, info.st_ino))
return entries, skipped
def _empty_report(limit: int) -> dict:
return {
"limitBytes": limit, "beforeBytes": 0, "afterBytes": 0,
"removedBytes": 0, "removedEntries": 0, "skippedInUse": 0,
"skippedChanged": 0, "skippedUnmanaged": 0, "oversizedEntries": 0,
"overLimitBytes": 0, "skippedConcurrentSweep": False, "errors": [],
}
def _prune_kind(cache: Path, kind: str, limit: int) -> dict:
report = _empty_report(limit)
namespace = cache / kind
try:
entries, skipped = _scan(namespace)
except OSError as exc:
report["errors"].append(str(exc))
return report
report["skippedUnmanaged"] = skipped
remaining = report["beforeBytes"] = sum(entry.size for entry in entries)
remaining_count = len(entries)
for entry in sorted(entries, key=lambda item: (item.mtime_ns, item.path.name)):
if remaining <= limit:
break
# Keep the last entry if it alone exceeds the budget. Older oversized
# entries are still evictable, so they cannot accumulate unboundedly.
if remaining_count == 1 and entry.size > limit:
continue
try:
lease = acquire_cache_lease(cache, kind, entry.path.name, exclusive=True, blocking=False)
if lease is None:
report["skippedInUse"] += 1
continue
with lease:
info = _directory(entry.path)
if info is None:
report["skippedChanged"] += 1
continue
if (info.st_dev, info.st_ino, info.st_mtime_ns) != (
entry.device, entry.inode, entry.mtime_ns,
):
# It was republished or reused after sorting; defer it to
# the next sweep instead of evicting a freshly used build.
report["skippedChanged"] += 1
continue
size = _tree_size(entry.path)
if size is None:
report["skippedUnmanaged"] += 1
continue
shutil.rmtree(entry.path)
remaining -= entry.size
remaining_count -= 1
report["removedBytes"] += size
report["removedEntries"] += 1
except (OSError, RuntimeError) as exc:
# Cleanup is best effort. A locked executable, permissions or a
# damaged unrelated cache entry must not fail a valid simulation.
report["errors"].append(str(exc))
try:
current, _ = _scan(namespace)
report["afterBytes"] = sum(entry.size for entry in current)
report["oversizedEntries"] = sum(entry.size > limit for entry in current)
except OSError as exc:
report["errors"].append(str(exc))
report["afterBytes"] = remaining
report["overLimitBytes"] = max(0, report["afterBytes"] - limit)
return report
def _empty_orphan_report() -> dict:
return {
"orphanStagesRemoved": 0, "orphanStagesBytes": 0,
"skippedInUse": 0, "skippedUnmanaged": 0,
"skippedConcurrentSweep": False, "errors": [],
}
def _prune_orphan_stages(cache: Path) -> dict:
"""Remove only named stages whose builder's lease is no longer held.
Callers acquire the embedded key's shared lease BEFORE creating the stage
and retain it until publication or cleanup. The OS releases that protection
if the builder is killed; no age threshold or unreliable PID test is needed.
Root/models stages use models leases; objects stages use objects leases.
"""
report = _empty_orphan_report()
for namespace, kind in ((cache, "models"), (cache / "models", "models"),
(cache / "objects", "objects")):
try:
if _directory(namespace) is None:
continue
with os.scandir(namespace) as children:
candidates = []
for child in children:
match = _STAGE.fullmatch(child.name)
if match is not None:
candidates.append((Path(child.path), match.group(1)))
elif child.name.startswith("building-"):
report["skippedUnmanaged"] += 1
except OSError as exc:
report["errors"].append(f"{namespace}: {exc}")
continue
for stage, key in candidates:
try:
lease = acquire_cache_lease(cache, kind, key, exclusive=True, blocking=False)
if lease is None:
report["skippedInUse"] += 1
continue
with lease:
if _directory(stage) is None:
report["skippedUnmanaged"] += 1
continue
size = _tree_size(stage)
if size is None:
report["skippedUnmanaged"] += 1
continue
shutil.rmtree(stage)
report["orphanStagesRemoved"] += 1
report["orphanStagesBytes"] += size
except (OSError, RuntimeError) as exc:
report["errors"].append(f"{stage}: {exc}")
return report
def prune_cache(
cache_dir: Path, *, model_limit_bytes: int = MODEL_LIMIT_BYTES,
object_limit_bytes: int = OBJECT_LIMIT_BYTES,
) -> dict:
"""Apply separate soft LRU byte budgets to managed models and objects.
In-use entries (including conservative shard collisions), the final entry
when it alone exceeds the budget, unknown names, links and legacy root-level
builds are retained. Returned counts explain any remaining overage. The
cache may transiently exceed its budgets while builds or solvers are active.
Strictly named stages abandoned by killed builders are removed under their
matching lease and counted separately in the orphanStages report.
"""
for limit in (model_limit_bytes, object_limit_bytes):
if isinstance(limit, bool) or not isinstance(limit, int) or limit < 0:
raise ValueError("Native cache byte limits must be nonnegative integers.")
cache = Path(cache_dir).absolute()
empty = {
"models": _empty_report(model_limit_bytes),
"objects": _empty_report(object_limit_bytes),
"orphanStages": _empty_orphan_report(),
}
try:
if _directory(cache) is None:
# Do not create a missing cache or inspect an alias to a directory.
return empty
# One additional stable lock file (257 total) serializes sweep decisions
# without serializing readers/builds. Otherwise simultaneous sweeps can
# each remove a different entry and incorrectly discard the final one.
sweep = _acquire_lock_file(
cache, "models", "0" * 64, "prune.lock", exclusive=True, blocking=False,
)
if sweep is None:
for report in empty.values():
report["skippedConcurrentSweep"] = True
return empty
with sweep:
orphan_stages = _prune_orphan_stages(cache)
return {
"models": _prune_kind(cache, "models", model_limit_bytes),
"objects": _prune_kind(cache, "objects", object_limit_bytes),
"orphanStages": orphan_stages,
}
except (OSError, RuntimeError) as exc:
for report in empty.values():
report["errors"].append(str(exc))
return empty
+34
View File
@@ -0,0 +1,34 @@
"""Reviewed native component module exports and their compilation dependencies."""
from __future__ import annotations
import re
EXPORTS = {
"properties": frozenset({
"native_properties_init", "native_temperature_ph_context", "native_density_context",
"native_gas_init", "native_gas", "native_gas_context", "native_medium_init",
"native_density", "native_temperature_ph", "native_viscosity",
"native_medium_gas", "native_medium_gas_context",
}),
"orifice": frozenset({"native_orifice", "native_orifice_context", "native_medium_orifice", "native_medium_orifice_context"}),
"pipe": frozenset({"native_pipe_resistance", "native_pipe_flow", "native_pipe_flow_context",
"native_pipe_flow_cached", "native_pipe_flow_cached_context",
"native_pipe_diagnostics", "native_pipe_diagnostics_context"}),
"mechanics": frozenset({"native_contact", "native_stop_motion", "native_limit_force"}),
"signal": frozenset({"native_signal", "native_signal_break"}),
}
DEPENDENCIES = {"orifice": ("properties",), "pipe": ("properties",)}
def component_modules(source: str) -> tuple[str, ...]:
# Generated C refers to these functions directly. Include identifier uses
# (also function pointers), conservatively including any comments/strings.
identifiers = set(re.findall(r"\bnative_[A-Za-z0-9_]+\b", source))
selected = {name for name, exports in EXPORTS.items() if identifiers & exports}
pending = list(selected)
while pending:
for dependency in DEPENDENCIES.get(pending.pop(), ()):
if dependency not in selected:
selected.add(dependency)
pending.append(dependency)
return tuple(name for name in EXPORTS if name in selected)
+10 -2
View File
@@ -105,6 +105,7 @@ def execute_native(build: NativeBuild, config: SolveIVPConfig, sample_step: floa
payload["buildKey"] = build.manifest["buildKey"] payload["buildKey"] = build.manifest["buildKey"]
payload["cacheHit"] = build.cache_hit payload["cacheHit"] = build.cache_hit
payload["buildSeconds"] = build.seconds payload["buildSeconds"] = build.seconds
payload["buildDetails"] = build.details
if activity_tracker is not None: if activity_tracker is not None:
activity_tracker.record_native_progress(payload["simulatedUntil"], payload["nfev"], payload["acceptedSteps"]) activity_tracker.record_native_progress(payload["simulatedUntil"], payload["nfev"], payload["acceptedSteps"])
return payload return payload
@@ -115,13 +116,20 @@ def simulate_native(network, config, *, sample_step, progress_callback=None,
if config.method not in ("RK45", "BDF"): if config.method not in ("RK45", "BDF"):
raise NativeCapabilityError(f"Native v1 does not support method {config.method}.") raise NativeCapabilityError(f"Native v1 does not support method {config.method}.")
if progress_callback: if progress_callback:
progress_callback(0.0, "initializing") progress_callback(0.0, "native-generation")
program = compile_native_program(network) program = compile_native_program(network)
build = build_native(program) build = build_native(program, progress_callback=(
(lambda phase: progress_callback(0.0, phase)) if progress_callback else None
))
try:
with tempfile.TemporaryDirectory(prefix="native-simulation-") as directory: with tempfile.TemporaryDirectory(prefix="native-simulation-") as directory:
data = execute_native(build, config, sample_step, run_dir=Path(directory), data = execute_native(build, config, sample_step, run_dir=Path(directory),
cancel_check=cancel_check, progress_callback=progress_callback, cancel_check=cancel_check, progress_callback=progress_callback,
activity_tracker=activity_tracker, raw_series=raw_series) activity_tracker=activity_tracker, raw_series=raw_series)
finally:
# Keep the executable pinned through process exit and result reading;
# only then may cache eviction reclaim this completed model version.
build.close()
totals = { totals = {
"nfev": data["nfev"], "njev": data["njev"], "nlu": data["nlu"], "nfev": data["nfev"], "njev": data["njev"], "nlu": data["nlu"],
"acceptedStepCount": data["acceptedSteps"], "rejectedStepCount": data["rejectedSteps"], "acceptedStepCount": data["acceptedSteps"], "rejectedStepCount": data["rejectedSteps"],
File diff suppressed because it is too large. Load diff
@@ -0,0 +1,115 @@
# 按需编译与分层缓存优化
日期:2026-09-12。基线提交 `aa4951b14ea6453138e3e877f1167a28100f486f`。本轮已接入正常构建入口,采用“可复用目标文件 + 完整模型缓存 + 容量预算/LRU”。同机八路首次构建中位 3.7742 → 2.4875 s,减少 34.09%;完整缓存命中的检查增加约 0.13 s。数值方程、牛顿求根、着色雅可比、`rtol=1e-8`、采样和结果编码保持当前已接受版本。
## 编译与复用方式
原来 `kernels.c` 中的实现拆为物性、孔口、管路、机械、信号五个功能模块。59 个原函数体逐字一致。构建器从生成代码中查找实际引用的导出函数,补齐依赖后选择模块;例如管路、孔口依赖物性。固定运行库仍需编译。粒度是功能模块,不是每个元件实例;同一模块内的相关函数会一起编译。
首次只编译所需单元,最多 4 个并行任务。每个单元预处理成确切的 C 内容,以预处理字节、逻辑文件名、编译器和编译选项计算对象键,再编译这份内容。缓存有效性覆盖实际使用的宏和间接头文件。完整模型另核对生成源码/头文件、组件合同、对象键、链接库和依赖声明。
系统改参数通常只影响生成的 `model.c`;拓扑或状态布局变化也可能使包含 `model.h` 的运行单元失效。完整模型仍在缓存时直接复用可执行文件,否则复用未变化的 `.o` 再链接。时间范围、步长、rtol 等运行选项不编入模型。缓存跨服务重启保留;服务重启本身不要求重新编译。
A/B/C/D 对应的模块对象可共同留在对象池,各系统的可执行文件也可分别留在模型池。容量回收后可能只保留部分对象或模型;再用到被清理内容时自动重建。这样能跨项目共享相同内容,也能限制长期增长。
## 文件位置与容量管理
所有新编译缓存都在 `app/data/native-builds/`,已被 Git 忽略:
| 位置 | 内容 | 默认容量预算 |
| --- | --- | ---: |
| `models/<SHA>/` | 完整模型可执行文件、生成源码、合同清单及必要 DLL | 256 MiB |
| `objects/<SHA>/` | 可复用目标文件及校验清单 | 128 MiB |
| `.locks/` | 固定分片的进程使用锁和清理锁 | 最多 257 个空锁文件 |
环境变量 `SIMULATION_NATIVE_MODEL_CACHE_MB`、`SIMULATION_NATIVE_OBJECT_CACHE_MB` 可设为非负整数 MiB。按最近使用时间淘汰较久未用内容;两池分别计量。模型淘汰不强制删除其共享对象,对象淘汰也不影响已链接模型。再次运行被淘汰的模型只是需要重新构建,不会丢失工程或仿真结果。
预算是**安全软上限**:正在构建/运行的条目跳过,锁分片碰撞也可能暂缓清理;单个条目已超过预算时保留最后一份并报告超额。计量为受管理文件的逻辑字节,不含文件系统元数据、锁、正在写入的临时目录以及旧格式缓存。因此不能将 384 MiB 理解为磁盘占用的绝对硬上限。
旧格式的根级 `<SHA>/` 未自动迁移或删除,本机检查时有 229 份、89,754,121 字节(85.60 MiB),不计入新预算,新代码也不会继续向旧格式追加。保留它们是为了不影响仍可能使用旧构建器的进程;它们的位置与新 `models/objects/` 分开,后续可在确认旧进程停止后单独整理。
普通失败清理临时目录;进程被强制终止遗留的新格式 `building-<SHA>-<随机后缀>/`,在后续清理时先取得对应排他锁,再回收。并发清理由全局锁串行化,避免重复淘汰。缓存缺文件、校验失败、符号链接或合同不一致时明确报错,不执行未经核验的缓存。
## 构建成本对照
使用冻结的基线源码和当前候选,各 1 轮预热、3 轮正式测量,每轮从独立空应用缓存开始;交替版本顺序,整个序列共 48 个案例。没有清空操作系统文件缓存。编译器、编译参数、链接库、各案例的生成 C/头文件/XML 均核对一致。统计是两组各自的中位数。
本表计时包含构建函数及紧邻的使用锁释放/容量清理,不含 Python 导入、XML/C 生成和求解。旧版没有对象阶段细分,缺失值保留为空,不用差值伪造。
| 场景 | 原版 | 新版 | 变化 |
| --- | ---: | ---: | ---: |
| 八路首次构建(空应用缓存) | 3.7742 s | 2.4875 s | 减少 34.09% |
| 相同八路再次运行 | 0.0347 s | 0.1632 s | 增加 128.6 ms |
| 八路改一个管长参数 | 3.9009 s | 2.4838 s | 减少 36.33% |
| 改参数后切回原八路 | 0.0335 s | 0.1868 s | 增加 153.4 ms |
| 八路切换到修正四路 | 2.4908 s | 1.1109 s | 减少 55.40% |
| 四路切回已缓存八路 | 0.0332 s | 0.1580 s | 增加 124.9 ms |
参数案例仅在隔离副本中将 `amesim_pnl0001_1.le` 从 1 改为 1.001 m;`tests/data` 正式模型未改。切四路使用现有修正四路工程,最后重新用原八路。
新版首次编译 12 个单元;仅改管长时复用 11 个、重编 1 个;切四路复用 7 个、重编 5 个。八路和四路实际上都使用五类功能模块,所以这组速度改善来自并行构建和对象复用,不能归结为它们省掉了某个未用元件模块。未用模块不会触发编译/失效的行为另有专项测试。
| 新版场景 | 预处理墙钟 | 并行编译墙钟 | 链接墙钟 |
| --- | ---: | ---: | ---: |
| 八路首次 | 0.0965 s | 2.2163 s | 0.0490 s |
| 八路改参数 | 0.0925 s | 2.2261 s | 0.0494 s |
| 切换四路 | 0.0956 s | 0.8880 s | 0.0490 s |
| 相同八路完整命中 | 0.0966 s | 0 | 0 |
阶段是嵌套观测值,各阶段的中位数不能相加后当作一次实测总时长。`compileSeconds` 是并行任务墙钟之和,不是 CPU 时间,也不能与 `compileWallSeconds` 累加。
仍有两点代价:
- 完整命中仍做预处理和内容校验,检查从 34.7 ms 增到 163.2 ms。保留这层检查可准确识别头文件/宏变化;本轮不把它描述为命中速度优化。
- 八路生成的 `model.c` 单独编译仍约 2.23 s,占主要成本。改一个参数虽然少编 11 个单元,墙钟仍接近新版首次构建;本轮没有把模型参数改成运行时载入。
运行序列结束时,旧缓存逻辑文件 4,230,318 字节,新缓存 5,064,224 字节(其中共享对象 788,363 字节),增加约 19.7%。首次单模型分别为 1,638,937 / 2,034,665 字节。新增对象池用一些空间换取重用,容量预算负责控制后续增长。性能案例没有触及预算;实际淘汰在独立并发/LRU 测试中验证。
## 数值与网页验证
修正八路 0~10 s、BDF、`rtol=1e-8` 完整运行。原生 `series/final/finalState` 的 **1,790,486 个 binary64 值逐位一致**,包括正负零。参考来自上一轮已接受的着色雅可比结果。`nfev=22853`、`njev=433`、雅可比 RHS 12124、普通 CVODE RHS 10729、接受步 6660、拒绝步 359、事件 1、启动 4、雅可比回退 0,均未变化。
用生产前端静态资源和真实浏览器在临时 8024 服务执行 1 次预热 + 3 次测量。首次完整模型未命中,但已有 7 个对象可用,因此它不是上表的全空缓存案例。首次构建 2.5649 s、点击到可查看 6.1815 s。随后三次中位数:
| 网页过程 | 耗时 |
| --- | ---: |
| 后台构建缓存检查 | 0.1976 s |
| C 求解 | 2.6489 s |
| 点击运行 → 结果可查看 | 3.7586 s |
| 点击运行 → 浏览器 IndexedDB 保存完成 | 3.8605 s |
| CSV 点击 → 下载保存完成 | 1.0348 s |
可查看是完成状态和可用按钮的 DOM 观测;保存是事务完成后的索引发布观测;下载包含浏览器通知及自动化 `saveAs`。过程存在重叠,不能累加。此次是本机回环地址验证,未复测远程端口转发链路,也不拿历史网页样本计算本轮配对加速比。
4 次完整网页结果、CSV、刷新恢复均与参考逐数值一致,共核对 7,154,280 个 CSV 单元;所有 CSV 文件字节相同。前端只调整了准备阶段文案/进度识别,显示生成、检查缓存、编译及链接的实际阶段。
本轮未重新运行 Amesim 软件或给出新增 Amesim 速度结论。四路已有外部曲线基准仍纳入回归,门槛保持不变;八路数值结论是对已接受的 native 版本等价。
## 回归与适用范围
- 93 项后端回归通过(54.123 s):元件/物性/管路、50 个网络冻结基准、四路曲线门槛、BDF/RK45、生成器、事件、取消、雅可比、结果接口等。
- 收尾构建/存储专项 37 项通过(7.397 s,与前述部分重叠):真实 GCC 构建、跨模型复用、参数/头文件/宏/编译选项/链接库失效、缓存完整性、编译链接失败、并发发布、活跃进程期间清理、LRU/软预算、符号链接、清理失败、被杀进程遗留目录回收。
- 前端构建通过;仿真超时与活动看门狗 12 项通过;实际浏览器四轮下载与恢复核对通过。
- 诊断工具已适配新的模块结构,避免聚合入口和模块重复编译;历史管路插桩工具做了语法/插桩准备检查,本轮未重跑其整套历史性能实验。
实际平台为 Linux x86_64、Python 3.12.3、GCC 13.3.0、SUNDIALS 7.4.0。Windows 使用 `LockFileEx` 的共享/排他锁实现,但没有 Windows 实机测试。最终补充生成 C/H 显式 LF 写出,避免 Windows 默认 CRLF 导致身份校验不一致;以模拟平台换行的真实构建回归验证。该补充在正式计时之后,对本次 Linux 写出字节没有影响,未为此重复 48 次计时。来源摘要分别保存计时版本与最终版本 SHA。
## 记录与复现
轻量 [机器可读摘要](assets/2026-09-12/native-build-cache-summary.json) 保存场景统计、来源 SHA、容量规则和网页核对。原始记录位于 Git 忽略的 `test/incremental-build-20260912/`:
- `baseline-source/`、`source-manifest.json`:原版本冻结源码与清单。
- `benchmark/summary.json`、`benchmark/timings.csv`、各轮案例目录:48 次构建记录。
- `browser/summary.json`、`browser-equality.json`、`native-bitwise-equality.json`:网页计时与一致性。
- `regression.log`、`frontend-build.log`、`frontend-regression.log`:验证日志;最终换行专项结果另见摘要。
```sh
.venv/bin/python tests/manual/benchmark_native_build_cache.py \
--baseline-root test/incremental-build-20260912/baseline-source \
--output-dir test/incremental-build-20260912/reproduction \
--warmups 1 --repeats 3 --run
.venv/bin/python -m unittest tests.test_native_build_cache tests.test_native_cache_storage -v
```
本轮没有安装新环境,环境/缓存/大结果均留在忽略目录。5173、8000 保持停止,临时 8024 已停止。此前结果传输压缩仍只在备份目录,未启用到生产。改动保留在工作区,未提交或推送 Git。
@@ -0,0 +1,9 @@
# 2026-09-12:按需编译与分层缓存
原生构建改为按实际引用选择功能模块,复用内容校验的目标文件,保留完整模型缓存。两池默认预算分别为 128 MiB 和 256 MiB,按最近使用时间清理;跨进程使用锁保护活跃构建/运行,超大单条目和在用内容允许暂时超额。旧根级缓存单独保留,新代码不继续追加旧格式。
冻结原版本同机对照,八路首次构建 3.7742→2.4875 s(-34.09%)、修改一个管长参数 3.9009→2.4838 s(-36.33%)、切四路 2.4908→1.1109 s(-55.40%)。完整模型命中检查增加约 0.13 s,已披露。八路原生 1,790,486 个 binary64 值逐位一致;网页三次中位可查看 3.7586 s、保存 3.8605 s、CSV 1.0348 s,四轮完整结果/下载/刷新一致。
93 项后端回归、收尾 37 项缓存专项(部分重叠)、前端构建和 12 项超时/看门狗测试通过。Windows 换行做了模拟回归,未实机验证 Windows 锁实现。正常 5173/8000 服务保持停止,临时验证服务已停止;传输压缩仍仅归档备份。本轮未安装环境、提交或推送 Git。
实现、缓存目录、容量边界、阶段成本和原始记录见 [完整报告](../other/按需编译与分层缓存优化-2026-09-12.md)。
+6 -1
View File
@@ -7749,7 +7749,7 @@ function FlowWorkbench() {
"info", "info",
`开始仿真:${projectNameRef.current || "untitled"},时间范围 ${project.simulation.t_start} - ${project.simulation.t_stop} s`, `开始仿真:${projectNameRef.current || "untitled"},时间范围 ${project.simulation.t_start} - ${project.simulation.t_stop} s`,
); );
await runBusyAction("正在编译模型并运行仿真...", async () => { await runBusyAction("正在准备并运行仿真...", async () => {
const xml = buildSystemXml(project, componentDefinitions, nodesRef.current); const xml = buildSystemXml(project, componentDefinitions, nodesRef.current);
const simulationId = createSimulationId(); const simulationId = createSimulationId();
activeSimulationIdRef.current = simulationId; activeSimulationIdRef.current = simulationId;
@@ -7774,6 +7774,11 @@ function FlowWorkbench() {
"compilation", "compilation",
"initialization", "initialization",
"initializing", "initializing",
"native-generation",
"native-cache-check",
"native-cache-hit",
"native-compilation",
"native-linking",
].includes(event.phase); ].includes(event.phase);
const progressMessage = const progressMessage =
watchdog?.status === "slow-active" watchdog?.status === "slow-active"
+6 -3
View File
@@ -66,15 +66,18 @@ BDF 默认使用结构着色差分,无需环境变量或网页选项。修正
- 网页和 Python CLI 默认 `rtol=1e-8`;生成的状态绝对误差限为质量 `1e-14 kg`、内能 `1e-8 J`、速度/位移 `1e-12`(各自 SI 单位)。独立 C 程序默认 `rtol=1e-6`,对照时应显式传入。CLI 可覆盖 rtol;本版不支持自定义 atol 或 first_step。不同积分器相同局部容差不保证全局曲线误差完全相同。 - 网页和 Python CLI 默认 `rtol=1e-8`;生成的状态绝对误差限为质量 `1e-14 kg`、内能 `1e-8 J`、速度/位移 `1e-12`(各自 SI 单位)。独立 C 程序默认 `rtol=1e-6`,对照时应显式传入。CLI 可覆盖 rtol;本版不支持自定义 atol 或 first_step。不同积分器相同局部容差不保证全局曲线误差完全相同。
- 时间信号显式分段,塑性/反弹端挡用稠密插值定位并重启。试探 RHS 不修改已接受状态。柔性接触沿用现有分段力公式,不改变刚度或阻尼来提速。 - 时间信号显式分段,塑性/反弹端挡用稠密插值定位并重启。试探 RHS 不修改已接受状态。柔性接触沿用现有分段力公式,不改变刚度或阻尼来提速。
- 每任务独立进程,支持进度、取消及超时。进程崩溃不会作为成功返回,受控失败保留最后接受状态。 - 每任务独立进程,支持进度、取消及超时。进程崩溃不会作为成功返回,受控失败保留最后接受状态。
- 编译缓存位于 `app/data/native-builds/`,校验模型、组件合同、源码、编译选项及依赖哈希。本版参数特化入 C,修改模型数值参数会重新编译;修改时间、步长、rtol、采样选项可复用 EXE。 - 编译缓存位于 `app/data/native-builds/`:`models/<SHA>/` 保存完整模型,`objects/<SHA>/` 保存可跨模型复用的模块目标文件。按当前模型使用的元件函数及其依赖选择模块,最多并行编译 4 个缺失单元。预处理后的实际 C 内容、工具链和编译选项组成对象键;完整模型键另含生成源码、组件合同及链接依赖。模型数值参数仍特化入 C,但仅重编受到影响的单元;时间、步长、rtol、采样选项仍是运行参数。
- 完整模型默认预算 256 MiB、对象预算 128 MiB,可分别设置非负整数环境变量 `SIMULATION_NATIVE_MODEL_CACHE_MB`、`SIMULATION_NATIVE_OBJECT_CACHE_MB`。按目录最近使用时间执行 LRU;活跃构建/运行及最后一个单独超额的条目保留并报告超额,因此是安全软上限。预算计算受管理文件的逻辑字节,不含旧版根级缓存、锁及文件系统元数据。
- 缓存读写带跨进程使用锁,命中时核验工件和身份,损坏缓存明确失败。对象复制到构建私有目录后再链接;模型使用锁一直保留到进程退出和结果读回。失败正常清理临时文件;进程被杀遗留的新格式构建目录在下一次清理时按锁状态回收。旧格式缓存保留,不自动迁移或删除,避免影响旧服务。
- Python 调用者若长期保留 `NativeBuild`,需在最后一次执行/打包后调用 `build.close()` 释放使用保护并触发清理;网页 runner 自动处理。返回的 `buildDetails` 区分完整命中、目标文件命中/编译次数、预处理、并行编译墙钟、编译任务耗时之和、链接和容量清理结果。
## 代码职责 ## 代码职责
- `app/simulation/native_codegen/input.py`:CLI 输入适配。 - `app/simulation/native_codegen/input.py`:CLI 输入适配。
- `contracts.py`:逐组件 C 实现版本白名单,新增模型或版本不会自动视为已支持。 - `contracts.py`:逐组件 C 实现版本白名单,新增模型或版本不会自动视为已支持。
- `compiler.py` / `extended.py`:能力检查、状态/输出布局、连接分组、常系数约束消元、C 生成。保留已验证的简单拓扑快速生成路径,两条路径均只运行 C 数值代码。 - `compiler.py` / `extended.py`:能力检查、状态/输出布局、连接分组、常系数约束消元、C 生成。保留已验证的简单拓扑快速生成路径,两条路径均只运行 C 数值代码。
- `build.py` / `runner.py`:编译缓存、依赖打包、隔离执行与结果适配。 - `build.py` / `modules.py` / `cache_storage.py`:按需模块、两层编译缓存、完整性校验与有界 LRU;`runner.py`:隔离执行、使用保护与结果适配。
- `native/components/kernels.c`:空气/氦气物性、阀与管路流量、信号、摩擦接触和端挡原语。 - `native/components/modules/`:物性、孔口、管路、机械和信号模块。`kernels.c` 仅供诊断的聚合入口,生产构建不再将聚合入口与模块重复编译。
- `native/runtime/`:RK45、CVODE 适配、事件定位、采样与 CLI。 - `native/runtime/`:RK45、CVODE 适配、事件定位、采样与 CLI。
- `app/simulation/backends.py` / `results.py`:共用执行入口与结果合同。 - `app/simulation/backends.py` / `results.py`:共用执行入口与结果合同。
+11 -485
View File
@@ -1,486 +1,12 @@
/* Helium Peng-Robinson and compressible-orifice kernels. /* Compatibility translation unit for standalone kernel diagnostics.
* Ported from the project's Python physical equations; validated independently * Production builds select individual module sources and compile each once;
* against Python at ordinary, reverse-flow and contact trial states. * never compile this entry together with those files. The internal-linkage
* switch preserves the original single-translation-unit diagnostic behavior.
* Paths also resolve through -I native/include when a harness copies this text.
*/ */
#include "kernels.h" #define NATIVE_COMPONENT_AMALGAMATION 1
#include <math.h> #include "../components/modules/properties.c"
#include <stddef.h> #include "../components/modules/orifice.c"
#include "../components/modules/mechanics.c"
#define RU 8.31446261815324 #include "../components/modules/signal.c"
#define MOLAR_MASS 0.004002602 #include "../components/modules/pipe.c"
#define TC 5.1953
#define PC 227460.0
#define OMEGA (-0.382)
static const double PI=3.1415926535897932384626433832795;
static const double pr_a=.457235583*RU*RU*TC*TC/PC;
static const double pr_b=.07779607*RU*TC/PC;
static const double kappa=.37464+1.54226*OMEGA-.26992*OMEGA*OMEGA;
static const double rg=RU/MOLAR_MASS;
static const NativeMedium helium_medium={1,RU/MOLAR_MASS,2.5*RU/MOLAR_MASS,293.15,0,1.96e-5,293.15,79.4};
static int same_medium(const NativeMedium *a,const NativeMedium *b) {
return a->real_helium==b->real_helium && a->R==b->R && a->cp==b->cp &&
a->Tref==b->Tref && a->slope==b->slope && a->mu==b->mu && a->muT==b->muT && a->S==b->S;
}
void native_properties_init(NativePropertyCache *cache,NativePropertyState *states,size_t capacity) {
cache->states=states;cache->count=0;cache->capacity=states?capacity:0;
}
static NativePropertyState *property_new(NativePropertyCache *cache,const NativeMedium *m,
double p,double T,NativePropertyState *scratch) {
int valid=p>0 && T>0 && isfinite(p) && isfinite(T);
NativePropertyState *s=valid && cache && cache->count<cache->capacity ? &cache->states[cache->count++] : scratch;
*s=(NativePropertyState){0};s->medium=*m;s->p=p;s->T=T;s->valid=valid?NATIVE_PROPERTY_PT:0;
return s;
}
static NativePropertyState *property_pt(NativePropertyCache *cache,const NativeMedium *m,
double p,double T,NativePropertyState *scratch) {
if(cache)for(size_t i=0;i<cache->count;i++) {
NativePropertyState *s=&cache->states[i];
if((s->valid&NATIVE_PROPERTY_PT) && s->p==p && s->T==T && same_medium(&s->medium,m))return s;
}
return property_new(cache,m,p,T,scratch);
}
static double property_density(NativePropertyState *s) {
if(!(s->valid&NATIVE_PROPERTY_RHO)) {
s->rho=native_density(&s->medium,s->p,s->T);
if(s->rho>0 && isfinite(s->rho))s->valid|=NATIVE_PROPERTY_RHO;
}
return s->rho;
}
static double property_viscosity(NativePropertyState *s) {
if(!(s->valid&NATIVE_PROPERTY_MU)) {
s->mu=native_viscosity(&s->medium,s->T,0);
if(s->mu>0 && isfinite(s->mu))s->valid|=NATIVE_PROPERTY_MU;
}
return s->mu;
}
static void remember_gas(NativePropertyCache *cache,const NativeMedium *m,const NativeGas *gas) {
/* Below helium's critical temperature retain the existing vapor-root/PH
selection. A future phase-aware medium contract can carry that state. */
if(!cache || (m->real_helium && gas->T<=TC))return;
NativePropertyState scratch,*s=property_pt(cache,m,gas->p,gas->T,&scratch);
if(!(s->valid&NATIVE_PROPERTY_PT) || !isfinite(gas->h))return;
if((s->valid&NATIVE_PROPERTY_H) && s->h!=gas->h)
s=property_new(cache,m,gas->p,gas->T,&scratch);
s->h=gas->h;s->valid|=NATIVE_PROPERTY_H;
if(gas->rho>0 && isfinite(gas->rho)){s->rho=gas->rho;s->valid|=NATIVE_PROPERTY_RHO;}
}
double native_temperature_ph_context(NativePropertyCache *cache,const NativeMedium *m,double p,double h) {
if(cache)for(size_t i=0;i<cache->count;i++) {
NativePropertyState *s=&cache->states[i];
if((s->valid&NATIVE_PROPERTY_H) && s->p==p && s->h==h && same_medium(&s->medium,m))return s->T;
}
double T=native_temperature_ph(m,p,h);
if(cache && isfinite(h) && T>0 && isfinite(T) && p>0 && isfinite(p)) {
NativePropertyState scratch,*s=property_pt(cache,m,p,T,&scratch);
if((s->valid&NATIVE_PROPERTY_H) && s->h!=h)s=property_new(cache,m,p,T,&scratch);
s->h=h;s->valid|=NATIVE_PROPERTY_H;
}
return T;
}
double native_density_context(NativePropertyCache *cache,const NativeMedium *m,double p,double T) {
NativePropertyState scratch;return property_density(property_pt(cache,m,p,T,&scratch));
}
static double cube_root(double x) { return x==0 ? 0 : copysign(pow(fabs(x),1.0/3.0),x); }
static void attraction(double T, double *a, double *da, double *dda) {
double tr=T/TC, sr=sqrt(tr), base=1+kappa*(1-sr);
*a=pr_a*base*base;
*da=pr_a*(-base*kappa/(TC*sr));
*dda=pr_a*kappa/(2*TC*TC)*(kappa/tr+base/(tr*sr));
}
static double z_factor(double p,double T) {
double a,da,dda; attraction(T,&a,&da,&dda);
double A=a*p/(RU*RU*T*T), B=pr_b*p/(RU*T);
double ca=-(1-B),cb=A-3*B*B-2*B,cc=-(A*B-B*B-B*B*B);
double pp=cb-ca*ca/3,qq=2*ca*ca*ca/27-ca*cb/3+cc;
double disc=pow(qq/2,2)+pow(pp/3,3),off=-ca/3,roots[3]; int n;
if(disc>1e-14) { roots[0]=cube_root(-qq/2+sqrt(disc))+cube_root(-qq/2-sqrt(disc))+off;n=1; }
else if(fabs(disc)<=1e-14) { double u=cube_root(-qq/2);roots[0]=2*u+off;roots[1]=-u+off;n=2; }
else { if(pp>=0) return NAN; double radius=2*sqrt(-pp/3);
double arg=(3*qq/(2*pp))*sqrt(-3/pp),theta=acos(fmax(-1,fmin(1,arg)))/3;
for(int i=0;i<3;i++) { roots[i]=radius*cos(theta-2*PI*i/3)+off; }
n=3;
}
double z=-INFINITY;for(int i=0;i<n;i++) if(roots[i]>B && isfinite(roots[i])) z=fmax(z,roots[i]);
return isfinite(z)?z:NAN;
}
static double density(double p,double T) { return MOLAR_MASS/(z_factor(p,T)*RU*T/p); }
static double log_volume(double rho) {
double v=MOLAR_MASS/rho,sq=sqrt(2.0);
return log((v+(1+sq)*pr_b)/(v+(1-sq)*pr_b))/(2*sq*pr_b*MOLAR_MASS);
}
static double u_departure(double T,double rho) {
double a,da,dda;attraction(T,&a,&da,&dda);return (T*da-a)*log_volume(rho);
}
static double h_departure(double p,double T) {
double a,da,dda;attraction(T,&a,&da,&dda);
double z=z_factor(p,T),B=pr_b*p/(RU*T),sq=sqrt(2.0);
double dep=RU*T*(z-1)+(T*da-a)*log((z+(1+sq)*B)/(z+(1-sq)*B))/(2*sq*pr_b);
return dep/MOLAR_MASS;
}
static double u_ideal(double T) { return rg*(1.5*T-745.375); }
static double h_ideal(double T) { return rg*(2.5*T-745.375); }
static double temperature_u(double u) { return (u/rg+745.375)/1.5; }
static double temperature_h(double h) { return (h/rg+745.375)/2.5; }
static double pressure_rho(double T,double rho) {
double v=MOLAR_MASS/rho,a,da,dda;attraction(T,&a,&da,&dda);
if(v<=pr_b || T<=0) return NAN;
return RU*T/(v-pr_b)-a/(v*(v+pr_b)+pr_b*(v-pr_b));
}
static NativeGas gas_properties(double m,double U,double V) {
NativeGas g;g.rho=m/V;g.u=U/m;
double T=fmax(temperature_u(g.u),2.2);
for(int i=0;i<16;i++) { double next=fmax(temperature_u(g.u-u_departure(T,g.rho)),2.2);
int done=fabs(next-T)<=1e-10*fmax(T,1);T=next;if(done) break; }
g.T=T;g.p=pressure_rho(T,g.rho);
/* Same energy reference as u_ideal/h_ideal. Avoid a redundant cubic solve
in the single-root region; preserve vapor-root semantics below TC. */
g.h=T>TC ? g.u+g.p/g.rho : h_ideal(T)+h_departure(g.p,T);return g;
}
static double temperature_ph(double p,double h) {
double T=fmax(temperature_h(h),2.2);
for(int i=0;i<16;i++) { double next=fmax(temperature_h(h-h_departure(p,T)),2.2);
int done=fabs(next-T)<=1e-10*fmax(T,1);T=next;if(done) break; }return T;
}
static void local_isentropic(NativePropertyState *s) {
if(s->valid&NATIVE_PROPERTY_ISENTROPIC)return;
double p=s->p,T=s->T,rho=property_density(s),v=MOLAR_MASS/rho,a,da,dda;attraction(T,&a,&da,&dda);
double d=v*(v+pr_b)+pr_b*(v-pr_b);
double dpT=RU/(v-pr_b)-da/d;
double dpR=(-RU*T/pow(v-pr_b,2)+a*2*(v+pr_b)/(d*d))*(-MOLAR_MASS/(rho*rho));
double cv=1.5*rg+T*dda*log_volume(rho);
double cp=cv+T*dpT*dpT/(rho*rho*dpR),gamma=cp/cv;
s->isentropic_factor=p/(rho*dpR*gamma);s->isentropic_exponent=p*(gamma-1)/(gamma*T*dpT);
if(isfinite(s->isentropic_factor) && isfinite(s->isentropic_exponent))s->valid|=NATIVE_PROPERTY_ISENTROPIC;
}
static double isentropic(NativePropertyCache *cache,NativePropertyState *up,double pd) {
local_isentropic(up);if(pd>=up->p)return up->isentropic_factor;
double Td=fmax(up->T*pow(fmax(pd/up->p,1e-12),up->isentropic_exponent),2.2);
NativePropertyState scratch,*down=property_pt(cache,&up->medium,fmax(pd,1),Td,&scratch);
local_isentropic(down);return .5*(up->isentropic_factor+down->isentropic_factor);
}
static double subsonic_cm(double r,double gamma,double rho,double T,double p) {
return sqrt(fmax(2/(1-gamma)*rho*T/p*(pow(r,2*gamma)-pow(r,1+gamma)),0));
}
static void state_valve(NativePropertyCache *cache,NativePropertyState *up,double pd,double *cm,double *vel) {
double p=up->p,T=up->T;pd=fmax(fmin(pd,p),0);
const NativeMedium *m=&up->medium;
double cp=m->cp+m->slope*(T-m->Tref);
double factor=m->real_helium?isentropic(cache,up,pd):(cp-m->R)/cp;
double g=fmax(1e-9,fmin(1-1e-9,factor)),rho=fmax(property_density(up),1e-12);
double r=fmax(pd/p,0),critical=pow(2*g/(g+1),1/(1-g)),eff;
if(r<=critical) { eff=critical;*cm=sqrt(2/(1+g)*rho*T/p)*pow(2*g/(g+1),g/(1-g));*vel=sqrt(2/(1+g)*p/rho); }
else { eff=r;*cm=subsonic_cm(r,g,rho,T,p);*vel=sqrt(fmax(2/(1-g)*p/rho*(1-pow(r,1-g)),0)); }
double ref=subsonic_cm(.9999,g,rho,T,p);
if(*cm>0 && ref>0) { double smooth=tanh(fmax(12*fabs(*cm/ref)*log(eff)/log(.9999),0));*cm*=smooth;*vel*=smooth; }
}
static void medium_valve(NativePropertyCache *cache,const NativeMedium *m,double p,double pd,double T,double *cm,double *vel) {
NativePropertyState scratch,*up=property_pt(cache,m,fmax(p,1),fmax(T,1),&scratch);
state_valve(cache,up,pd,cm,vel);
}
int native_gas_init(double p, double T, double volume, double *mU) {
if (!(p > 0 && T >= 2.2 && volume > 0)) return 0;
double rho = density(p, T);
mU[0] = rho * volume;
mU[1] = mU[0] * (u_ideal(T) + u_departure(T, rho));
return isfinite(mU[0]) && isfinite(mU[1]) && mU[0] > 0;
}
int native_gas(double m, double U, double volume, NativeGas *gas) {
if (!(m > 0 && volume > 0) || !isfinite(U)) return 0;
*gas = gas_properties(m, U, volume);
return gas->p > 0 && isfinite(gas->p) && isfinite(gas->h);
}
int native_gas_context(NativePropertyCache *cache,double m,double U,double V,NativeGas *gas) {
int ok=native_gas(m,U,V,gas);if(ok)remember_gas(cache,&helium_medium,gas);return ok;
}
int native_orifice(double p1, double p2, double h1, double h2,
double cq_area, double opening, double *flow,
double *cm, double *velocity) {
return native_orifice_context(NULL,p1,p2,h1,h2,cq_area,opening,flow,cm,velocity);
}
int native_orifice_context(NativePropertyCache *cache,double p1,double p2,double h1,double h2,
double cq_area,double opening,double *flow,double *cm,double *velocity) {
int forward = p1 >= p2;
double p = forward ? p1 : p2, pd = forward ? p2 : p1;
double T = native_temperature_ph_context(cache,&helium_medium,fmax(p,1),forward?h1:h2);
medium_valve(cache,&helium_medium,p,pd,T,cm,velocity);
double sign = forward ? 1 : -1;
*velocity *= sign;
*flow = opening == 0 || fabs(p1-p2) <= 1e-8 ? 0 :
sign * cq_area * opening * fmax(p, 1) * *cm / sqrt(fmax(T, 1));
if (opening == 0) *velocity = 0;
return isfinite(*flow) && isfinite(*cm) && isfinite(*velocity);
}
double native_contact(double penetration, double velocity, double stiffness,
double damping, double pdis, int signed_force) {
if (penetration <= 0) return 0;
double fraction = pdis > 0 ? -expm1(-penetration / pdis) : 1;
double force = stiffness * penetration + fraction * damping * velocity;
return signed_force == 1 ? force : fmax(force, 0);
}
void native_stop_motion(double x, double v, double lower, double upper,
double *acceleration, double *velocity) {
double vt = 1e-12 * fmax(fabs(v), 1);
if ((x <= lower + 1e-12*fmax(fabs(lower),1) && v <= vt && *acceleration <= 0) ||
(x >= upper - 1e-12*fmax(fabs(upper),1) && v >= -vt && *acceleration >= 0)) {
*acceleration = 0;
*velocity = 0;
}
}
double native_signal(double t, double start, int stages, int cyclic, const double *data) {
double elapsed = fmax(t-start,0), duration = 0, offset = 0;
for (int i=0;i<stages;i++) duration += data[16+i];
if (cyclic && duration > 0) elapsed = fmod(elapsed,duration);
for (int i=0;i<stages;i++) {
double d = data[16+i];
if (elapsed < offset+d || i == stages-1)
return d <= 0 ? data[8+i] : data[i] + (elapsed-offset)/d*(data[8+i]-data[i]);
offset += d;
}
return data[8+stages-1];
}
double native_signal_break(double t, double end, double start, int stages,
int cyclic, const double *data) {
double duration=0, offset=0, result=end;
for (int i=0;i<stages;i++) duration += data[16+i];
for (int i=0;i<stages;i++) {
double event=start+offset;
if (cyclic && duration > 0 && event <= t) {
double cycle=fmax(0,floor((t-event)/duration)+1);
event += cycle*duration;
if (event <= t) event += duration;
}
if (event > t) result=fmin(result,event);
offset += data[16+i];
}
return result;
}
static double ideal_temperature(const NativeMedium *m, double energy, double c) {
double delta = energy - c*m->Tref;
if (fabs(m->slope) <= 1e-15) return m->Tref + delta/c;
double root = sqrt(fmax(c*c + 2*m->slope*delta, 0));
double a = (-c+root)/m->slope, b = (-c-root)/m->slope;
return m->Tref + (fabs(a)<=fabs(b) ? a : b);
}
int native_medium_init(const NativeMedium *medium, double p, double T, double V,
int legacy_ideal_initial, double *mU) {
if (!(p>0 && T>0 && V>0)) return 0;
if (medium->real_helium && !legacy_ideal_initial) return native_gas_init(p,T,V,mU);
double mass=p*V/(medium->R*T),dt=T-medium->Tref;
double u=medium->real_helium?u_ideal(T):(medium->cp-medium->R)*T+.5*medium->slope*dt*dt;
mU[0]=mass;mU[1]=mass*u;
return isfinite(mU[0]) && isfinite(mU[1]);
}
double native_density(const NativeMedium *m, double p, double T) {
return m->real_helium ? density(p,T) : p/(m->R*T);
}
double native_temperature_ph(const NativeMedium *m, double p, double h) {
return m->real_helium ? temperature_ph(p,h) : ideal_temperature(m,h,m->cp);
}
double native_viscosity(const NativeMedium *m, double T, int diagnostic) {
/* Retain the ABI argument; flow and diagnostics use the same property. */
(void)diagnostic;
if (m->real_helium)
return 1e-7*exp(.7501594*log(T)+35.76324/T-2212.129/(T*T)+.9212635);
return m->mu*pow(T/m->muT,1.5)*(m->muT+m->S)/(T+m->S);
}
int native_medium_gas(const NativeMedium *medium, double m, double U, double V, NativeGas *g) {
if (medium->real_helium) return native_gas(m,U,V,g);
if (!(m>0 && V>0)) return 0;
g->u=U/m; g->rho=m/V;
g->T=ideal_temperature(medium,g->u,medium->cp-medium->R);
g->p=g->rho*medium->R*g->T;
double dt=g->T-medium->Tref;
g->h=medium->cp*g->T+.5*medium->slope*dt*dt;
return g->T>0 && isfinite(g->p) && isfinite(g->h);
}
int native_medium_gas_context(NativePropertyCache *cache,const NativeMedium *m,double mass,double U,double V,NativeGas *g) {
int ok=native_medium_gas(m,mass,U,V,g);if(ok)remember_gas(cache,m,g);return ok;
}
int native_medium_orifice(const NativeMedium *m, double p1, double p2, double h1, double h2,
double area, double opening, double *q, double *cm, double *v) {
return native_medium_orifice_context(NULL,m,p1,p2,h1,h2,area,opening,q,cm,v);
}
int native_medium_orifice_context(NativePropertyCache *cache,const NativeMedium *m,double p1,double p2,double h1,double h2,
double area,double opening,double *q,double *cm,double *v) {
double p=fmax(p1,p2),pd=fmin(p1,p2),sign=p1>=p2?1:-1;
double T=fmax(native_temperature_ph_context(cache,m,fmax(p,1),p1>=p2?h1:h2),1);
medium_valve(cache,m,p,pd,T,cm,v);
*q=fabs(p1-p2)<=1e-8?0:sign*area*opening*fmax(p,1)*(*cm)/sqrt(T);
*v*=fabs(opening)<=1e-12?0:sign;
return isfinite(*q) && isfinite(*cm) && isfinite(*v);
}
static double pipe_rough_limit(double rr) {
return rr>0 ? 1/pow(-2*log10(rr/3.7),2) : 0;
}
static double pipe_friction_prepared(double re, double rr, double rough_limit) {
if(re<=0) return 64000000;
double lam=64/re;
if(re<=89.96829989) return lam;
double smooth=1/pow(-1.8*log10(6.9/re),2),turb=smooth;
if(rr>0) { double r=re*rr,weight=r*r/(r*r+180*180);turb+=weight*(rough_limit-smooth); }
double trans=pow((re-89.96829989)/2741.96700831,8.37293695);
return lam+trans/(1+trans)*(turb-lam);
}
static double pipe_friction(double re, double rr) {
return pipe_friction_prepared(re,rr,pipe_rough_limit(rr));
}
/* Derivative of the existing friction blend, local to this scalar solve. */
static double pipe_friction_derivative(double re,double rr,double rough,double *df) {
double lam=64/re,dl=-lam/re;
if(re<=89.96829989){*df=dl;return lam;}
double a=-1.8*log10(6.9/re),smooth=1/(a*a);
double ds=-2*smooth/a*1.8/(log(10.0)*re),turb=smooth,dt=ds;
if(rr>0){double r=re*rr,w=r*r/(r*r+180*180),dw=2*w*(1-w)/re;
turb+=w*(rough-smooth);dt=ds*(1-w)+dw*(rough-smooth);}
double z=pow((re-89.96829989)/2741.96700831,8.37293695),b=z/(1+z);
double db=8.37293695*b*(1-b)/(re-89.96829989);
*df=dl*(1-b)+b*dt+db*(turb-lam);return lam+b*(turb-lam);
}
static double pipe_checked_solution(double re,double f,double K,double rr,double rough,
double flow_per_re,NativePipeSolve *status) {
double target=sqrt(K/f),change=fabs(target-re);
/* Separate the absolute and relative tests so a large flow scale cannot
make both sides overflow and accidentally satisfy the stopping test. */
if(!isfinite(target) || !isfinite(target*flow_per_re) ||
!(change<=re*1e-10 || change*flow_per_re<=1e-13))return NAN;
double value=target*target*pipe_friction_prepared(target,rr,rough);
status->relative_residual=fabs(value/K-1);
if(isfinite(value) && status->relative_residual<=1e-9) {
status->converged=1;return target;
}
return NAN;
}
double native_pipe_resistance(double K,double rr,double flow_per_re,NativePipeSolve *status) {
NativePipeSolve local={0,0,0,INFINITY};if(!status)status=&local;
*status=local;
if(!(K>=0 && rr>=0 && flow_per_re>0) || !isfinite(K) || !isfinite(rr) || !isfinite(flow_per_re))return NAN;
if(K==0){status->converged=1;status->relative_residual=0;return 0;}
if(K/64<=89.96829989) {
double re=K/64;
status->relative_residual=fabs(64*re/K-1);
if(re>0 && isfinite(re*flow_per_re) && status->relative_residual<=1e-9) {
status->converged=1;return re;
}
return NAN;
}
double rough=pipe_rough_limit(rr),lo=0,hi=fmax(sqrt(K/.02),1),df;
if(!isfinite(rough) || !isfinite(hi))return NAN;
int bracketed=0;
for(int i=0;i<128;i++) {
double value=hi*hi*pipe_friction_prepared(hi,rr,rough);
if(!isfinite(value))return NAN;
if(value>=K){bracketed=1;break;}
lo=hi;hi*=2;
if(!isfinite(hi))return NAN;
}
/* F(0)=-K; each positive lower endpoint was checked while expanding.
Never start iteration with an unchecked or non-finite upper residual. */
if(!bracketed)return NAN;
double re=fmax(lo,fmin(sqrt(K/.02),hi)),previous_residual=INFINITY;
int previous_newton=0;
for(int i=0;i<128;i++) {
status->iterations=i+1;
double f=pipe_friction_derivative(re,rr,rough,&df),value=re*re*f,F=value-K;
if(!(f>0) || !isfinite(f) || !isfinite(value))return NAN;
status->relative_residual=fabs(value/K-1);
double target=pipe_checked_solution(re,f,K,rr,rough,flow_per_re,status);
if(status->converged)return target;
if(F>0)hi=re;else lo=re;
double slope=2*re*f+re*re*df;
double next=slope>0 && isfinite(slope) ? re-F/slope : NAN;
/* An in-bracket Newton step is useful only if it reduces |F|.
After a step that fails to halve it, bisect the retained bracket;
successive Newton steps therefore cannot stagnate near an endpoint. */
int bisect=(previous_newton && fabs(F)>.5*previous_residual) ||
!(slope>0) || !isfinite(slope) || !isfinite(next) || next<=lo || next>=hi;
if(bisect) {
next=lo+.5*(hi-lo);status->bisections++;
if(next<=lo || next>=hi) {
/* Adjacent floating-point endpoints: inspect both actual
candidates, then fail if neither meets the original tests. */
double endpoints[]={lo,hi};
for(int j=0;j<2;j++) {
double endpoint=endpoints[j];
target=pipe_checked_solution(endpoint,pipe_friction_prepared(endpoint,rr,rough),
K,rr,rough,flow_per_re,status);
if(status->converged)return target;
}
return NAN;
}
}
previous_newton=!bisect;previous_residual=fabs(F);
re=next;
}
return NAN;
}
double native_pipe_flow(const NativeMedium *m, double p1, double p2, double T,
double d, double length, double rr, int kind) {
return native_pipe_flow_context(NULL,m,p1,p2,T,d,length,rr,kind);
}
double native_pipe_flow_context(NativePropertyCache *cache,const NativeMedium *m,double p1,double p2,double T,
double d,double length,double rr,int kind) {
if(fabs(p1-p2)<=1e-8) return 0;
double p=fmax(fmax(p1,p2),1),pd=fmin(p1,p2),sign=p1>p2?1:-1;
T=fmax(T,1);
NativePropertyState scratch,*up=property_pt(cache,m,p,T,&scratch);
double area=PI*d*d/4,mu=property_viscosity(up),den=PI*d*mu;
/* PNL0001/2/3 share compressible flow and its near-equilibrium smoothing.
PNL0003 differs in storage placement, not in the resistance law. */
double cm,vel;state_valve(cache,up,pd,&cm,&vel);
if(kind==0) {
double lam=pow(area*p*cm,2)/(16*PI*mu*length*T);
if(4*lam/den<=1000) return sign*lam;
}
double base=area*p*cm/sqrt(T),K=pow(4*base/den,2)*d/length;
return sign*native_pipe_resistance(K,rr,den/4,NULL)*den/4;
}
double native_pipe_flow_cached(NativePipeCache *cache, const NativeMedium *m,
double p1, double p2, double T, double d,
double length, double rr, int kind) {
return native_pipe_flow_cached_context(NULL,cache,m,p1,p2,T,d,length,rr,kind);
}
double native_pipe_flow_cached_context(NativePropertyCache *properties,NativePipeCache *cache,const NativeMedium *m,
double p1,double p2,double T,double d,double length,double rr,int kind) {
if(cache->valid && cache->p1==p1 && cache->p2==p2 && cache->T==T &&
cache->diameter==d && cache->length==length && cache->roughness==rr && cache->kind==kind &&
same_medium(&cache->medium,m))
return cache->flow;
double result=properties?native_pipe_flow_context(properties,m,p1,p2,T,d,length,rr,kind):
native_pipe_flow(m,p1,p2,T,d,length,rr,kind);
cache->valid=0;
if(isfinite(result)) {
cache->medium=*m;cache->p1=p1;cache->p2=p2;cache->T=T;
cache->diameter=d;cache->length=length;cache->roughness=rr;cache->kind=kind;
cache->flow=result;cache->valid=1;
}
return result;
}
void native_pipe_diagnostics(const NativeMedium *m, double q, double p, double T,
double d, double length, double rr, int diagnostic, double *r) {
native_pipe_diagnostics_context(NULL,m,q,p,T,d,length,rr,diagnostic,r);
}
void native_pipe_diagnostics_context(NativePropertyCache *cache,const NativeMedium *m,double q,double p,double T,
double d,double length,double rr,int diagnostic,double *r) {
(void)diagnostic;
NativePropertyState scratch,*state=property_pt(cache,m,p,T,&scratch);
double area=PI*d*d/4,re=4*fabs(q)/(PI*d*property_viscosity(state)),ff=pipe_friction(re,rr);
r[0]=re;
r[1]=fabs(q)*sqrt(T)/fmax(sqrt(d/(length*ff))*area*p,1e-18);
r[2]=q/(fmax(property_density(state),1e-12)*area);r[3]=ff;
}
double native_limit_force(double penetration, double velocity, double stiffness,
double damping, double depth, int signed_force) {
if(penetration<=0) return 0;
double force=stiffness*penetration+(depth>0?fmin(penetration/depth,1):1)*damping*velocity;
return signed_force==1?force:fmax(force,0);
}
+27
View File
@@ -0,0 +1,27 @@
#include "kernels.h"
#include <math.h>
double native_contact(double penetration, double velocity, double stiffness,
double damping, double pdis, int signed_force) {
if (penetration <= 0) return 0;
double fraction = pdis > 0 ? -expm1(-penetration / pdis) : 1;
double force = stiffness * penetration + fraction * damping * velocity;
return signed_force == 1 ? force : fmax(force, 0);
}
void native_stop_motion(double x, double v, double lower, double upper,
double *acceleration, double *velocity) {
double vt = 1e-12 * fmax(fabs(v), 1);
if ((x <= lower + 1e-12*fmax(fabs(lower),1) && v <= vt && *acceleration <= 0) ||
(x >= upper - 1e-12*fmax(fabs(upper),1) && v >= -vt && *acceleration >= 0)) {
*acceleration = 0;
*velocity = 0;
}
}
double native_limit_force(double penetration, double velocity, double stiffness,
double damping, double depth, int signed_force) {
if(penetration<=0) return 0;
double force=stiffness*penetration+(depth>0?fmin(penetration/depth,1):1)*damping*velocity;
return signed_force==1?force:fmax(force,0);
}
+35
View File
@@ -0,0 +1,35 @@
#include "component_properties_internal.h"
#include <math.h>
int native_orifice(double p1, double p2, double h1, double h2,
double cq_area, double opening, double *flow,
double *cm, double *velocity) {
return native_orifice_context(NULL,p1,p2,h1,h2,cq_area,opening,flow,cm,velocity);
}
int native_orifice_context(NativePropertyCache *cache,double p1,double p2,double h1,double h2,
double cq_area,double opening,double *flow,double *cm,double *velocity) {
int forward = p1 >= p2;
double p = forward ? p1 : p2, pd = forward ? p2 : p1;
double T = native_temperature_ph_context(cache,&helium_medium,fmax(p,1),forward?h1:h2);
medium_valve(cache,&helium_medium,p,pd,T,cm,velocity);
double sign = forward ? 1 : -1;
*velocity *= sign;
*flow = opening == 0 || fabs(p1-p2) <= 1e-8 ? 0 :
sign * cq_area * opening * fmax(p, 1) * *cm / sqrt(fmax(T, 1));
if (opening == 0) *velocity = 0;
return isfinite(*flow) && isfinite(*cm) && isfinite(*velocity);
}
int native_medium_orifice(const NativeMedium *m, double p1, double p2, double h1, double h2,
double area, double opening, double *q, double *cm, double *v) {
return native_medium_orifice_context(NULL,m,p1,p2,h1,h2,area,opening,q,cm,v);
}
int native_medium_orifice_context(NativePropertyCache *cache,const NativeMedium *m,double p1,double p2,double h1,double h2,
double area,double opening,double *q,double *cm,double *v) {
double p=fmax(p1,p2),pd=fmin(p1,p2),sign=p1>=p2?1:-1;
double T=fmax(native_temperature_ph_context(cache,m,fmax(p,1),p1>=p2?h1:h2),1);
medium_valve(cache,m,p,pd,T,cm,v);
*q=fabs(p1-p2)<=1e-8?0:sign*area*opening*fmax(p,1)*(*cm)/sqrt(T);
*v*=fabs(opening)<=1e-12?0:sign;
return isfinite(*q) && isfinite(*cm) && isfinite(*v);
}
+163
View File
@@ -0,0 +1,163 @@
#include "component_properties_internal.h"
#include "component_constants_internal.h"
#include <math.h>
static double pipe_rough_limit(double rr) {
return rr>0 ? 1/pow(-2*log10(rr/3.7),2) : 0;
}
static double pipe_friction_prepared(double re, double rr, double rough_limit) {
if(re<=0) return 64000000;
double lam=64/re;
if(re<=89.96829989) return lam;
double smooth=1/pow(-1.8*log10(6.9/re),2),turb=smooth;
if(rr>0) { double r=re*rr,weight=r*r/(r*r+180*180);turb+=weight*(rough_limit-smooth); }
double trans=pow((re-89.96829989)/2741.96700831,8.37293695);
return lam+trans/(1+trans)*(turb-lam);
}
static double pipe_friction(double re, double rr) {
return pipe_friction_prepared(re,rr,pipe_rough_limit(rr));
}
/* Derivative of the existing friction blend, local to this scalar solve. */
static double pipe_friction_derivative(double re,double rr,double rough,double *df) {
double lam=64/re,dl=-lam/re;
if(re<=89.96829989){*df=dl;return lam;}
double a=-1.8*log10(6.9/re),smooth=1/(a*a);
double ds=-2*smooth/a*1.8/(log(10.0)*re),turb=smooth,dt=ds;
if(rr>0){double r=re*rr,w=r*r/(r*r+180*180),dw=2*w*(1-w)/re;
turb+=w*(rough-smooth);dt=ds*(1-w)+dw*(rough-smooth);}
double z=pow((re-89.96829989)/2741.96700831,8.37293695),b=z/(1+z);
double db=8.37293695*b*(1-b)/(re-89.96829989);
*df=dl*(1-b)+b*dt+db*(turb-lam);return lam+b*(turb-lam);
}
static double pipe_checked_solution(double re,double f,double K,double rr,double rough,
double flow_per_re,NativePipeSolve *status) {
double target=sqrt(K/f),change=fabs(target-re);
/* Separate the absolute and relative tests so a large flow scale cannot
make both sides overflow and accidentally satisfy the stopping test. */
if(!isfinite(target) || !isfinite(target*flow_per_re) ||
!(change<=re*1e-10 || change*flow_per_re<=1e-13))return NAN;
double value=target*target*pipe_friction_prepared(target,rr,rough);
status->relative_residual=fabs(value/K-1);
if(isfinite(value) && status->relative_residual<=1e-9) {
status->converged=1;return target;
}
return NAN;
}
double native_pipe_resistance(double K,double rr,double flow_per_re,NativePipeSolve *status) {
NativePipeSolve local={0,0,0,INFINITY};if(!status)status=&local;
*status=local;
if(!(K>=0 && rr>=0 && flow_per_re>0) || !isfinite(K) || !isfinite(rr) || !isfinite(flow_per_re))return NAN;
if(K==0){status->converged=1;status->relative_residual=0;return 0;}
if(K/64<=89.96829989) {
double re=K/64;
status->relative_residual=fabs(64*re/K-1);
if(re>0 && isfinite(re*flow_per_re) && status->relative_residual<=1e-9) {
status->converged=1;return re;
}
return NAN;
}
double rough=pipe_rough_limit(rr),lo=0,hi=fmax(sqrt(K/.02),1),df;
if(!isfinite(rough) || !isfinite(hi))return NAN;
int bracketed=0;
for(int i=0;i<128;i++) {
double value=hi*hi*pipe_friction_prepared(hi,rr,rough);
if(!isfinite(value))return NAN;
if(value>=K){bracketed=1;break;}
lo=hi;hi*=2;
if(!isfinite(hi))return NAN;
}
/* F(0)=-K; each positive lower endpoint was checked while expanding.
Never start iteration with an unchecked or non-finite upper residual. */
if(!bracketed)return NAN;
double re=fmax(lo,fmin(sqrt(K/.02),hi)),previous_residual=INFINITY;
int previous_newton=0;
for(int i=0;i<128;i++) {
status->iterations=i+1;
double f=pipe_friction_derivative(re,rr,rough,&df),value=re*re*f,F=value-K;
if(!(f>0) || !isfinite(f) || !isfinite(value))return NAN;
status->relative_residual=fabs(value/K-1);
double target=pipe_checked_solution(re,f,K,rr,rough,flow_per_re,status);
if(status->converged)return target;
if(F>0)hi=re;else lo=re;
double slope=2*re*f+re*re*df;
double next=slope>0 && isfinite(slope) ? re-F/slope : NAN;
/* An in-bracket Newton step is useful only if it reduces |F|.
After a step that fails to halve it, bisect the retained bracket;
successive Newton steps therefore cannot stagnate near an endpoint. */
int bisect=(previous_newton && fabs(F)>.5*previous_residual) ||
!(slope>0) || !isfinite(slope) || !isfinite(next) || next<=lo || next>=hi;
if(bisect) {
next=lo+.5*(hi-lo);status->bisections++;
if(next<=lo || next>=hi) {
/* Adjacent floating-point endpoints: inspect both actual
candidates, then fail if neither meets the original tests. */
double endpoints[]={lo,hi};
for(int j=0;j<2;j++) {
double endpoint=endpoints[j];
target=pipe_checked_solution(endpoint,pipe_friction_prepared(endpoint,rr,rough),
K,rr,rough,flow_per_re,status);
if(status->converged)return target;
}
return NAN;
}
}
previous_newton=!bisect;previous_residual=fabs(F);
re=next;
}
return NAN;
}
double native_pipe_flow(const NativeMedium *m, double p1, double p2, double T,
double d, double length, double rr, int kind) {
return native_pipe_flow_context(NULL,m,p1,p2,T,d,length,rr,kind);
}
double native_pipe_flow_context(NativePropertyCache *cache,const NativeMedium *m,double p1,double p2,double T,
double d,double length,double rr,int kind) {
if(fabs(p1-p2)<=1e-8) return 0;
double p=fmax(fmax(p1,p2),1),pd=fmin(p1,p2),sign=p1>p2?1:-1;
T=fmax(T,1);
NativePropertyState scratch,*up=property_pt(cache,m,p,T,&scratch);
double area=PI*d*d/4,mu=property_viscosity(up),den=PI*d*mu;
/* PNL0001/2/3 share compressible flow and its near-equilibrium smoothing.
PNL0003 differs in storage placement, not in the resistance law. */
double cm,vel;state_valve(cache,up,pd,&cm,&vel);
if(kind==0) {
double lam=pow(area*p*cm,2)/(16*PI*mu*length*T);
if(4*lam/den<=1000) return sign*lam;
}
double base=area*p*cm/sqrt(T),K=pow(4*base/den,2)*d/length;
return sign*native_pipe_resistance(K,rr,den/4,NULL)*den/4;
}
double native_pipe_flow_cached(NativePipeCache *cache, const NativeMedium *m,
double p1, double p2, double T, double d,
double length, double rr, int kind) {
return native_pipe_flow_cached_context(NULL,cache,m,p1,p2,T,d,length,rr,kind);
}
double native_pipe_flow_cached_context(NativePropertyCache *properties,NativePipeCache *cache,const NativeMedium *m,
double p1,double p2,double T,double d,double length,double rr,int kind) {
if(cache->valid && cache->p1==p1 && cache->p2==p2 && cache->T==T &&
cache->diameter==d && cache->length==length && cache->roughness==rr && cache->kind==kind &&
same_medium(&cache->medium,m))
return cache->flow;
double result=properties?native_pipe_flow_context(properties,m,p1,p2,T,d,length,rr,kind):
native_pipe_flow(m,p1,p2,T,d,length,rr,kind);
cache->valid=0;
if(isfinite(result)) {
cache->medium=*m;cache->p1=p1;cache->p2=p2;cache->T=T;
cache->diameter=d;cache->length=length;cache->roughness=rr;cache->kind=kind;
cache->flow=result;cache->valid=1;
}
return result;
}
void native_pipe_diagnostics(const NativeMedium *m, double q, double p, double T,
double d, double length, double rr, int diagnostic, double *r) {
native_pipe_diagnostics_context(NULL,m,q,p,T,d,length,rr,diagnostic,r);
}
void native_pipe_diagnostics_context(NativePropertyCache *cache,const NativeMedium *m,double q,double p,double T,
double d,double length,double rr,int diagnostic,double *r) {
(void)diagnostic;
NativePropertyState scratch,*state=property_pt(cache,m,p,T,&scratch);
double area=PI*d*d/4,re=4*fabs(q)/(PI*d*property_viscosity(state)),ff=pipe_friction(re,rr);
r[0]=re;
r[1]=fabs(q)*sqrt(T)/fmax(sqrt(d/(length*ff))*area*p,1e-18);
r[2]=q/(fmax(property_density(state),1e-12)*area);r[3]=ff;
}
+241
View File
@@ -0,0 +1,241 @@
/* Helium Peng-Robinson and compressible-orifice kernels.
* Ported from the project's Python physical equations; validated independently
* against Python at ordinary, reverse-flow and contact trial states.
*/
#include "component_properties_internal.h"
#include "component_constants_internal.h"
#include <math.h>
#include <stddef.h>
#define RU 8.31446261815324
#define MOLAR_MASS 0.004002602
#define TC 5.1953
#define PC 227460.0
#define OMEGA (-0.382)
static const double pr_a=.457235583*RU*RU*TC*TC/PC;
static const double pr_b=.07779607*RU*TC/PC;
static const double kappa=.37464+1.54226*OMEGA-.26992*OMEGA*OMEGA;
static const double rg=RU/MOLAR_MASS;
NATIVE_COMPONENT_INTERNAL const NativeMedium helium_medium={1,RU/MOLAR_MASS,2.5*RU/MOLAR_MASS,293.15,0,1.96e-5,293.15,79.4};
NATIVE_COMPONENT_INTERNAL int same_medium(const NativeMedium *a,const NativeMedium *b) {
return a->real_helium==b->real_helium && a->R==b->R && a->cp==b->cp &&
a->Tref==b->Tref && a->slope==b->slope && a->mu==b->mu && a->muT==b->muT && a->S==b->S;
}
void native_properties_init(NativePropertyCache *cache,NativePropertyState *states,size_t capacity) {
cache->states=states;cache->count=0;cache->capacity=states?capacity:0;
}
static NativePropertyState *property_new(NativePropertyCache *cache,const NativeMedium *m,
double p,double T,NativePropertyState *scratch) {
int valid=p>0 && T>0 && isfinite(p) && isfinite(T);
NativePropertyState *s=valid && cache && cache->count<cache->capacity ? &cache->states[cache->count++] : scratch;
*s=(NativePropertyState){0};s->medium=*m;s->p=p;s->T=T;s->valid=valid?NATIVE_PROPERTY_PT:0;
return s;
}
NATIVE_COMPONENT_INTERNAL NativePropertyState *property_pt(NativePropertyCache *cache,const NativeMedium *m,
double p,double T,NativePropertyState *scratch) {
if(cache)for(size_t i=0;i<cache->count;i++) {
NativePropertyState *s=&cache->states[i];
if((s->valid&NATIVE_PROPERTY_PT) && s->p==p && s->T==T && same_medium(&s->medium,m))return s;
}
return property_new(cache,m,p,T,scratch);
}
NATIVE_COMPONENT_INTERNAL double property_density(NativePropertyState *s) {
if(!(s->valid&NATIVE_PROPERTY_RHO)) {
s->rho=native_density(&s->medium,s->p,s->T);
if(s->rho>0 && isfinite(s->rho))s->valid|=NATIVE_PROPERTY_RHO;
}
return s->rho;
}
NATIVE_COMPONENT_INTERNAL double property_viscosity(NativePropertyState *s) {
if(!(s->valid&NATIVE_PROPERTY_MU)) {
s->mu=native_viscosity(&s->medium,s->T,0);
if(s->mu>0 && isfinite(s->mu))s->valid|=NATIVE_PROPERTY_MU;
}
return s->mu;
}
static void remember_gas(NativePropertyCache *cache,const NativeMedium *m,const NativeGas *gas) {
/* Below helium's critical temperature retain the existing vapor-root/PH
selection. A future phase-aware medium contract can carry that state. */
if(!cache || (m->real_helium && gas->T<=TC))return;
NativePropertyState scratch,*s=property_pt(cache,m,gas->p,gas->T,&scratch);
if(!(s->valid&NATIVE_PROPERTY_PT) || !isfinite(gas->h))return;
if((s->valid&NATIVE_PROPERTY_H) && s->h!=gas->h)
s=property_new(cache,m,gas->p,gas->T,&scratch);
s->h=gas->h;s->valid|=NATIVE_PROPERTY_H;
if(gas->rho>0 && isfinite(gas->rho)){s->rho=gas->rho;s->valid|=NATIVE_PROPERTY_RHO;}
}
double native_temperature_ph_context(NativePropertyCache *cache,const NativeMedium *m,double p,double h) {
if(cache)for(size_t i=0;i<cache->count;i++) {
NativePropertyState *s=&cache->states[i];
if((s->valid&NATIVE_PROPERTY_H) && s->p==p && s->h==h && same_medium(&s->medium,m))return s->T;
}
double T=native_temperature_ph(m,p,h);
if(cache && isfinite(h) && T>0 && isfinite(T) && p>0 && isfinite(p)) {
NativePropertyState scratch,*s=property_pt(cache,m,p,T,&scratch);
if((s->valid&NATIVE_PROPERTY_H) && s->h!=h)s=property_new(cache,m,p,T,&scratch);
s->h=h;s->valid|=NATIVE_PROPERTY_H;
}
return T;
}
double native_density_context(NativePropertyCache *cache,const NativeMedium *m,double p,double T) {
NativePropertyState scratch;return property_density(property_pt(cache,m,p,T,&scratch));
}
static double cube_root(double x) { return x==0 ? 0 : copysign(pow(fabs(x),1.0/3.0),x); }
static void attraction(double T, double *a, double *da, double *dda) {
double tr=T/TC, sr=sqrt(tr), base=1+kappa*(1-sr);
*a=pr_a*base*base;
*da=pr_a*(-base*kappa/(TC*sr));
*dda=pr_a*kappa/(2*TC*TC)*(kappa/tr+base/(tr*sr));
}
static double z_factor(double p,double T) {
double a,da,dda; attraction(T,&a,&da,&dda);
double A=a*p/(RU*RU*T*T), B=pr_b*p/(RU*T);
double ca=-(1-B),cb=A-3*B*B-2*B,cc=-(A*B-B*B-B*B*B);
double pp=cb-ca*ca/3,qq=2*ca*ca*ca/27-ca*cb/3+cc;
double disc=pow(qq/2,2)+pow(pp/3,3),off=-ca/3,roots[3]; int n;
if(disc>1e-14) { roots[0]=cube_root(-qq/2+sqrt(disc))+cube_root(-qq/2-sqrt(disc))+off;n=1; }
else if(fabs(disc)<=1e-14) { double u=cube_root(-qq/2);roots[0]=2*u+off;roots[1]=-u+off;n=2; }
else { if(pp>=0) return NAN; double radius=2*sqrt(-pp/3);
double arg=(3*qq/(2*pp))*sqrt(-3/pp),theta=acos(fmax(-1,fmin(1,arg)))/3;
for(int i=0;i<3;i++) { roots[i]=radius*cos(theta-2*PI*i/3)+off; }
n=3;
}
double z=-INFINITY;for(int i=0;i<n;i++) if(roots[i]>B && isfinite(roots[i])) z=fmax(z,roots[i]);
return isfinite(z)?z:NAN;
}
static double density(double p,double T) { return MOLAR_MASS/(z_factor(p,T)*RU*T/p); }
static double log_volume(double rho) {
double v=MOLAR_MASS/rho,sq=sqrt(2.0);
return log((v+(1+sq)*pr_b)/(v+(1-sq)*pr_b))/(2*sq*pr_b*MOLAR_MASS);
}
static double u_departure(double T,double rho) {
double a,da,dda;attraction(T,&a,&da,&dda);return (T*da-a)*log_volume(rho);
}
static double h_departure(double p,double T) {
double a,da,dda;attraction(T,&a,&da,&dda);
double z=z_factor(p,T),B=pr_b*p/(RU*T),sq=sqrt(2.0);
double dep=RU*T*(z-1)+(T*da-a)*log((z+(1+sq)*B)/(z+(1-sq)*B))/(2*sq*pr_b);
return dep/MOLAR_MASS;
}
static double u_ideal(double T) { return rg*(1.5*T-745.375); }
static double h_ideal(double T) { return rg*(2.5*T-745.375); }
static double temperature_u(double u) { return (u/rg+745.375)/1.5; }
static double temperature_h(double h) { return (h/rg+745.375)/2.5; }
static double pressure_rho(double T,double rho) {
double v=MOLAR_MASS/rho,a,da,dda;attraction(T,&a,&da,&dda);
if(v<=pr_b || T<=0) return NAN;
return RU*T/(v-pr_b)-a/(v*(v+pr_b)+pr_b*(v-pr_b));
}
static NativeGas gas_properties(double m,double U,double V) {
NativeGas g;g.rho=m/V;g.u=U/m;
double T=fmax(temperature_u(g.u),2.2);
for(int i=0;i<16;i++) { double next=fmax(temperature_u(g.u-u_departure(T,g.rho)),2.2);
int done=fabs(next-T)<=1e-10*fmax(T,1);T=next;if(done) break; }
g.T=T;g.p=pressure_rho(T,g.rho);
/* Same energy reference as u_ideal/h_ideal. Avoid a redundant cubic solve
in the single-root region; preserve vapor-root semantics below TC. */
g.h=T>TC ? g.u+g.p/g.rho : h_ideal(T)+h_departure(g.p,T);return g;
}
static double temperature_ph(double p,double h) {
double T=fmax(temperature_h(h),2.2);
for(int i=0;i<16;i++) { double next=fmax(temperature_h(h-h_departure(p,T)),2.2);
int done=fabs(next-T)<=1e-10*fmax(T,1);T=next;if(done) break; }return T;
}
static void local_isentropic(NativePropertyState *s) {
if(s->valid&NATIVE_PROPERTY_ISENTROPIC)return;
double p=s->p,T=s->T,rho=property_density(s),v=MOLAR_MASS/rho,a,da,dda;attraction(T,&a,&da,&dda);
double d=v*(v+pr_b)+pr_b*(v-pr_b);
double dpT=RU/(v-pr_b)-da/d;
double dpR=(-RU*T/pow(v-pr_b,2)+a*2*(v+pr_b)/(d*d))*(-MOLAR_MASS/(rho*rho));
double cv=1.5*rg+T*dda*log_volume(rho);
double cp=cv+T*dpT*dpT/(rho*rho*dpR),gamma=cp/cv;
s->isentropic_factor=p/(rho*dpR*gamma);s->isentropic_exponent=p*(gamma-1)/(gamma*T*dpT);
if(isfinite(s->isentropic_factor) && isfinite(s->isentropic_exponent))s->valid|=NATIVE_PROPERTY_ISENTROPIC;
}
static double isentropic(NativePropertyCache *cache,NativePropertyState *up,double pd) {
local_isentropic(up);if(pd>=up->p)return up->isentropic_factor;
double Td=fmax(up->T*pow(fmax(pd/up->p,1e-12),up->isentropic_exponent),2.2);
NativePropertyState scratch,*down=property_pt(cache,&up->medium,fmax(pd,1),Td,&scratch);
local_isentropic(down);return .5*(up->isentropic_factor+down->isentropic_factor);
}
static double subsonic_cm(double r,double gamma,double rho,double T,double p) {
return sqrt(fmax(2/(1-gamma)*rho*T/p*(pow(r,2*gamma)-pow(r,1+gamma)),0));
}
NATIVE_COMPONENT_INTERNAL void state_valve(NativePropertyCache *cache,NativePropertyState *up,double pd,double *cm,double *vel) {
double p=up->p,T=up->T;pd=fmax(fmin(pd,p),0);
const NativeMedium *m=&up->medium;
double cp=m->cp+m->slope*(T-m->Tref);
double factor=m->real_helium?isentropic(cache,up,pd):(cp-m->R)/cp;
double g=fmax(1e-9,fmin(1-1e-9,factor)),rho=fmax(property_density(up),1e-12);
double r=fmax(pd/p,0),critical=pow(2*g/(g+1),1/(1-g)),eff;
if(r<=critical) { eff=critical;*cm=sqrt(2/(1+g)*rho*T/p)*pow(2*g/(g+1),g/(1-g));*vel=sqrt(2/(1+g)*p/rho); }
else { eff=r;*cm=subsonic_cm(r,g,rho,T,p);*vel=sqrt(fmax(2/(1-g)*p/rho*(1-pow(r,1-g)),0)); }
double ref=subsonic_cm(.9999,g,rho,T,p);
if(*cm>0 && ref>0) { double smooth=tanh(fmax(12*fabs(*cm/ref)*log(eff)/log(.9999),0));*cm*=smooth;*vel*=smooth; }
}
NATIVE_COMPONENT_INTERNAL void medium_valve(NativePropertyCache *cache,const NativeMedium *m,double p,double pd,double T,double *cm,double *vel) {
NativePropertyState scratch,*up=property_pt(cache,m,fmax(p,1),fmax(T,1),&scratch);
state_valve(cache,up,pd,cm,vel);
}
int native_gas_init(double p, double T, double volume, double *mU) {
if (!(p > 0 && T >= 2.2 && volume > 0)) return 0;
double rho = density(p, T);
mU[0] = rho * volume;
mU[1] = mU[0] * (u_ideal(T) + u_departure(T, rho));
return isfinite(mU[0]) && isfinite(mU[1]) && mU[0] > 0;
}
int native_gas(double m, double U, double volume, NativeGas *gas) {
if (!(m > 0 && volume > 0) || !isfinite(U)) return 0;
*gas = gas_properties(m, U, volume);
return gas->p > 0 && isfinite(gas->p) && isfinite(gas->h);
}
int native_gas_context(NativePropertyCache *cache,double m,double U,double V,NativeGas *gas) {
int ok=native_gas(m,U,V,gas);if(ok)remember_gas(cache,&helium_medium,gas);return ok;
}
static double ideal_temperature(const NativeMedium *m, double energy, double c) {
double delta = energy - c*m->Tref;
if (fabs(m->slope) <= 1e-15) return m->Tref + delta/c;
double root = sqrt(fmax(c*c + 2*m->slope*delta, 0));
double a = (-c+root)/m->slope, b = (-c-root)/m->slope;
return m->Tref + (fabs(a)<=fabs(b) ? a : b);
}
int native_medium_init(const NativeMedium *medium, double p, double T, double V,
int legacy_ideal_initial, double *mU) {
if (!(p>0 && T>0 && V>0)) return 0;
if (medium->real_helium && !legacy_ideal_initial) return native_gas_init(p,T,V,mU);
double mass=p*V/(medium->R*T),dt=T-medium->Tref;
double u=medium->real_helium?u_ideal(T):(medium->cp-medium->R)*T+.5*medium->slope*dt*dt;
mU[0]=mass;mU[1]=mass*u;
return isfinite(mU[0]) && isfinite(mU[1]);
}
double native_density(const NativeMedium *m, double p, double T) {
return m->real_helium ? density(p,T) : p/(m->R*T);
}
double native_temperature_ph(const NativeMedium *m, double p, double h) {
return m->real_helium ? temperature_ph(p,h) : ideal_temperature(m,h,m->cp);
}
double native_viscosity(const NativeMedium *m, double T, int diagnostic) {
/* Retain the ABI argument; flow and diagnostics use the same property. */
(void)diagnostic;
if (m->real_helium)
return 1e-7*exp(.7501594*log(T)+35.76324/T-2212.129/(T*T)+.9212635);
return m->mu*pow(T/m->muT,1.5)*(m->muT+m->S)/(T+m->S);
}
int native_medium_gas(const NativeMedium *medium, double m, double U, double V, NativeGas *g) {
if (medium->real_helium) return native_gas(m,U,V,g);
if (!(m>0 && V>0)) return 0;
g->u=U/m; g->rho=m/V;
g->T=ideal_temperature(medium,g->u,medium->cp-medium->R);
g->p=g->rho*medium->R*g->T;
double dt=g->T-medium->Tref;
g->h=medium->cp*g->T+.5*medium->slope*dt*dt;
return g->T>0 && isfinite(g->p) && isfinite(g->h);
}
int native_medium_gas_context(NativePropertyCache *cache,const NativeMedium *m,double mass,double U,double V,NativeGas *g) {
int ok=native_medium_gas(m,mass,U,V,g);if(ok)remember_gas(cache,m,g);return ok;
}
+33
View File
@@ -0,0 +1,33 @@
#include "kernels.h"
#include <math.h>
double native_signal(double t, double start, int stages, int cyclic, const double *data) {
double elapsed = fmax(t-start,0), duration = 0, offset = 0;
for (int i=0;i<stages;i++) duration += data[16+i];
if (cyclic && duration > 0) elapsed = fmod(elapsed,duration);
for (int i=0;i<stages;i++) {
double d = data[16+i];
if (elapsed < offset+d || i == stages-1)
return d <= 0 ? data[8+i] : data[i] + (elapsed-offset)/d*(data[8+i]-data[i]);
offset += d;
}
return data[8+stages-1];
}
double native_signal_break(double t, double end, double start, int stages,
int cyclic, const double *data) {
double duration=0, offset=0, result=end;
for (int i=0;i<stages;i++) duration += data[16+i];
for (int i=0;i<stages;i++) {
double event=start+offset;
if (cyclic && duration > 0 && event <= t) {
double cycle=fmax(0,floor((t-event)/duration)+1);
event += cycle*duration;
if (event <= t) event += duration;
}
if (event > t) result=fmin(result,event);
offset += data[16+i];
}
return result;
}
@@ -0,0 +1,10 @@
#ifndef NATIVE_COMPONENT_CONSTANTS_INTERNAL_H
#define NATIVE_COMPONENT_CONSTANTS_INTERNAL_H
/* Include only in modules using PI, so each standalone translation unit can
* fold the unchanged double constant without an external data dependency.
* The include guard also keeps the diagnostic amalgamation valid.
*/
static const double PI=3.1415926535897932384626433832795;
#endif
@@ -0,0 +1,31 @@
#ifndef NATIVE_COMPONENT_PROPERTIES_INTERNAL_H
#define NATIVE_COMPONENT_PROPERTIES_INTERNAL_H
#include "kernels.h"
/* Private interfaces shared by the flow modules. Gas/EOS formulas, property
* reuse and isentropic calculations stay together in properties.c. This
* switch is only for the diagnostic amalgamation in components/kernels.c;
* separately compiled production modules share ordinary external symbols.
*/
#ifdef NATIVE_COMPONENT_AMALGAMATION
#define NATIVE_COMPONENT_INTERNAL static
static const NativeMedium helium_medium;
#else
#define NATIVE_COMPONENT_INTERNAL
extern const NativeMedium helium_medium;
#endif
NATIVE_COMPONENT_INTERNAL int same_medium(const NativeMedium *, const NativeMedium *);
NATIVE_COMPONENT_INTERNAL NativePropertyState *property_pt(
NativePropertyCache *, const NativeMedium *, double p, double T,
NativePropertyState *scratch);
NATIVE_COMPONENT_INTERNAL double property_density(NativePropertyState *);
NATIVE_COMPONENT_INTERNAL double property_viscosity(NativePropertyState *);
NATIVE_COMPONENT_INTERNAL void state_valve(
NativePropertyCache *, NativePropertyState *, double pd, double *cm, double *velocity);
NATIVE_COMPONENT_INTERNAL void medium_valve(
NativePropertyCache *, const NativeMedium *, double p, double pd, double T,
double *cm, double *velocity);
#endif
+5 -1
View File
@@ -291,7 +291,11 @@ def main():
'mode':'plain' if args.plain else 'profile','port':args.port, 'mode':'plain' if args.plain else 'profile','port':args.port,
'python':sys.version,'platform':sys.platform, 'python':sys.version,'platform':sys.platform,
'frontendFiles':{str(p.relative_to(out/'frontend')):sha256(p.read_bytes()).hexdigest() for p in (out/'frontend').rglob('*') if p.is_file()}, 'frontendFiles':{str(p.relative_to(out/'frontend')):sha256(p.read_bytes()).hexdigest() for p in (out/'frontend').rglob('*') if p.is_file()},
'productionKernelSha256':sha256((ROOT/'native/components/kernels.c').read_bytes()).hexdigest()} # Retain the historical entry hash; module/header hashes now
# identify the numerical sources behind that diagnostic entry.
'productionKernelSha256':sha256((ROOT/'native/components/kernels.c').read_bytes()).hexdigest(),
'productionNativeSourceHashes':{p.relative_to(ROOT/'native').as_posix():sha256(p.read_bytes()).hexdigest()
for p in sorted((ROOT/'native').rglob('*')) if p.is_file() and p.suffix in ('.c','.h')}}
(out/'environment.json').write_text(json.dumps(metadata,indent=2)+'\n') (out/'environment.json').write_text(json.dumps(metadata,indent=2)+'\n')
import uvicorn import uvicorn
uvicorn.run(profile.app(api.app) if profile else api.app,host='127.0.0.1',port=args.port) uvicorn.run(profile.app(api.app) if profile else api.app,host='127.0.0.1',port=args.port)
@@ -0,0 +1,382 @@
"""Prepare or serially measure frozen/current native builds; never run a model.
Example (fresh output directory; omit --run for preparation only)::
.venv/bin/python tests/manual/benchmark_native_build_cache.py \
--baseline-root test/incremental-build-20260912/baseline-source \
--output-dir test/incremental-build-20260912/benchmark \
--warmups 1 --repeats 3 --run
Each version/round gets an empty application cache, followed by A/full hit,
A-prime/changed pipe length, A/full hit, four branches/first build, A/full hit.
Application-cache cold does not mean OS page-cache cold. Versions run in separate
Python processes, with alternating version order by round. Preparation, imports,
hashing and recording are outside build timing. NativeBuild.close() is measured
separately and as part of an adjacent build-and-release wall interval. Compilation
retains each production builder's concurrency and flags. No solver, frontend,
strace or previous diagnostic timings are invoked/combined.
"""
from __future__ import annotations
import argparse
from copy import deepcopy
import csv
from datetime import datetime, timezone
from hashlib import sha256
import json
import math
import os
from pathlib import Path
import platform
import shutil
import statistics
import subprocess
import sys
import time
ROOT = Path(__file__).resolve().parents[2]
SCRIPT = Path(__file__).resolve()
SCENARIOS = (
("cold", "a", False), ("same_model", "a", True),
("parameter_change", "a_prime", False), ("return_to_a", "a", True),
("four_branches", "four", False), ("return_from_four", "a", True),
)
METRICS = ("buildSeconds", "buildCallWallSeconds", "releaseWallSeconds",
"buildAndReleaseWallSeconds", "executableBytes", "cacheBytes",
"preprocessSeconds", "compileSeconds", "compileWallSeconds",
"linkSeconds", "unitCount", "objectCacheHits", "objectCompilations")
ENVIRONMENT = ("SIMULATION_NATIVE_CC", "SUNDIALS_ROOT",
"SIMULATION_NATIVE_MODEL_CACHE_MB", "SIMULATION_NATIVE_OBJECT_CACHE_MB")
def digest(path: Path) -> str:
with path.open("rb") as stream:
value = sha256()
for block in iter(lambda: stream.read(1024 * 1024), b""):
value.update(block)
return value.hexdigest()
def json_digest(value) -> str:
return sha256(json.dumps(value, sort_keys=True, separators=(",", ":")).encode()).hexdigest()
def write_json(path: Path, value) -> None:
path.write_text(json.dumps(value, ensure_ascii=False, indent=2, allow_nan=False) + "\n", encoding="utf-8")
def source_files(root: Path) -> dict:
paths = list((root / "app").rglob("*.py"))
paths += [p for directory in ("native", "schemas") for p in (root / directory).rglob("*") if p.is_file()]
return {p.relative_to(root).as_posix(): digest(p) for p in sorted(paths)
if "__pycache__" not in p.parts and not p.relative_to(root).as_posix().startswith("app/data/")}
def source_record(root: Path) -> dict:
if not (root / "app/simulation/native_codegen/build.py").is_file() or not (root / "native").is_dir():
raise ValueError(f"Incomplete source root: {root}")
frozen = root / "source-manifest.json"
if frozen.exists():
manifest = json.loads(frozen.read_text())
for name, expected in manifest["trackedFiles"].items():
if digest(root / name) != expected:
raise ValueError(f"Frozen source integrity failure: {root / name}")
revision = {"gitHead": manifest["head"], "frozenManifestSha256": digest(frozen), "worktreeStatus": None}
else:
revision = {"gitHead": subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=root, text=True).strip(),
"worktreeStatus": subprocess.check_output(["git", "status", "--porcelain", "--", "app", "native", "schemas"], cwd=root, text=True)}
files = source_files(root)
return {"root": str(root), **revision, "sourceFiles": files, "sourceFilesSha256": json_digest(files)}
def assert_source(record: dict) -> None:
if source_files(Path(record["root"])) != record["sourceFiles"]:
raise RuntimeError(f"Source changed after preparation: {record['root']}")
def make_inputs(args, output: Path) -> tuple[dict, dict]:
directory = output / "inputs"
directory.mkdir()
paths = {"a": directory / "model-a.json", "a_prime": directory / "model-a-prime.json", "four": directory / "model-four.json"}
shutil.copyfile(args.input, paths["a"])
shutil.copyfile(args.four_input, paths["four"])
original = json.loads(paths["a"].read_text(encoding="utf-8"))
variant = deepcopy(original)
matches = [(i, node) for i, node in enumerate(variant["nodes"]) if node["id"] == args.variant_node]
if len(matches) != 1:
raise ValueError(f"Expected exactly one variant node: {args.variant_node}")
index, node = matches[0]
data = node["data"]
if data["modelType"] != "amesim_pnl0001":
raise ValueError("The controlled length variant requires an amesim_pnl0001 pipe")
old = data["parameters"]["le"]
if type(old) not in (int, float) or not math.isfinite(old) or old <= 0:
raise ValueError("Pipe le must be a positive numeric SI value")
new = old * 1.001
if not math.isfinite(new) or new == old:
raise ValueError("Pipe length variant did not produce a finite changed value")
data["parameters"]["le"] = new
write_json(paths["a_prime"], variant)
data["parameters"]["le"] = old
if variant != original:
raise RuntimeError("Variant changed fields beyond the selected pipe length")
inputs = {key: {"path": str(path), "sha256": digest(path)} for key, path in paths.items()}
change = {"nodeId": node["id"], "modelType": data["modelType"], "jsonPointer": f"/nodes/{index}/data/parameters/le",
"before": old, "after": new, "storedUnit": "m", "factor": 1.001,
"note": "Stored numeric values are SI; editor parameterUnits do not rescale them."}
return inputs, change
def generate(config: dict, version: str, *, save: bool):
source = config["versions"][version]
assert_source(source)
root = Path(source["root"])
# Every worker starts a fresh interpreter; neither version imports the other.
sys.path.insert(0, str(root))
from app.main import compile_system_xml_network
from app.simulation.native_codegen.compiler import compile_native_program
from app.simulation.native_codegen.input import load_input
from app.simulation.native_codegen import build as builder
for name, module in tuple(sys.modules.items()):
location = getattr(module, "__file__", None)
if (name == "app" or name.startswith("app.")) and location and not Path(location).resolve().is_relative_to(root):
raise RuntimeError(f"Mixed source imports: {name}: {location}")
if builder.NATIVE.resolve() != root / "native":
raise RuntimeError("Builder native source root differs from the selected version")
generated = Path(config["outputDir"]) / version / "generated"
if save:
generated.mkdir(parents=True)
programs, records = {}, {}
for key, item in config["inputs"].items():
path = Path(item["path"])
if digest(path) != item["sha256"]:
raise RuntimeError(f"Input changed after preparation: {path}")
started = time.perf_counter()
xml, document = load_input(path)
program = compile_native_program(compile_system_xml_network(document))
preparation = time.perf_counter() - started
contract = program.manifest()
record = {"inputSha256": item["sha256"], "xmlSha256": sha256(xml).hexdigest(),
"sourceSha256": sha256(program.source.encode()).hexdigest(), "headerSha256": sha256(program.header.encode()).hexdigest(),
"contractSha256": json_digest(contract), "stateCount": len(program.state_keys),
"variableCount": len(program.variables)}
if save:
directory = generated / key
directory.mkdir()
(directory / "input.xml").write_bytes(xml)
(directory / "model.c").write_text(program.source, encoding="utf-8")
(directory / "model.h").write_text(program.header, encoding="utf-8")
write_json(directory / "contract.json", contract)
write_json(directory / "preparation.json", {**record, "generationSeconds": preparation})
programs[key], records[key] = program, record
if programs["a"].header != programs["a_prime"].header or programs["a"].state_keys != programs["a_prime"].state_keys:
raise RuntimeError("Length variant changed model header/state layout")
if programs["a"].source == programs["a_prime"].source:
raise RuntimeError("Length variant did not change generated C")
if not save and records != config["generated"][version]:
raise RuntimeError("Regenerated model differs from prepared artifacts")
assert_source(source)
return builder, programs, records
def cache_size(cache: Path) -> dict:
files = [p for p in cache.rglob("*") if p.is_file()]
return {"logicalFileBytes": sum(p.stat().st_size for p in files), "fileCount": len(files),
"modelsBytes": sum(p.stat().st_size for p in files if p.relative_to(cache).parts[0] == "models"),
"objectsBytes": sum(p.stat().st_size for p in files if p.relative_to(cache).parts[0] == "objects")}
def worker(args) -> None:
config = json.loads(args.worker_config.read_text())
if digest(SCRIPT) != config["scriptSha256"]:
raise RuntimeError("Benchmark script changed after preparation")
if {key: os.environ.get(key) for key in ENVIRONMENT} != config["environment"]:
raise RuntimeError("Build environment changed after preparation")
builder, programs, records = generate(config, args.version, save=args.worker == "prepare")
if args.worker == "prepare":
write_json(Path(config["outputDir"]) / args.version / "prepared-models.json", records)
return
directory = Path(config["outputDir"]) / args.version / args.label
directory.mkdir()
cache = directory / "cache"
if cache.exists():
raise RuntimeError("Cold build requires a previously nonexistent cache directory")
rows = []
for scenario, model, expected_hit in SCENARIOS:
case = directory / scenario
case.mkdir()
before = time.perf_counter()
built = builder.build_native(programs[model], cache_dir=cache)
returned = time.perf_counter()
release_start = time.perf_counter()
if hasattr(built, "close"):
built.close()
released = time.perf_counter()
manifest = built.manifest
copied_manifest = case / "build-manifest.json"
write_json(copied_manifest, manifest)
details = getattr(built, "details", {})
size = cache_size(cache)
row = {"version": args.version, "run": args.label, "warmup": args.label.startswith("warmup-"),
"scenario": scenario, "model": model, "buildSeconds": built.seconds,
"buildCallWallSeconds": returned - before, "releaseWallSeconds": released - release_start,
"buildAndReleaseWallSeconds": released - before, "cacheHit": built.cache_hit,
"buildKey": manifest["buildKey"], "details": details,
"executable": str(built.executable), "executableSha256": digest(built.executable),
"executableBytes": built.executable.stat().st_size, "cacheBytes": size["logicalFileBytes"], "cacheSize": size,
"sourceFilesSha256": config["versions"][args.version]["sourceFilesSha256"],
"generated": records[model], "compiler": manifest["compiler"], "compilerFlags": manifest["compilerFlags"],
"dependencyHashes": manifest["dependencyHashes"], "sourceHashes": manifest["sourceHashes"],
"nativeHeaderHashes": manifest.get("nativeHeaderHashes"),
"manifest": str(copied_manifest.relative_to(Path(config["outputDir"]))), "manifestSha256": digest(copied_manifest)}
row["checks"] = {"expectedCacheHit": built.cache_hit == expected_hit,
"parameterChangeReusesOtherObjects": (details.get("objectCompilations") == 1 and
details.get("objectCacheHits") == details.get("unitCount", 0) - 1)
if scenario == "parameter_change" and "objectCompilations" in details else None}
write_json(case / "measurement.json", row)
rows.append(row)
if any(value is False for value in row["checks"].values()):
raise RuntimeError(f"Unexpected cache behavior in {args.version}/{args.label}/{scenario}; measurement preserved")
assert_source(config["versions"][args.version])
write_json(directory / "measurements.json", {"complete": True, "solverExecuted": False, "rows": rows})
def invoke(config: dict, mode: str, version: str, label: str | None = None) -> dict:
output = Path(config["outputDir"])
command = [sys.executable, str(SCRIPT), "--worker", mode, "--worker-config", str(output / "prepared.json"), "--version", version]
if label:
command += ["--label", label]
log = output / "logs" / (version + "-" + (label or mode))
start = time.perf_counter()
with log.with_suffix(".stdout.log").open("wb") as stdout, log.with_suffix(".stderr.log").open("wb") as stderr:
process = subprocess.run(command, cwd=config["versions"][version]["root"], stdout=stdout, stderr=stderr, timeout=config["workerTimeout"])
observed = {"command": command, "processWallSeconds": time.perf_counter() - start, "exitCode": process.returncode,
"version": version, "run": label, "scope": "Whole worker including import/generation/checks/recording; not build latency."}
write_json(log.with_suffix(".process.json"), observed)
if process.returncode:
raise RuntimeError(f"Worker failed ({process.returncode}); see {log.with_suffix('.stderr.log')}")
return observed
def stat(values: list) -> dict:
return {"n": len(values), "min": min(values), "median": statistics.median(values), "max": max(values)} if values else {"n": 0, "min": None, "median": None, "max": None}
def metric(row: dict, name: str):
return row[name] if name in row else row["details"].get(name)
def summarize(config: dict, rows: list, processes: list) -> dict:
expected = 2 * (config["warmups"] + config["repeats"]) * len(SCENARIOS)
if len(rows) != expected:
raise RuntimeError(f"Incomplete benchmark: expected {expected} rows, found {len(rows)}")
compilers = {(r["compiler"], tuple(r["compilerFlags"])) for r in rows}
libraries = {json_digest({k: v for k, v in r["dependencyHashes"].items() if "/" not in k}) for r in rows}
if len(compilers) != 1 or len(libraries) != 1:
raise RuntimeError("Compiler/flags or linked library bytes differ across benchmark runs")
statistics_by_version = {}
for version in config["versions"]:
statistics_by_version[version] = {}
for scenario, _, _ in SCENARIOS:
selected = [r for r in rows if r["version"] == version and r["scenario"] == scenario and not r["warmup"]]
if len(selected) != config["repeats"]:
raise RuntimeError(f"Incomplete formal group: {version}/{scenario}")
statistics_by_version[version][scenario] = {key: stat([metric(r, key) for r in selected if metric(r, key) is not None]) for key in METRICS}
changes = {}
for scenario, _, _ in SCENARIOS:
changes[scenario] = {}
for key in ("buildSeconds", "buildCallWallSeconds", "buildAndReleaseWallSeconds"):
old = statistics_by_version["baseline"][scenario][key]["median"]
new = statistics_by_version["candidate"][scenario][key]["median"]
changes[scenario][key] = {"baselineMedian": old, "candidateMedian": new,
"reductionPercent": 100 * (1 - new / old) if old else None,
"speedup": old / new if new else None, "method": "ratio of group medians"}
return {"complete": True, "solverExecuted": False, "prepared": "prepared.json", "scriptSha256": config["scriptSha256"],
"contract": config["contract"], "checks": {"compilerAndFlagsIdentical": True, "linkedLibrariesIdentical": True,
"generatedModelsIdenticalAcrossVersions": True, "variantHeaderAndStateLayoutUnchanged": True},
"statistics": statistics_by_version, "changes": changes, "rows": rows, "workers": processes}
def main() -> None:
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument("--baseline-root", type=Path, default=ROOT / "test/incremental-build-20260912/baseline-source")
parser.add_argument("--candidate-root", type=Path, default=ROOT)
parser.add_argument("--input", type=Path, default=ROOT / "tests/data/test-mql-8-corrected.json")
parser.add_argument("--four-input", type=Path, default=ROOT / "tests/data/test-mql-4-corrected.json")
parser.add_argument("--variant-node", default="amesim_pnl0001_1")
parser.add_argument("--output-dir", type=Path)
parser.add_argument("--warmups", type=int, default=1)
parser.add_argument("--repeats", type=int, default=3)
parser.add_argument("--worker-timeout", type=float, default=300)
parser.add_argument("--run", action="store_true")
parser.add_argument("--worker", choices=("prepare", "round"), help=argparse.SUPPRESS)
parser.add_argument("--worker-config", type=Path, help=argparse.SUPPRESS)
parser.add_argument("--version", choices=("baseline", "candidate"), help=argparse.SUPPRESS)
parser.add_argument("--label", help=argparse.SUPPRESS)
args = parser.parse_args()
if args.worker:
if not args.worker_config or not args.version or (args.worker == "round" and not args.label):
parser.error("Internal worker arguments are incomplete")
worker(args)
return
if not args.output_dir or args.warmups < 1 or args.repeats < 1 or not math.isfinite(args.worker_timeout) or args.worker_timeout <= 0:
parser.error("A fresh --output-dir, warmups/repeats >= 1 and a positive worker timeout are required")
output = args.output_dir.resolve()
if not output.is_relative_to(ROOT / "test") or output.exists():
parser.error("Choose a previously nonexistent directory under the ignored test/ tree")
versions = {"baseline": source_record(args.baseline_root.resolve()), "candidate": source_record(args.candidate_root.resolve())}
output.mkdir(parents=True)
(output / "logs").mkdir()
try:
inputs, change = make_inputs(args, output)
config = {"createdAtUtc": datetime.now(timezone.utc).isoformat(), "outputDir": str(output), "versions": versions,
"script": str(SCRIPT), "scriptSha256": digest(SCRIPT), "python": sys.version, "pythonExecutable": sys.executable,
"platform": platform.platform(), "cpuCount": os.cpu_count(), "environment": {key: os.environ.get(key) for key in ENVIRONMENT},
"inputs": inputs, "variant": change, "warmups": args.warmups, "repeats": args.repeats,
"workerTimeout": args.worker_timeout, "runRequested": args.run,
"contract": {"cold": "Independent empty application cache per version/round; OS caches are not flushed.",
"sequence": [name for name, _, _ in SCENARIOS], "solverExecuted": False,
"fourBranchCase": "Switch from the full eight-branch project to the corrected four-branch project, then restore A; no cached files are manually deleted.",
"wallTiming": "Builder call and adjacent release are measured separately; generation, hashing, imports and recording are outside these spans.",
"overlap": "buildSeconds is inside buildCallWallSeconds, which is inside buildAndReleaseWallSeconds. Detail spans are nested; compileSeconds sums concurrent TU walls and is not additive to compileWallSeconds.",
"unavailable": "Old builder has no object/stage details; absent measurements are null, never inferred from differences.",
"comparison": "Formal group-median ratios; warmups excluded. No old strace timings or solver performance claims."}}
write_json(output / "prepared.json", config)
for version in versions:
invoke(config, "prepare", version)
config["generated"] = {version: json.loads((output / version / "prepared-models.json").read_text()) for version in versions}
if config["generated"]["baseline"] != config["generated"]["candidate"]:
raise RuntimeError("Generated model/XML/header/contract differs between source versions")
write_json(output / "prepared.json", config)
if not args.run:
print(json.dumps({"preparedOnly": True, "outputDir": str(output), "compiled": False, "solverExecuted": False}))
return
rows, processes = [], []
for index in range(args.warmups + args.repeats):
label = f"warmup-{index + 1}" if index < args.warmups else f"run-{index - args.warmups + 1}"
order = ("baseline", "candidate") if index % 2 == 0 else ("candidate", "baseline")
for version in order:
processes.append(invoke(config, "round", version, label))
rows.extend(json.loads((output / version / label / "measurements.json").read_text())["rows"])
for record in versions.values():
assert_source(record)
summary = summarize(config, rows, processes)
write_json(output / "summary.json", summary)
with (output / "timings.csv").open("w", newline="", encoding="utf-8") as stream:
writer = csv.DictWriter(stream, fieldnames=["version", "run", "warmup", "scenario", "metric", "value", "n", "min", "median", "max"])
writer.writeheader()
for row in rows:
for key in METRICS:
writer.writerow({**{k: row[k] for k in ("version", "run", "warmup", "scenario")}, "metric": key, "value": metric(row, key)})
for version, groups in summary["statistics"].items():
for scenario, values in groups.items():
for key, value in values.items():
writer.writerow({"version": version, "run": "formal-statistics", "warmup": False, "scenario": scenario, "metric": key, **value})
print(json.dumps({"complete": True, "summary": str(output / "summary.json"), "solverExecuted": False}))
except Exception as exc:
write_json(output / "failure.json", {"complete": False, "error": str(exc), "type": type(exc).__name__, "solverExecuted": False})
raise
if __name__ == "__main__":
main()
+79 -6
View File
@@ -33,6 +33,76 @@ NEW_CALL = '''double base=area*p*cm/sqrt(T),K=pow(4*base/den,2)*d/length;
return sign*native_pipe_resistance(K,rr,den/4,NULL)*den/4;''' return sign*native_pipe_resistance(K,rr,den/4,NULL)*den/4;'''
def pipe_source_path(native: Path) -> Path:
"""Locate executable pipe formulas, never a modular aggregation entry."""
module = native / 'components/modules/pipe.c'
legacy = native / 'components/kernels.c'
if module.is_file():
target = module
elif (native / 'components/modules').exists():
raise ValueError(f'Incomplete modular native snapshot: {module} is missing')
else:
target = legacy
if not target.is_file():
raise ValueError(f'Native snapshot has no audited pipe source: {native}')
source = target.read_text()
if source.count('double native_pipe_resistance(') != 1 or source.count('double native_pipe_flow(') != 1:
raise ValueError(f'Unsupported pipe source layout: {target}; refusing an unmodified or uninstrumented run')
return target
def working_native_paths() -> list[str]:
return sorted('native/' + path.relative_to(ROOT / 'native').as_posix()
for path in (ROOT / 'native').rglob('*') if path.is_file())
def build_snapshot(program, native: Path, cache: Path, *, required_sources=()):
"""Build exact snapshot sources and verify the modified files were compiled.
Historical monolithic trees retain their original set of C translation
units. This temporary selector override is confined to the standalone
manual process; production source selection is always restored afterwards.
An incompatible old model/runtime ABI fails compilation explicitly.
"""
native = native.resolve()
pipe = pipe_source_path(native)
required = (pipe, *required_sources)
old_native = builder.NATIVE
old_selector = getattr(builder, '_runtime_sources', None)
legacy = pipe.name == 'kernels.c'
try:
builder.NATIVE = native
if legacy and old_selector is not None:
builder._runtime_sources = lambda _: sorted(native.rglob('*.c'))
elif not legacy and old_selector is None:
raise ValueError('A modular snapshot requires the incremental native source selector; use its matching checkout')
selected = (builder._runtime_sources(program) if old_selector is not None
else sorted(native.rglob('*.c')))
if any(path.resolve() not in {source.resolve() for source in selected} for path in required):
raise ValueError('The model/build selector does not compile the modified pipe/RHS source; refusing a misleading run')
try:
build = builder.build_native(program, cache_dir=cache)
except RuntimeError as exc:
raise RuntimeError(
f'Cannot build this native snapshot ({native}). Its source layout or model/runtime ABI may be incompatible; '
'use matching compiler/runtime revisions. No benchmark was executed.'
) from exc
recorded = build.manifest.get('sourceHashes', {})
for path in required:
suffix = 'native/' + path.resolve().relative_to(native).as_posix()
matches = [digest for name, digest in recorded.items()
if name == suffix or name.replace('\\', '/').endswith('/' + suffix)]
if matches != [sha256(path.read_bytes()).hexdigest()]:
if hasattr(build, 'close'):
build.close()
raise ValueError(f'Build manifest does not prove the modified source was compiled: {suffix}')
return build
finally:
builder.NATIVE = old_native
if old_selector is not None:
builder._runtime_sources = old_selector
def main(): def main():
parser=argparse.ArgumentParser(description=__doc__) parser=argparse.ArgumentParser(description=__doc__)
parser.add_argument('input',type=Path) parser.add_argument('input',type=Path)
@@ -42,7 +112,8 @@ def main():
args=parser.parse_args() args=parser.parse_args()
if args.runs<1:parser.error('--runs must be positive') if args.runs<1:parser.error('--runs must be positive')
out=args.output_dir.resolve();out.mkdir(parents=True,exist_ok=True) out=args.output_dir.resolve();out.mkdir(parents=True,exist_ok=True)
if (out/'summary.json').exists():parser.error('Choose a fresh output directory') if (out/'summary.json').exists() or any((out/name).exists() for name in ('fixed-point','previous-newton','guarded-newton')):
parser.error('Choose a fresh output directory; existing variant sources must not be mixed')
revision=subprocess.check_output(['git','rev-parse',args.baseline_ref],cwd=ROOT,text=True).strip() revision=subprocess.check_output(['git','rev-parse',args.baseline_ref],cwd=ROOT,text=True).strip()
xml,doc=load_input(args.input) xml,doc=load_input(args.input)
program=compile_native_program(compile_system_xml_network(doc)) program=compile_native_program(compile_system_xml_network(doc))
@@ -55,16 +126,16 @@ def main():
try: try:
for variant in ('fixed-point','previous-newton','guarded-newton'): for variant in ('fixed-point','previous-newton','guarded-newton'):
directory=out/variant directory=out/variant
for name in paths: variant_paths = working_native_paths() if variant == 'guarded-newton' else paths
for name in variant_paths:
if variant=='guarded-newton':data=(ROOT/name).read_bytes() if variant=='guarded-newton':data=(ROOT/name).read_bytes()
else:data=subprocess.check_output(['git','show',f'{revision}:{name}'],cwd=ROOT) else:data=subprocess.check_output(['git','show',f'{revision}:{name}'],cwd=ROOT)
target=directory/name;target.parent.mkdir(parents=True,exist_ok=True);target.write_bytes(data) target=directory/name;target.parent.mkdir(parents=True,exist_ok=True);target.write_bytes(data)
if variant=='fixed-point': if variant=='fixed-point':
target=directory/'native/components/kernels.c';source=target.read_text() target=pipe_source_path(directory/'native');source=target.read_text()
if source.count(NEW_CALL)!=1:raise ValueError('Baseline pipe flow layout does not match the audited fixed-point substitution') if source.count(NEW_CALL)!=1:raise ValueError('Baseline pipe flow layout does not match the audited fixed-point substitution')
target.write_text(source.replace(NEW_CALL,OLD_ITERATION)) target.write_text(source.replace(NEW_CALL,OLD_ITERATION))
builder.NATIVE=directory/'native' variants[variant]=build_snapshot(program,directory/'native',out/'cache')
variants[variant]=builder.build_native(program,cache_dir=out/'cache')
finally:builder.NATIVE=original_native finally:builder.NATIVE=original_native
rows=[] rows=[]
for index in range(args.runs+1): for index in range(args.runs+1):
@@ -80,7 +151,9 @@ def main():
summary={'baselineRef':revision,'input':str(args.input.resolve()), summary={'baselineRef':revision,'input':str(args.input.resolve()),
'inputSha256':sha256(args.input.read_bytes()).hexdigest(),'xmlSha256':sha256(xml).hexdigest(), 'inputSha256':sha256(args.input.read_bytes()).hexdigest(),'xmlSha256':sha256(xml).hexdigest(),
'settings':vars(config),'sampleStep':doc.simulation.sample_step,'rows':rows, 'settings':vars(config),'sampleStep':doc.simulation.sample_step,'rows':rows,
'variants':{name:{'sourceSha256':sha256((out/name/'native/components/kernels.c').read_bytes()).hexdigest(), 'variants':{name:{'sourceSha256':sha256(pipe_source_path(out/name/'native').read_bytes()).hexdigest(),
'pipeSource':str(pipe_source_path(out/name/'native').relative_to(out/name)),
'nativeSourceHashes':build.manifest['sourceHashes'],
'buildKey':build.manifest['buildKey'], 'buildKey':build.manifest['buildKey'],
'medianSolveSeconds':statistics.median(row['solveSeconds'] for row in rows if row['variant']==name and row['run']>0), 'medianSolveSeconds':statistics.median(row['solveSeconds'] for row in rows if row['variant']==name and row['run']>0),
'medianProcessSeconds':statistics.median(row['processWallSeconds'] for row in rows if row['variant']==name and row['run']>0)} 'medianProcessSeconds':statistics.median(row['processWallSeconds'] for row in rows if row['variant']==name and row['run']>0)}
+18 -3
View File
@@ -220,9 +220,15 @@ def prepare(args: argparse.Namespace) -> dict:
# Reject numerical/runtime drift; a sparse timing-only cached main is allowed # Reject numerical/runtime drift; a sparse timing-only cached main is allowed
# because control and our current writer share the numeric model contract. # because control and our current writer share the numeric model contract.
differences = [] differences = []
for name, expected in manifest["sourceHashes"].items(): recorded_sources = dict(manifest["sourceHashes"])
if manifest.get("cacheVersion", 1) >= 2:
headers = manifest.get("nativeHeaderHashes")
if not isinstance(headers, dict) or not headers:
raise RuntimeError("Cached build lacks native header hashes; rebuild before profiling")
recorded_sources.update(headers)
for name, expected in recorded_sources.items():
relative = name.split("native/", 1)[-1] relative = name.split("native/", 1)[-1]
current = ROOT / "native" / relative current = cache / relative if relative == "model.c" else ROOT / "native" / relative
if digest(current) != expected: if digest(current) != expected:
differences.append(relative) differences.append(relative)
if any(name != "runtime/main.c" for name in differences): if any(name != "runtime/main.c" for name in differences):
@@ -243,7 +249,16 @@ def prepare(args: argparse.Namespace) -> dict:
for name in ("model.c", "model.h", "manifest.json"): for name in ("model.c", "model.h", "manifest.json"):
shutil.copy2(cache / name, output / name) shutil.copy2(cache / name, output / name)
instrument(native) instrument(native)
command = [compiler, *manifest["compilerFlags"], "-I", str(output), "-I", str(native / "include"), "-I", str(sundials / "include"), str(output / "model.c"), *map(str, sorted(native.rglob("*.c"))), "-Wl,--start-group", *map(str, libraries), "-Wl,--end-group", "-lm", "-o", str(output / "profiled-model")] # Match the cached translation units. Compiling the compatibility entry
# together with its new modules would define every component twice.
sources = [native / name.split("native/", 1)[-1]
for name in manifest["sourceHashes"]
if name.endswith(".c") and name != "model.c"]
if not sources or (native / "components/kernels.c" in sources and
any(path.parent == native / "components/modules" for path in sources)):
raise RuntimeError("Cached translation units are missing or mix amalgamation and modules")
sources.append(native / "runtime/compute_profile.c")
command = [compiler, *manifest["compilerFlags"], "-I", str(output), "-I", str(native / "include"), "-I", str(sundials / "include"), str(output / "model.c"), *map(str, sources), "-Wl,--start-group", *map(str, libraries), "-Wl,--end-group", "-lm", "-o", str(output / "profiled-model")]
prepared = {"controlExecutable": str(cache / "model"), "profiledExecutable": str(output / "profiled-model"), "buildCommand": command, "runtimeArguments": numerical_arguments, "verifyJacobian": "--verify-jacobian" in numerical_arguments, "algorithm": "production-automatic", "cacheDir": str(cache), "cachedMainDifference": differences, "stateCount": len(manifest["stateKeys"]), "modelSha256": digest(output / "model.c"), "compiler": compiler_version, "warmups": args.warmups, "repeats": args.repeats, "scope": "Inclusive times overlap. Exclusive scope times partition integration including instrumentation overhead. No component or libc sub-cost inference.", "sourceHashes": {str(p.relative_to(output)): digest(p) for p in sorted(native.rglob("*")) if p.is_file()}} prepared = {"controlExecutable": str(cache / "model"), "profiledExecutable": str(output / "profiled-model"), "buildCommand": command, "runtimeArguments": numerical_arguments, "verifyJacobian": "--verify-jacobian" in numerical_arguments, "algorithm": "production-automatic", "cacheDir": str(cache), "cachedMainDifference": differences, "stateCount": len(manifest["stateKeys"]), "modelSha256": digest(output / "model.c"), "compiler": compiler_version, "warmups": args.warmups, "repeats": args.repeats, "scope": "Inclusive times overlap. Exclusive scope times partition integration including instrumentation overhead. No component or libc sub-cost inference.", "sourceHashes": {str(p.relative_to(output)): digest(p) for p in sorted(native.rglob("*")) if p.is_file()}}
write_json(output / "prepared.json", prepared) write_json(output / "prepared.json", prepared)
return prepared return prepared
+37 -14
View File
@@ -24,7 +24,9 @@ from app.simulation.backends import simulation_config
from app.simulation.native_codegen import build as builder from app.simulation.native_codegen import build as builder
from app.simulation.native_codegen.compiler import compile_native_program from app.simulation.native_codegen.compiler import compile_native_program
from app.simulation.native_codegen.input import load_input from app.simulation.native_codegen.input import load_input
from tests.manual.benchmark_native_pipe_solver import build_snapshot, pipe_source_path, working_native_paths
SNAPSHOT_HELPER = ROOT / 'tests/manual/benchmark_native_pipe_solver.py'
VARIANTS = ('guarded-newton', 'previous-newton', 'fixed-point') VARIANTS = ('guarded-newton', 'previous-newton', 'fixed-point')
FIELDS = '''cache_requests cache_hits cache_misses flow_calls zero_pressure_calls FIELDS = '''cache_requests cache_hits cache_misses flow_calls zero_pressure_calls
pnl00r_analytic_calls resistance_calls scalar_analytic_calls iterative_calls pnl00r_analytic_calls resistance_calls scalar_analytic_calls iterative_calls
@@ -214,7 +216,7 @@ def instrument(source: str, variant: str) -> str:
limit = '(kind==0?64:16)' if fixed else '80' if variant == 'previous-newton' else '128' limit = '(kind==0?64:16)' if fixed else '80' if variant == 'previous-newton' else '128'
prefix = PROFILE_PREFIX.replace('@VARIANT@', variant).replace('@FIXED@', str(int(fixed))) prefix = PROFILE_PREFIX.replace('@VARIANT@', variant).replace('@FIXED@', str(int(fixed)))
prefix = prefix.replace('@LIMIT@', limit).replace('@FIELDS@', ' '.join(f'X({name})' for name in FIELDS)) prefix = prefix.replace('@LIMIT@', limit).replace('@FIELDS@', ' '.join(f'X({name})' for name in FIELDS))
source = replace_once(source, '#include <stddef.h>', '#include <stddef.h>\n' + prefix) source = replace_once(source, '#include <math.h>', '#include <math.h>\n' + prefix)
# Exhaustion is marked at the actual loop fall-through, not inferred from # Exhaustion is marked at the actual loop fall-through, not inferred from
# visiting the last allowed iteration (which can still converge). # visiting the last allowed iteration (which can still converge).
start = source.index('double native_pipe_resistance(') start = source.index('double native_pipe_resistance(')
@@ -259,6 +261,19 @@ def instrument(source: str, variant: str) -> str:
return source return source
def instrument_common(source: str) -> str:
"""Count both ordinary and canonical Jacobian RHS evaluations exactly once."""
calls = [('model_eval', ' return model_eval(t,y,dy,w);')]
if 'int native_jacobian_rhs(' in source:
calls.append(('model_eval_jacobian', ' return model_eval_jacobian(t,y,dy,w);'))
for function, fragment in calls:
source = replace_once(source, fragment,
' extern void pipe_profile_rhs_enter(void),pipe_profile_rhs_leave(void);\n'
f' pipe_profile_rhs_enter();int ok={function}(t,y,dy,w);pipe_profile_rhs_leave();return ok;')
return source
def git(*arguments: str) -> str: def git(*arguments: str) -> str:
return subprocess.check_output(['git', *arguments], cwd=ROOT, text=True).strip() return subprocess.check_output(['git', *arguments], cwd=ROOT, text=True).strip()
@@ -268,11 +283,14 @@ def prepare(args, out: Path) -> dict:
metadata = json.loads((out / 'prepared.json').read_text()) metadata = json.loads((out / 'prepared.json').read_text())
if metadata['input_sha256'] != sha256(args.input.read_bytes()).hexdigest(): if metadata['input_sha256'] != sha256(args.input.read_bytes()).hexdigest():
raise ValueError('Prepared model no longer matches the input file') raise ValueError('Prepared model no longer matches the input file')
if metadata['script_sha256'] != sha256(Path(__file__).read_bytes()).hexdigest(): if (metadata['script_sha256'] != sha256(Path(__file__).read_bytes()).hexdigest()
or metadata.get('snapshot_helper_sha256') != sha256(SNAPSHOT_HELPER.read_bytes()).hexdigest()):
raise ValueError('Diagnostic helper changed; choose a fresh output directory') raise ValueError('Diagnostic helper changed; choose a fresh output directory')
return metadata return metadata
if any((out / variant).exists() for variant in VARIANTS):
raise ValueError('Partially prepared variant directories exist; choose a fresh output directory')
previous = git('rev-parse', args.previous_ref) previous = git('rev-parse', args.previous_ref)
current = git('rev-parse', args.current_ref) current = 'working-tree' if args.current_ref == 'working-tree' else git('rev-parse', args.current_ref)
xml, doc = load_input(args.input) xml, doc = load_input(args.input)
config = simulation_config(doc.simulation) config = simulation_config(doc.simulation)
if config.rtol != 1e-8 or config.t_start != 0 or config.t_stop != 10: if config.rtol != 1e-8 or config.t_start != 0 or config.t_stop != 10:
@@ -281,37 +299,40 @@ def prepare(args, out: Path) -> dict:
out.mkdir(parents=True, exist_ok=True) out.mkdir(parents=True, exist_ok=True)
(out / 'input.xml').write_bytes(xml) (out / 'input.xml').write_bytes(xml)
(out / 'input.json').write_bytes(args.input.read_bytes()) (out / 'input.json').write_bytes(args.input.read_bytes())
paths = git('ls-tree', '-r', '--name-only', current, 'native').splitlines()
metadata = dict(input=str(args.input.resolve()), input_sha256=sha256(args.input.read_bytes()).hexdigest(), metadata = dict(input=str(args.input.resolve()), input_sha256=sha256(args.input.read_bytes()).hexdigest(),
xml_sha256=sha256(xml).hexdigest(), script_sha256=sha256(Path(__file__).read_bytes()).hexdigest(), xml_sha256=sha256(xml).hexdigest(), script_sha256=sha256(Path(__file__).read_bytes()).hexdigest(),
snapshot_helper_sha256=sha256(SNAPSHOT_HELPER.read_bytes()).hexdigest(),
previous_revision=previous, current_revision=current, settings=vars(config), previous_revision=previous, current_revision=current, settings=vars(config),
sample_step=doc.simulation.sample_step, variants={}, sample_step=doc.simulation.sample_step, variants={},
measurement_note='Instrumented times are diagnostic overhead and MUST NOT be used as production benchmark results.', measurement_note='Instrumented times are diagnostic overhead and MUST NOT be used as production benchmark results.',
counter_scope='rhs counts native_rhs/model_eval calls including rejected trials and Jacobian differences; non_rhs includes initialization/output/probe evaluations.', counter_scope='rhs counts native_rhs/model_eval and, where present, native_jacobian_rhs/model_eval_jacobian calls including rejected trials and Jacobian differences; non_rhs includes initialization/output/probe evaluations.',
replay_scope='All guarded RHS resistance calls, including scalar analytic low-Re cases; no sampling or deduplication. Cache hits, zero pressure difference and direct PNL00R analytic calls are excluded and counted separately.', replay_scope='All guarded RHS resistance calls, including scalar analytic low-Re cases; no sampling or deduplication. Cache hits, zero pressure difference and direct PNL00R analytic calls are excluded and counted separately.',
capture_format='Native-endian IEEE-754 binary64 records: base, diameter, length, relative roughness, den=pi*d*mu, kind (six doubles, 48 bytes). Replay on the same host.', capture_format='Native-endian IEEE-754 binary64 records: base, diameter, length, relative roughness, den=pi*d*mu, kind (six doubles, 48 bytes). Replay on the same host.',
residual_test='At actual returned q, independently factored long-double f(Re) evaluates abs(Re^2*f(Re)/K-1)<=1e-9. Separate from each algorithm stopping rule.') residual_test='At actual returned q, independently factored long-double f(Re) evaluates abs(Re^2*f(Re)/K-1)<=1e-9. Separate from each algorithm stopping rule.')
original_native = builder.NATIVE original_native = builder.NATIVE
prepared_builds = [] # Pin earlier variants while later variants are built.
try: try:
for variant in VARIANTS: for variant in VARIANTS:
directory = out / variant directory = out / variant
revision = current if variant == 'guarded-newton' else previous revision = current if variant == 'guarded-newton' else previous
paths = (working_native_paths() if revision == 'working-tree'
else git('ls-tree', '-r', '--name-only', revision, 'native').splitlines())
for name in paths: for name in paths:
target = directory / name target = directory / name
target.parent.mkdir(parents=True, exist_ok=True) target.parent.mkdir(parents=True, exist_ok=True)
target.write_bytes(subprocess.check_output(['git', 'show', f'{revision}:{name}'], cwd=ROOT)) data = ((ROOT / name).read_bytes() if revision == 'working-tree'
kernel = directory / 'native/components/kernels.c' else subprocess.check_output(['git', 'show', f'{revision}:{name}'], cwd=ROOT))
target.write_bytes(data)
kernel = pipe_source_path(directory / 'native')
original_hash = sha256(kernel.read_bytes()).hexdigest() original_hash = sha256(kernel.read_bytes()).hexdigest()
kernel.write_text(instrument(kernel.read_text(), variant)) kernel.write_text(instrument(kernel.read_text(), variant))
common = directory / 'native/runtime/common.c' common = directory / 'native/runtime/common.c'
common.write_text(replace_once(common.read_text(), ' return model_eval(t,y,dy,w);', common.write_text(instrument_common(common.read_text()))
''' extern void pipe_profile_rhs_enter(void),pipe_profile_rhs_leave(void);
pipe_profile_rhs_enter();int ok=model_eval(t,y,dy,w);pipe_profile_rhs_leave();return ok;'''))
main = directory / 'native/runtime/main.c' main = directory / 'native/runtime/main.c'
main.write_text(replace_once(main.read_text(), 'int main(int argc, char **argv) {', main.write_text(replace_once(main.read_text(), 'int main(int argc, char **argv) {',
'int main(int argc, char **argv) {\n extern void pipe_profile_install(void);pipe_profile_install();')) 'int main(int argc, char **argv) {\n extern void pipe_profile_install(void);pipe_profile_install();'))
builder.NATIVE = directory / 'native' build = build_snapshot(program, directory / 'native', out / 'cache', required_sources=(common, main))
build = builder.build_native(program, cache_dir=out / 'cache') prepared_builds.append(build)
replay_source = directory / 'replay.c' replay_source = directory / 'replay.c'
replay_source.write_text(REPLAY_MAIN) replay_source.write_text(REPLAY_MAIN)
replay = directory / ('replay.exe' if os.name == 'nt' else 'replay') replay = directory / ('replay.exe' if os.name == 'nt' else 'replay')
@@ -323,7 +344,8 @@ def prepare(args, out: Path) -> dict:
if compiled.returncode: if compiled.returncode:
raise RuntimeError(f'Replay compilation failed: {compiled.stderr}') raise RuntimeError(f'Replay compilation failed: {compiled.stderr}')
metadata['variants'][variant] = dict(executable=str(build.executable), replay=str(replay), metadata['variants'][variant] = dict(executable=str(build.executable), replay=str(replay),
build_key=build.manifest['buildKey'], original_kernel_sha256=original_hash, build_key=build.manifest['buildKey'], pipe_source=str(kernel.relative_to(directory)),
compiled_source_hashes=build.manifest['sourceHashes'], original_kernel_sha256=original_hash,
instrumented_kernel_sha256=sha256(kernel.read_bytes()).hexdigest()) instrumented_kernel_sha256=sha256(kernel.read_bytes()).hexdigest())
print(f'Prepared {variant}', flush=True) print(f'Prepared {variant}', flush=True)
finally: finally:
@@ -423,7 +445,8 @@ def main():
parser.add_argument('--input', type=Path, default=ROOT / 'tests/data/test-mql-8-corrected.json') parser.add_argument('--input', type=Path, default=ROOT / 'tests/data/test-mql-8-corrected.json')
parser.add_argument('--output-dir', type=Path, required=True) parser.add_argument('--output-dir', type=Path, required=True)
parser.add_argument('--previous-ref', default='5d5a2e1') parser.add_argument('--previous-ref', default='5d5a2e1')
parser.add_argument('--current-ref', default='808c484') parser.add_argument('--current-ref', default='808c484',
help='Audited guarded-solver revision, or working-tree for the current modular sources')
parser.add_argument('--timeout', type=float, default=120) parser.add_argument('--timeout', type=float, default=120)
parser.add_argument('--prepare-only', action='store_true') parser.add_argument('--prepare-only', action='store_true')
args = parser.parse_args() args = parser.parse_args()
+25
View File
@@ -0,0 +1,25 @@
"""Read the diagnostic kernel translation unit before test-only injection.
Only component module C includes are expanded. Header includes and the
amalgamation linkage switch remain for the ordinary C preprocessor to handle.
An older, unsplit kernels.c is returned unchanged.
"""
from pathlib import Path
import re
NATIVE = Path(__file__).resolve().parents[1] / "native"
_SOURCE_INCLUDE = re.compile(r'^[ \t]*#[ \t]*include[ \t]+"([^"\n]+\.c)"[ \t]*$', re.MULTILINE)
def native_kernel_source(native: Path = NATIVE) -> str:
entry = native / "components/kernels.c"
modules = (native / "components/modules").resolve()
def expand(match: re.Match[str]) -> str:
source = (entry.parent / match[1]).resolve()
if not source.is_relative_to(modules):
return match[0]
return source.read_text(encoding="utf-8")
return _SOURCE_INCLUDE.sub(expand, entry.read_text(encoding="utf-8"))
+528
View File
@@ -0,0 +1,528 @@
"""Exercise the native build cache with real, tiny C translation units.
These tests isolate their sources, libraries and caches in a temporary directory.
They execute the linked programs to check behavior, and observe compiler commands
to distinguish actual object reuse from a reported cache hit.
"""
from __future__ import annotations
from concurrent.futures import ThreadPoolExecutor
from contextlib import ExitStack
from dataclasses import replace
from hashlib import sha256
import json
import os
from pathlib import Path
import shutil
import subprocess
import sys
import tempfile
import threading
import unittest
from unittest.mock import patch
from app.simulation.native_codegen import build as native_build
from app.simulation.native_codegen.compiler import NativeProgram
from app.simulation.native_codegen.cache_storage import prune_cache
from app.simulation.native_codegen.runner import execute_native
from app.simulation.config import SolveIVPConfig
@unittest.skipUnless(sys.platform.startswith("linux") and shutil.which("gcc"),
"Small real-compiler fixtures currently require Linux gcc")
class NativeBuildCacheTests(unittest.TestCase):
def setUp(self):
self.temporary = tempfile.TemporaryDirectory(prefix="native-build-cache-tests-")
self.addCleanup(self.temporary.cleanup)
self.root = Path(self.temporary.name)
self.native = self.root / "native"
self.cache = self.root / "cache"
self.sundials = self.root / "sundials"
self.compiler = str(Path(shutil.which("gcc")).resolve())
self.real_run = subprocess.run
self.compiler_version = self.real_run(
[self.compiler, "--version"], capture_output=True, text=True,
check=True, timeout=10,
).stdout.splitlines()[0]
self.commands: list[tuple[str, ...]] = []
self.command_lock = threading.Lock()
self._write("native/THIRD_PARTY_NOTICES.txt", "Fixture code belongs to this test.\n")
self._write("native/include/values.h", '#include "nested/value.h"\n')
self._write("native/include/nested/value.h", "#define COMMON_VALUE 3\n")
self._write("native/include/signal_config.h", "#define SIGNAL_VALUE 5\n")
self._write("native/runtime/main.c", '''#include "model.h"
#include <stdio.h>
int common_value(void);
int main(void) {
printf("%.17g\\n", model_value() + common_value() + HEADER_OFFSET);
return 0;
}
''')
self._write("native/runtime/common.c", '''#include "values.h"
int common_value(void) { return COMMON_VALUE; }
''')
for name in ("rk45", "cvode_solver", "json_numbers"):
self._write(f"native/runtime/{name}.c", "/* unused runtime fixture */\n")
self._write("native/encoding/ryu/d2s.c", "/* unused encoder fixture */\n")
self._write("native/components/modules/signal.c", '''#include "signal_config.h"
double native_signal(void) { return SIGNAL_VALUE; }
''')
for name in ("properties", "pipe", "orifice", "mechanics"):
self._write(f"native/components/modules/{name}.c", "/* unneeded component */\n")
self._write("sundials/include/cvode/cvode.h", "/* fixture */\n")
self._write("sundials/include/sundials/sundials_config.h", "/* fixture */\n")
# An empty archive is sufficient: the fixture does not call SUNDIALS.
for name in native_build.LIBRARIES:
self._write(f"sundials/lib/libsundials_{name}.a", b"!<arch>\n")
self.program = NativeProgram(
source='#include "model.h"\ndouble model_value(void) { return native_signal() + 1; }\n',
header="#define HEADER_OFFSET 0\ndouble model_value(void);\ndouble native_signal(void);\n",
state_keys=(), variables=(), component_types=(),
)
self.patches = ExitStack()
self.addCleanup(self.patches.close)
self.patches.enter_context(patch.object(native_build, "ROOT", self.root))
self.patches.enter_context(patch.object(native_build, "NATIVE", self.native))
self.patches.enter_context(patch.object(
native_build, "toolchain", return_value=(self.compiler, self.sundials, self.compiler_version),
))
self.patches.enter_context(patch.object(native_build.subprocess, "run", side_effect=self._run))
def _write(self, relative: str, content: str | bytes) -> Path:
path = self.root / relative
path.parent.mkdir(parents=True, exist_ok=True)
if isinstance(content, bytes):
path.write_bytes(content)
else:
path.write_text(content, encoding="utf-8")
return path
def _run(self, command, *args, **kwargs):
with self.command_lock:
self.commands.append(tuple(map(str, command)))
return self.real_run(command, *args, **kwargs)
def _build(self, program=None):
return native_build.build_native(program or self.program, cache_dir=self.cache)
def _compiled_count(self) -> int:
return sum("-c" in command for command in self.commands)
def _linked_count(self) -> int:
return sum("-o" in command and "-c" not in command and "-E" not in command
for command in self.commands)
def _output(self, build) -> float:
completed = self.real_run([str(build.executable)], capture_output=True, text=True,
check=True, timeout=10)
return float(completed.stdout.strip())
def _parameter_program(self, value: int):
return replace(self.program, source=self.program.source.replace(" + 1;", f" + {value};"))
def _objects(self):
return sorted(path for path in (self.cache / "objects").iterdir()
if len(path.name) == 64 and (path / "manifest.json").is_file())
def _object_for(self, suffix: str) -> Path:
matches = [path for path in self._objects()
if json.loads((path / "manifest.json").read_text())["sourceName"].endswith(suffix)]
self.assertEqual(len(matches), 1, suffix)
return matches[0]
def test_complete_hit_executes_same_program_without_compilation_or_linking(self):
first = self._build()
self.assertFalse(first.cache_hit)
self.assertEqual(self._output(first), 9)
self.assertGreater(self._compiled_count(), 1)
self.commands.clear()
second = self._build()
self.assertTrue(second.cache_hit)
self.assertEqual(first.manifest["buildKey"], second.manifest["buildKey"])
self.assertEqual(self._output(second), 9)
self.assertEqual(self._compiled_count(), 0)
self.assertEqual(self._linked_count(), 0)
def test_parameter_edit_reuses_shared_objects_and_recompiles_only_model(self):
first = self._build()
self.commands.clear()
second = self._build(self._parameter_program(11))
self.assertFalse(second.cache_hit)
self.assertNotEqual(first.manifest["buildKey"], second.manifest["buildKey"])
self.assertEqual(self._output(second), 19)
self.assertEqual(self._compiled_count(), 1)
self.assertEqual(self._linked_count(), 1)
self.commands.clear()
self.assertTrue(self._build().cache_hit)
self.assertEqual(self._compiled_count(), 0)
def test_model_header_edit_invalidates_only_translation_units_that_include_it(self):
self._build()
self.commands.clear()
changed = replace(self.program, header=self.program.header.replace("OFFSET 0", "OFFSET 7"))
result = self._build(changed)
self.assertEqual(self._output(result), 16)
# Both include model.h, but the model TU's preprocessed tokens are
# unchanged because it does not use HEADER_OFFSET. Only main must rebuild.
self.assertEqual(self._compiled_count(), 1)
def test_model_header_declaration_edit_rebuilds_all_actual_dependents(self):
self._build()
self.commands.clear()
changed = replace(self.program, header=self.program.header + "extern int unused_model_state;\n")
result = self._build(changed)
self.assertEqual(self._output(result), 9)
self.assertEqual(self._compiled_count(), 2)
def test_indirect_header_change_invalidates_the_actual_runtime_object(self):
first = self._build()
self.commands.clear()
self._write("native/include/nested/value.h", "#define COMMON_VALUE 13\n")
second = self._build()
self.assertFalse(second.cache_hit)
self.assertNotEqual(first.manifest["buildKey"], second.manifest["buildKey"])
self.assertEqual(self._output(second), 19)
self.assertEqual(self._compiled_count(), 1)
def test_unneeded_modules_and_headers_do_not_invalidate_or_enter_the_build(self):
first = self._build()
self.commands.clear()
self._write("native/components/modules/pipe.c", "this is deliberately invalid C\n")
self._write("native/components/modules/new_unused.c", "another deliberately invalid unit\n")
self._write("native/include/unused_new_header.h", "#error should never be included\n")
second = self._build()
self.assertTrue(second.cache_hit)
self.assertEqual(first.manifest["buildKey"], second.manifest["buildKey"])
self.assertEqual(self._output(second), 9)
self.assertEqual(self._compiled_count(), 0)
def test_used_module_change_rebuilds_one_object_and_changes_executable(self):
first = self._build()
self.commands.clear()
self._write("native/include/signal_config.h", "#define SIGNAL_VALUE 15\n")
second = self._build()
self.assertNotEqual(first.manifest["buildKey"], second.manifest["buildKey"])
self.assertEqual(self._output(second), 19)
self.assertEqual(self._compiled_count(), 1)
def test_compile_flags_and_compiler_identity_cannot_reuse_old_objects(self):
first = self._build()
original_count = self._compiled_count()
self.commands.clear()
with patch.object(native_build, "COMPILER_FLAGS", (*native_build.COMPILER_FLAGS, "-DTEST_CACHE_VARIANT=1")):
flags_build = self._build()
self.assertNotEqual(first.manifest["buildKey"], flags_build.manifest["buildKey"])
self.assertEqual(self._compiled_count(), original_count)
self.assertEqual(self._output(flags_build), 9)
self.commands.clear()
with patch.object(native_build, "toolchain", return_value=(self.compiler, self.sundials, self.compiler_version + " test revision")):
compiler_build = self._build()
self.assertNotEqual(first.manifest["buildKey"], compiler_build.manifest["buildKey"])
self.assertEqual(self._compiled_count(), original_count)
self.assertEqual(self._output(compiler_build), 9)
def test_library_change_relinks_but_reuses_compiled_objects(self):
first = self._build()
self.commands.clear()
# Valid ar symbol-table member with no entries, changing bytes only.
empty_object = self.root / "library-member.c"
empty_object.write_text("int library_fixture_symbol(void) { return 1; }\n")
object_path = self.root / "library-member.o"
self.real_run([self.compiler, "-c", str(empty_object), "-o", str(object_path)], check=True)
self.real_run(["ar", "r", str(self.sundials / "lib/libsundials_core.a"), str(object_path)],
capture_output=True, check=True)
second = self._build()
self.assertNotEqual(first.manifest["buildKey"], second.manifest["buildKey"])
self.assertEqual(self._compiled_count(), 0)
self.assertEqual(self._linked_count(), 1)
self.assertEqual(self._output(second), 9)
def test_model_manifest_requires_identity_and_complete_safe_artifacts(self):
built = self._build()
manifest_path = built.executable.parent / "manifest.json"
original = manifest_path.read_text()
manifest = json.loads(original)
mutations = {
"empty artifacts": lambda value: value.update(artifacts={}),
"missing executable": lambda value: value["artifacts"].pop(built.executable.name),
"missing generated header": lambda value: value["artifacts"].pop("model.h"),
"wrong build key": lambda value: value.update(buildKey="0" * 64),
"wrong cache version": lambda value: value.update(cacheVersion=-1),
"wrong object identities": lambda value: value.update(objectKeys=["0" * 64]),
"wrong result contract": lambda value: value.update(stateKeys=["unexpected.state"]),
"parent traversal": lambda value: value["artifacts"].update({"../outside": "0" * 64}),
"absolute path": lambda value: value["artifacts"].update({str(self.root / "outside"): "0" * 64}),
}
for label, mutate in mutations.items():
with self.subTest(label=label):
modified = json.loads(original)
mutate(modified)
manifest_path.write_text(json.dumps(modified))
try:
with self.assertRaises(RuntimeError):
self._build()
finally:
manifest_path.write_text(original)
self.assertEqual(manifest["cacheVersion"], 2)
self.assertTrue(self._build().cache_hit)
def test_model_source_cannot_be_replaced_by_updating_its_artifact_digest(self):
built = self._build()
source = built.executable.parent / "model.c"
source.write_text(self._parameter_program(12).source)
manifest_path = built.executable.parent / "manifest.json"
manifest = json.loads(manifest_path.read_text())
manifest["artifacts"]["model.c"] = sha256(source.read_bytes()).hexdigest()
manifest_path.write_text(json.dumps(manifest))
with self.assertRaises(RuntimeError):
self._build()
def test_changed_executable_and_artifact_symlink_are_rejected(self):
built = self._build()
executable = built.executable
original = executable.read_bytes()
executable.write_bytes(original + b"corrupt")
with self.assertRaises(RuntimeError):
self._build()
executable.write_bytes(original)
target = self.root / "outside-executable"
target.write_bytes(original)
executable.unlink()
executable.symlink_to(target)
with self.assertRaises(RuntimeError):
self._build()
def test_object_bytes_and_object_manifest_are_checked_when_reused(self):
first = self._build()
object_dir = self._object_for("signal.c")
unit = object_dir / "unit.o"
manifest_path = object_dir / "manifest.json"
original_object = unit.read_bytes()
original_manifest = manifest_path.read_text()
changed_program = self._parameter_program(2)
unit.write_bytes(original_object + b"corrupt")
# Complete executable reuse need not inspect disposable intermediate objects.
self.assertTrue(self._build().cache_hit)
with self.assertRaises(RuntimeError):
self._build(changed_program)
unit.write_bytes(original_object)
for label, mutate in {
"empty artifacts": lambda value: value.update(artifacts={}),
"wrong key": lambda value: value.update(objectKey="0" * 64),
"wrong version": lambda value: value.update(cacheVersion=-1),
"wrong source": lambda value: value.update(sourceName="native/another.c"),
"wrong preprocessing": lambda value: value.update(preprocessedSha256="0" * 64),
"wrong compiler": lambda value: value.update(compiler={}),
"parent traversal": lambda value: value["artifacts"].update({"../unit.o": "0" * 64}),
}.items():
with self.subTest(label=label):
modified = json.loads(original_manifest)
mutate(modified)
manifest_path.write_text(json.dumps(modified))
try:
with self.assertRaises(RuntimeError):
self._build(changed_program)
finally:
manifest_path.write_text(original_manifest)
target = self.root / "outside-unit.o"
target.write_bytes(original_object)
unit.unlink()
unit.symlink_to(target)
with self.assertRaises(RuntimeError):
self._build(changed_program)
self.assertEqual(self._output(first), 9)
def test_library_change_during_link_is_rejected_before_publication(self):
first = self._build()
before = {path.name for path in (self.cache / "models").iterdir() if len(path.name) == 64}
source = self._write("replacement-library.c", "int replacement_library(void) { return 7; }\n")
object_path = self.root / "replacement-library.o"
alternate = self.root / "alternate.a"
self.real_run([self.compiler, "-c", str(source), "-o", str(object_path)], check=True)
self.real_run(["ar", "rc", str(alternate), str(object_path)], capture_output=True, check=True)
library = self.sundials / "lib/libsundials_core.a"
original = library.read_bytes()
linked_successfully = []
def replace_library_at_link(command, *args, **kwargs):
is_link = "-o" in command and "-c" not in command and "-E" not in command
if is_link:
library.write_bytes(alternate.read_bytes())
result = self._run(command, *args, **kwargs)
if is_link:
linked_successfully.append(result.returncode == 0)
return result
try:
with patch.object(native_build.subprocess, "run", side_effect=replace_library_at_link):
with self.assertRaises(RuntimeError):
self._build(self._parameter_program(2))
self.assertEqual(linked_successfully, [True])
published = {path.name for path in (self.cache / "models").iterdir() if len(path.name) == 64}
self.assertEqual(published, before)
self.assertEqual(self._output(first), 9)
finally:
library.write_bytes(original)
self.assertTrue(self._build().cache_hit)
def test_failed_compile_or_link_does_not_publish_a_model_or_break_old_cache(self):
first = self._build()
before = {path.name for path in (self.cache / "models").iterdir() if len(path.name) == 64}
for source in (
self.program.source + "this is invalid C;\n",
self.program.source + "int unresolved(void); int force_link_failure(void) { return unresolved(); }\n",
):
with self.subTest(source=source):
with self.assertRaises(RuntimeError):
self._build(replace(self.program, source=source))
published = {path.name for path in (self.cache / "models").iterdir() if len(path.name) == 64}
self.assertEqual(published, before)
self.assertTrue(self._build().cache_hit)
self.assertEqual(self._output(first), 9)
def test_concurrent_cold_builds_publish_complete_models_and_share_objects(self):
gate = threading.Barrier(2)
def build_together(program):
gate.wait(timeout=10)
return self._build(program)
with ThreadPoolExecutor(max_workers=2) as pool:
futures = [pool.submit(build_together, self.program) for _ in range(2)]
first, second = [future.result(timeout=60) for future in futures]
self.assertEqual(first.executable, second.executable)
self.assertEqual(self._output(first), 9)
self.assertEqual(self._output(second), 9)
self.assertTrue(self._build().cache_hit)
gate = threading.Barrier(2)
with ThreadPoolExecutor(max_workers=2) as pool:
futures = [pool.submit(build_together, self._parameter_program(value)) for value in (2, 3)]
changed = [future.result(timeout=60) for future in futures]
self.assertEqual([self._output(item) for item in changed], [10, 11])
self.assertNotEqual(changed[0].manifest["buildKey"], changed[1].manifest["buildKey"])
for path in self._objects():
self.assertTrue((path / "unit.o").is_file())
for item in (first, *changed):
manifest = json.loads((item.executable.parent / "manifest.json").read_text())
self.assertEqual(manifest["buildKey"], item.manifest["buildKey"])
self.assertTrue(all((item.executable.parent / name).is_file() for name in manifest["artifacts"]))
def test_generated_sources_keep_exact_lf_under_windows_default_translation(self):
write_text = Path.write_text
def windows_write_text(path, data, encoding=None, errors=None, newline=None):
# Python text output on Windows expands LF when newline is omitted.
# Emulate that default on Linux while honoring explicit newline="\n".
if newline is None:
data = data.replace("\n", "\r\n")
return write_text(path, data, encoding=encoding, errors=errors, newline="\n")
with patch.object(Path, "write_text", windows_write_text):
probe = self._write("platform-newline-probe.txt", "first\nsecond\n")
self.assertEqual(probe.read_bytes(), b"first\r\nsecond\r\n")
first = self._build()
self.assertEqual((first.executable.parent / "model.c").read_bytes(), self.program.source.encode())
self.assertEqual((first.executable.parent / "model.h").read_bytes(), self.program.header.encode())
self.assertEqual(self._output(first), 9)
self.commands.clear()
second = self._build()
self.assertTrue(second.cache_hit)
self.assertEqual(second.manifest["buildKey"], first.manifest["buildKey"])
self.assertEqual(self._output(second), 9)
self.assertEqual(self._compiled_count(), 0)
self.assertEqual(self._linked_count(), 0)
def test_capacity_sweep_protects_build_leases_and_close_allows_eviction(self):
older = self._build()
newer = self._build(self._parameter_program(2))
try:
report = prune_cache(self.cache, model_limit_bytes=0)
self.assertGreaterEqual(report["models"]["skippedInUse"], 2)
self.assertTrue(older.executable.is_file())
self.assertTrue(newer.executable.is_file())
self.assertEqual(self._output(older), 9)
# Release the newer lease first under the normal budget, avoiding
# any assumption that two model keys occupy distinct lock shards.
newer.close()
with patch.dict(os.environ, {"SIMULATION_NATIVE_MODEL_CACHE_MB": "0"}):
older.close()
self.assertFalse(older.executable.exists())
self.assertTrue(newer.executable.is_file())
self.assertEqual(self._output(newer), 10)
report = prune_cache(self.cache, model_limit_bytes=0)
self.assertEqual(report["models"]["removedEntries"], 0)
self.assertEqual(report["models"]["oversizedEntries"], 1)
self.assertGreater(report["models"]["overLimitBytes"], 0)
finally:
older.close()
newer.close()
def test_executing_native_worker_keeps_its_model_during_capacity_sweep(self):
self._write("native/runtime/main.c", r"""#include "model.h"
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <time.h>
int common_value(void);
int main(int argc, char **argv) {
const char *output = NULL;
const char *release = getenv("NATIVE_CACHE_TEST_RELEASE");
for (int i = 1; i + 1 < argc; ++i)
if (strcmp(argv[i], "--output") == 0) output = argv[i + 1];
if (!output || !release) return 64;
fprintf(stderr, "{\"phase\":\"integrating\",\"time\":0,\"nfev\":0,\"acceptedSteps\":0}\n");
fflush(stderr);
struct timespec pause = {0, 1000000};
int released = 0;
for (int i = 0; i < 10000; ++i) {
FILE *gate = fopen(release, "r");
if (gate) { fclose(gate); released = 1; break; }
nanosleep(&pause, NULL);
}
if (!released) return 70;
FILE *result = fopen(output, "w");
if (!result) return 74;
int written = fprintf(result,
"{\"simulatedUntil\":0.1,\"nfev\":1,\"acceptedSteps\":1,\"value\":%.17g}",
model_value() + common_value() + HEADER_OFFSET);
return fclose(result) != 0 || written < 0;
}
""")
active = self._build()
other = self._build(self._parameter_program(2))
release = self.root / "release-worker"
worker_running = threading.Event()
try:
with patch.dict(os.environ, {"NATIVE_CACHE_TEST_RELEASE": str(release)}):
with ThreadPoolExecutor(max_workers=1) as pool:
future = pool.submit(
execute_native, active, SolveIVPConfig(t_stop=0.1), 0.1,
run_dir=self.root / "worker-result", timeout=10,
progress_callback=lambda fraction, phase: worker_running.set(),
)
try:
self.assertTrue(worker_running.wait(timeout=5), "Native worker did not report readiness")
self.assertFalse(future.done())
report = prune_cache(self.cache, model_limit_bytes=0)
self.assertGreaterEqual(report["models"]["skippedInUse"], 2)
self.assertTrue(active.executable.is_file())
self.assertTrue((active.executable.parent / "manifest.json").is_file())
finally:
release.write_text("finish\n")
payload = future.result(timeout=15)
self.assertEqual(payload["value"], 9)
self.assertEqual(payload["buildKey"], active.manifest["buildKey"])
other.close()
with patch.dict(os.environ, {"SIMULATION_NATIVE_MODEL_CACHE_MB": "0"}):
active.close()
self.assertFalse(active.executable.exists())
self.assertTrue(other.executable.is_file())
finally:
release.write_text("finish\n")
active.close()
other.close()
if __name__ == "__main__":
unittest.main()
+376
View File
@@ -0,0 +1,376 @@
"""Use/eviction races and byte budgets for the immutable native build cache."""
from __future__ import annotations
import gc
import json
import os
from pathlib import Path
import subprocess
import sys
import tempfile
import unittest
from unittest.mock import patch
from app.simulation.native_codegen import cache_storage as storage
ROOT = Path(__file__).resolve().parents[1]
def key(number: int) -> str:
# Distinct first words make shard collisions explicit in the tests.
return f"{number:08x}" + "0" * 56
class NativeCacheStorageTests(unittest.TestCase):
def setUp(self):
self.temporary = tempfile.TemporaryDirectory(prefix="native-cache-storage-")
self.addCleanup(self.temporary.cleanup)
self.folder = Path(self.temporary.name)
self.cache = self.folder / "cache"
def entry(self, number: int, size: int, age: int, kind: str = "models") -> Path:
path = self.cache / kind / key(number)
path.mkdir(parents=True)
(path / "artifact").write_bytes(b"x" * size)
os.utime(path, ns=(1_000_000_000 + age, 1_000_000_000 + age))
return path
def lease(self, number: int, **kwargs):
lease = storage.acquire_cache_lease(self.cache, "models", key(number), **kwargs)
if lease is not None:
self.addCleanup(lease.close)
return lease
def test_overlapping_shared_leases_and_nonblocking_exclusion(self):
first = self.lease(0)
second = self.lease(0, blocking=False)
self.assertIsNotNone(second)
self.assertIsNone(self.lease(0, exclusive=True, blocking=False))
first.close()
self.assertTrue(first.closed)
self.assertIsNone(self.lease(0, exclusive=True, blocking=False))
second.close()
writer = self.lease(0, exclusive=True, blocking=False)
self.assertIsNotNone(writer)
self.assertIsNone(self.lease(0, blocking=False))
writer.close()
writer.close() # Explicit close and cleanup are idempotent.
with self.assertRaises(RuntimeError):
with writer:
pass
def test_separate_process_sees_shared_and_exclusive_lock_contract(self):
self.lease(1)
script = r'''
import json, sys
from pathlib import Path
from app.simulation.native_codegen.cache_storage import acquire_cache_lease
cache, key = Path(sys.argv[1]), sys.argv[2]
shared = acquire_cache_lease(cache, 'models', key, blocking=False)
exclusive = acquire_cache_lease(cache, 'models', key, exclusive=True, blocking=False)
print(json.dumps({'shared': shared is not None, 'exclusive': exclusive is not None}))
if shared: shared.close()
if exclusive: exclusive.close()
'''
result = subprocess.run(
[sys.executable, "-c", script, str(self.cache), key(1)], cwd=ROOT,
capture_output=True, text=True, timeout=15, check=True,
)
self.assertEqual(json.loads(result.stdout), {"shared": True, "exclusive": False})
def test_blocking_process_proceeds_when_last_reader_releases(self):
lease = self.lease(2)
script = r'''
import sys
from pathlib import Path
from app.simulation.native_codegen.cache_storage import acquire_cache_lease
print('waiting', flush=True)
with acquire_cache_lease(Path(sys.argv[1]), 'models', sys.argv[2], exclusive=True):
print('acquired', flush=True)
'''
process = subprocess.Popen(
[sys.executable, "-c", script, str(self.cache), key(2)], cwd=ROOT,
stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True,
)
self.addCleanup(lambda: process.kill() if process.poll() is None else None)
self.assertEqual(process.stdout.readline().strip(), "waiting")
self.assertIsNone(process.poll())
lease.close()
stdout, stderr = process.communicate(timeout=15)
self.assertEqual(process.returncode, 0, stderr)
self.assertEqual(stdout.strip(), "acquired")
def test_finalizer_releases_abandoned_lease(self):
lease = storage.acquire_cache_lease(self.cache, "models", key(3))
self.assertIsNone(self.lease(3, exclusive=True, blocking=False))
del lease
gc.collect()
self.assertIsNotNone(self.lease(3, exclusive=True, blocking=False))
def test_lru_removes_oldest_until_separate_budgets_are_met(self):
oldest = self.entry(0, 60, 1)
middle = self.entry(1, 60, 2)
newest = self.entry(2, 60, 3)
object_old = self.entry(3, 40, 1, "objects")
object_new = self.entry(4, 40, 2, "objects")
report = storage.prune_cache(self.cache, model_limit_bytes=120, object_limit_bytes=40)
self.assertFalse(oldest.exists())
self.assertTrue(middle.exists() and newest.exists())
self.assertFalse(object_old.exists())
self.assertTrue(object_new.exists())
self.assertEqual(report["models"]["afterBytes"], 120)
self.assertEqual(report["objects"]["afterBytes"], 40)
self.assertEqual(report["models"]["removedEntries"], 1)
self.assertEqual(report["models"]["errors"], [])
def test_held_entry_survives_and_recent_touch_changes_lru_order(self):
first = self.entry(0, 60, 1)
second = self.entry(1, 60, 2)
third = self.entry(2, 60, 3)
held = self.lease(0)
with self.lease(1):
storage.touch_cache_entry(self.cache, "models", key(1))
report = storage.prune_cache(self.cache, model_limit_bytes=60)
self.assertTrue(first.exists())
self.assertFalse(second.exists() or third.exists())
self.assertEqual(report["models"]["skippedInUse"], 1)
held.close()
# A separate round verifies touch order without an active-entry override.
old = self.entry(3, 60, 3)
with self.lease(0):
storage.touch_cache_entry(self.cache, "models", key(0))
storage.prune_cache(self.cache, model_limit_bytes=60)
self.assertTrue(first.exists())
self.assertFalse(old.exists())
def test_in_use_overage_is_reported_and_later_sweep_recovers(self):
first = self.entry(0, 60, 1)
second = self.entry(1, 60, 2)
a, b = self.lease(0), self.lease(1)
report = storage.prune_cache(self.cache, model_limit_bytes=60)
self.assertTrue(first.exists() and second.exists())
self.assertEqual(report["models"]["overLimitBytes"], 60)
self.assertEqual(report["models"]["skippedInUse"], 2)
a.close()
b.close()
report = storage.prune_cache(self.cache, model_limit_bytes=60)
self.assertFalse(first.exists())
self.assertTrue(second.exists())
self.assertEqual(report["models"]["overLimitBytes"], 0)
def test_only_final_oversized_entry_is_retained(self):
old = self.entry(0, 200, 1)
new = self.entry(1, 200, 2)
report = storage.prune_cache(self.cache, model_limit_bytes=100)
self.assertFalse(old.exists())
self.assertTrue(new.exists())
self.assertEqual(report["models"]["oversizedEntries"], 1)
self.assertEqual(report["models"]["overLimitBytes"], 100)
def test_lock_shards_are_bounded_and_collisions_only_defer_eviction(self):
for number in range(512):
with storage.acquire_cache_lease(self.cache, "models", key(number)):
pass
self.assertEqual(len(list((self.cache / ".locks").iterdir())), storage.LOCK_SHARDS)
self.assertTrue(all(path.stat().st_size == 0 for path in (self.cache / ".locks").iterdir()))
colliding = self.entry(128, 60, 1)
other = self.entry(1, 60, 2)
self.lease(0)
report = storage.prune_cache(self.cache, model_limit_bytes=60)
self.assertTrue(colliding.exists())
self.assertFalse(other.exists())
self.assertEqual(report["models"]["skippedInUse"], 1)
def test_legacy_unknown_and_linked_entries_are_not_followed_or_removed(self):
old = self.entry(0, 60, 1)
self.entry(1, 60, 2)
legacy = self.cache / key(50)
legacy.mkdir()
(legacy / "model").write_text("old cache")
unknown = self.cache / "models" / "notes.txt"
unknown.write_text("user notes")
outside = self.folder / "outside"
outside.mkdir()
(outside / "important").write_text("keep")
direct = self.cache / "models" / key(3)
nested = self.cache / "models" / key(4)
nested.mkdir()
try:
direct.symlink_to(outside, target_is_directory=True)
(nested / "outside").symlink_to(outside, target_is_directory=True)
except OSError as exc:
self.skipTest(f"Symlinks unavailable: {exc}")
report = storage.prune_cache(self.cache, model_limit_bytes=60)
self.assertFalse(old.exists())
self.assertEqual((legacy / "model").read_text(), "old cache")
self.assertEqual(unknown.read_text(), "user notes")
self.assertEqual((outside / "important").read_text(), "keep")
self.assertTrue(direct.is_symlink() and (nested / "outside").is_symlink())
self.assertEqual(report["models"]["skippedUnmanaged"], 3)
def test_invalid_keys_and_symlink_lock_files_are_rejected(self):
for kind, digest in (("../models", key(0)), ("models", "../outside"),
("models", "A" * 64), ("models", key(0) + "\n")):
with self.subTest(kind=kind, key=digest):
with self.assertRaises(ValueError):
storage.acquire_cache_lease(self.cache, kind, digest)
lock_dir = self.cache / ".locks"
lock_dir.mkdir(parents=True)
target = self.folder / "outside-lock"
target.write_text("untouched")
try:
(lock_dir / "models-000.lock").symlink_to(target)
except OSError as exc:
self.skipTest(f"Symlinks unavailable: {exc}")
with self.assertRaises(RuntimeError):
self.lease(0)
self.assertEqual(target.read_text(), "untouched")
def test_prune_serializes_sweeps_without_blocking_readers(self):
old = self.entry(0, 200, 1)
new = self.entry(1, 200, 2)
import shutil
original = shutil.rmtree
concurrent = []
def removing(path, *args, **kwargs):
concurrent.append(storage.prune_cache(self.cache, model_limit_bytes=100))
# Reader leases do not contend with the global sweep lock.
with self.lease(2, blocking=False):
pass
return original(path, *args, **kwargs)
with patch.object(storage.shutil, "rmtree", side_effect=removing):
storage.prune_cache(self.cache, model_limit_bytes=100)
self.assertFalse(old.exists())
self.assertTrue(new.exists())
self.assertEqual(len(concurrent), 1)
self.assertTrue(concurrent[0]["models"]["skippedConcurrentSweep"])
def stage(self, number: int, size: int, namespace: str = "root") -> Path:
parent = self.cache if namespace == "root" else self.cache / namespace
parent.mkdir(parents=True, exist_ok=True)
stage = Path(tempfile.mkdtemp(prefix=f"building-{key(number)}-", dir=parent))
(stage / "unfinished.o").write_bytes(b"x" * size)
return stage
def test_orphan_stages_are_removed_in_all_three_managed_locations(self):
root = self.stage(10, 30)
model = self.stage(11, 40, "models")
obj = self.stage(12, 50, "objects")
legacy = self.cache / "building-old12345"
legacy.mkdir()
unknown = self.cache / "models" / f"building-{key(13)}-short"
unknown.mkdir()
# Similar names inside an unrelated entry are outside the sweep scope.
nested = self.cache / key(14) / f"building-{key(15)}-abcdefgh"
nested.mkdir(parents=True)
report = storage.prune_cache(self.cache)
self.assertFalse(root.exists() or model.exists() or obj.exists())
self.assertTrue(legacy.exists() and unknown.exists() and nested.exists())
self.assertEqual(report["orphanStages"]["orphanStagesRemoved"], 3)
self.assertEqual(report["orphanStages"]["orphanStagesBytes"], 120)
self.assertEqual(report["orphanStages"]["skippedUnmanaged"], 2)
self.assertEqual(report["orphanStages"]["errors"], [])
def test_active_stages_use_their_correct_namespace_and_survive(self):
model_lease = self.lease(10)
object_lease = storage.acquire_cache_lease(self.cache, "objects", key(11))
self.addCleanup(object_lease.close)
root_active = self.stage(10, 30)
model_active = self.stage(10, 40, "models")
object_active = self.stage(11, 50, "objects")
collision = self.stage(10 + storage.LOCK_SHARDS, 20)
# Holding models/key(10) does not protect objects/key(10), or vice versa.
object_orphan = self.stage(10, 60, "objects")
model_orphan = self.stage(11, 70)
report = storage.prune_cache(self.cache)["orphanStages"]
self.assertTrue(all(path.exists() for path in (
root_active, model_active, object_active, collision,
)))
self.assertFalse(object_orphan.exists() or model_orphan.exists())
self.assertEqual(report["skippedInUse"], 4)
self.assertEqual(report["orphanStagesRemoved"], 2)
self.assertEqual(report["orphanStagesBytes"], 130)
model_lease.close()
object_lease.close()
report = storage.prune_cache(self.cache)["orphanStages"]
self.assertEqual(report["orphanStagesRemoved"], 4)
self.assertEqual(report["orphanStagesBytes"], 140)
def test_killed_builder_releases_stage_protection_for_next_sweep(self):
script = r"""
import sys, tempfile, time
from pathlib import Path
from app.simulation.native_codegen.cache_storage import acquire_cache_lease
cache, key = Path(sys.argv[1]), sys.argv[2]
lease = acquire_cache_lease(cache, 'models', key)
stage = Path(tempfile.mkdtemp(prefix='building-' + key + '-', dir=cache))
(stage / 'unfinished.o').write_bytes(b'x' * 37)
print(stage.name, flush=True)
time.sleep(60)
"""
process = subprocess.Popen(
[sys.executable, "-c", script, str(self.cache), key(16)], cwd=ROOT,
stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True,
)
self.addCleanup(lambda: process.kill() if process.poll() is None else None)
stage = self.cache / process.stdout.readline().strip()
self.assertTrue(stage.is_dir())
report = storage.prune_cache(self.cache)["orphanStages"]
self.assertTrue(stage.exists())
self.assertEqual(report["skippedInUse"], 1)
process.kill()
process.communicate(timeout=15)
report = storage.prune_cache(self.cache)["orphanStages"]
self.assertFalse(stage.exists())
self.assertEqual(report["orphanStagesRemoved"], 1)
self.assertEqual(report["orphanStagesBytes"], 37)
def test_orphan_stage_symlinks_and_symlink_namespaces_are_not_followed(self):
self.cache.mkdir()
outside = self.folder / "outside-stage"
outside.mkdir()
artifact = outside / "important"
artifact.write_text("keep")
direct = self.cache / f"building-{key(20)}-abcdefgh"
nested = self.stage(21, 10)
object_namespace = self.cache / "objects"
try:
direct.symlink_to(outside, target_is_directory=True)
(nested / "link").symlink_to(artifact)
object_namespace.symlink_to(outside, target_is_directory=True)
except OSError as exc:
self.skipTest(f"Symlinks unavailable: {exc}")
# A matching name behind a namespace link is never inspected or removed.
outside_stage = outside / f"building-{key(22)}-abcdefgh"
outside_stage.mkdir()
report = storage.prune_cache(self.cache)["orphanStages"]
self.assertEqual(report["orphanStagesRemoved"], 0)
self.assertEqual(report["skippedUnmanaged"], 2)
self.assertTrue(direct.is_symlink() and nested.exists() and outside_stage.exists())
self.assertEqual(artifact.read_text(), "keep")
def test_orphan_cleanup_failure_is_reported_without_removing_other_data(self):
stage = self.stage(24, 80)
with patch.object(storage.shutil, "rmtree", side_effect=PermissionError("busy stage")):
report = storage.prune_cache(self.cache)["orphanStages"]
self.assertTrue(stage.exists())
self.assertEqual(report["orphanStagesRemoved"], 0)
self.assertEqual(report["orphanStagesBytes"], 0)
self.assertEqual(len(report["errors"]), 1)
self.assertIn("busy stage", report["errors"][0])
def test_filesystem_cleanup_errors_do_not_fail_simulation(self):
self.entry(0, 60, 1)
self.entry(1, 60, 2)
with patch.object(storage.shutil, "rmtree", side_effect=PermissionError("busy")):
report = storage.prune_cache(self.cache, model_limit_bytes=60)
self.assertEqual(report["models"]["overLimitBytes"], 60)
self.assertTrue(report["models"]["errors"])
if __name__ == "__main__":
unittest.main()
+2 -1
View File
@@ -5,6 +5,7 @@ import tempfile
import unittest import unittest
from app.simulation.native_codegen.build import toolchain from app.simulation.native_codegen.build import toolchain
from tests.native_kernel_source import native_kernel_source
ROOT = Path(__file__).resolve().parents[1] ROOT = Path(__file__).resolve().parents[1]
@@ -16,7 +17,7 @@ class NativePipeCacheTests(unittest.TestCase):
compiler = toolchain()[0] compiler = toolchain()[0]
except (OSError, RuntimeError, subprocess.SubprocessError) as exc: except (OSError, RuntimeError, subprocess.SubprocessError) as exc:
self.skipTest(f"Native toolchain unavailable: {exc}") self.skipTest(f"Native toolchain unavailable: {exc}")
source = (ROOT / "native/components/kernels.c").read_text() source = native_kernel_source()
signature = "double d, double length, double rr, int kind) {" signature = "double d, double length, double rr, int kind) {"
self.assertEqual(source.count(signature), 1) self.assertEqual(source.count(signature), 1)
source = "static int pipe_calls;\n" + source.replace( source = "static int pipe_calls;\n" + source.replace(
+2 -1
View File
@@ -12,6 +12,7 @@ from app.simulation.core.medium import IdealGasMedium
from app.simulation.native_codegen.build import build_native, toolchain from app.simulation.native_codegen.build import build_native, toolchain
from app.simulation.native_codegen.extended import compile_extended_program from app.simulation.native_codegen.extended import compile_extended_program
from tests.test_native_catalog import Circuit from tests.test_native_catalog import Circuit
from tests.native_kernel_source import native_kernel_source
ROOT = Path(__file__).resolve().parents[1] ROOT = Path(__file__).resolve().parents[1]
@@ -109,7 +110,7 @@ int main(void) {
''' '''
with tempfile.TemporaryDirectory(prefix='native-pipe-root-') as tmp: with tempfile.TemporaryDirectory(prefix='native-pipe-root-') as tmp:
directory=Path(tmp);source=directory/'check.c';exe=directory/'check.exe' directory=Path(tmp);source=directory/'check.c';exe=directory/'check.exe'
source.write_text((ROOT/'native/components/kernels.c').read_text()+harness) source.write_text(native_kernel_source()+harness)
build=subprocess.run([self.compiler,'-std=c11','-O3','-Wall','-Wextra','-Werror', build=subprocess.run([self.compiler,'-std=c11','-O3','-Wall','-Wextra','-Werror',
'-ffp-contract=off','-fno-fast-math','-static-libgcc','-I',str(ROOT/'native/include'), '-ffp-contract=off','-fno-fast-math','-static-libgcc','-I',str(ROOT/'native/include'),
str(source),'-lm','-o',str(exe)],capture_output=True,text=True,timeout=60) str(source),'-lm','-o',str(exe)],capture_output=True,text=True,timeout=60)
+2 -1
View File
@@ -8,6 +8,7 @@ import shutil
import subprocess import subprocess
import tempfile import tempfile
import unittest import unittest
from tests.native_kernel_source import native_kernel_source
ROOT = Path(__file__).resolve().parents[1] ROOT = Path(__file__).resolve().parents[1]
@@ -67,7 +68,7 @@ class NativePipeSolverTests(unittest.TestCase):
cls.directory = tempfile.TemporaryDirectory(prefix='native-pipe-solver-') cls.directory = tempfile.TemporaryDirectory(prefix='native-pipe-solver-')
cls.addClassCleanup(cls.directory.cleanup) cls.addClassCleanup(cls.directory.cleanup)
cls.compiler = command cls.compiler = command
source = (ROOT / 'native/components/kernels.c').read_text() source = native_kernel_source()
cls.library = cls.build_library(source, 'ordinary') cls.library = cls.build_library(source, 'ordinary')
# Fault injection only in the temporary test translation unit. The # Fault injection only in the temporary test translation unit. The
+2 -1
View File
@@ -5,6 +5,7 @@ import tempfile
import unittest import unittest
from app.simulation.native_codegen.build import toolchain from app.simulation.native_codegen.build import toolchain
from tests.native_kernel_source import native_kernel_source
ROOT = Path(__file__).resolve().parents[1] ROOT = Path(__file__).resolve().parents[1]
@@ -15,7 +16,7 @@ class NativePropertyTests(unittest.TestCase):
compiler = toolchain()[0] compiler = toolchain()[0]
except (OSError, RuntimeError, subprocess.SubprocessError) as exc: except (OSError, RuntimeError, subprocess.SubprocessError) as exc:
self.skipTest(f"Native toolchain unavailable: {exc}") self.skipTest(f"Native toolchain unavailable: {exc}")
source = (ROOT / 'native/components/kernels.c').read_text() source = native_kernel_source()
for signature, counter in ( for signature, counter in (
('static double temperature_ph(double p,double h) {', 'ph_calls'), ('static double temperature_ph(double p,double h) {', 'ph_calls'),
('static double z_factor(double p,double T) {', 'z_calls'), ('static double z_factor(double p,double T) {', 'z_calls'),