C内核按库功能拆解，编译结果缓存区构建，编译过程与已有缓存结果对照功能实现
This commit is contained in:
lujingze committed
1 parent
aa4951b14e
commit
151e6e4b97
30 files changed
+4856
-604
No files matched your search
+7
-2
@@ -69,6 +69,11 @@ SimulationProgressEmitter = Callable[
|
||||
]
|
||||
SIMULATION_PHASE_MESSAGES = {
|
||||
"initializing": "正在初始化状态与方程",
|
||||
"native-generation": "正在生成原生模型代码",
|
||||
"native-cache-check": "正在检查已编译模型缓存",
|
||||
"native-cache-hit": "已复用模型缓存，正在启动求解器",
|
||||
"native-compilation": "正在编译模型与缺失的原生模块",
|
||||
"native-linking": "正在链接模型可执行文件",
|
||||
"integrating": "正在进行时间积分与压力流量求解",
|
||||
"postprocessing": "正在整理采样结果",
|
||||
"cancelled": "正在整理已终止仿真的部分结果",
|
||||
@@ -746,9 +751,9 @@ def _run_system_xml_simulation_profiled(
|
||||
emit(0, "validation", "正在校验 System XML")
|
||||
report = validate_system_xml_document(xml_bytes)
|
||||
document = _validated_xml_document_or_422(report)
|
||||
emit(0, "compilation", "正在编译组件与连接关系")
|
||||
emit(0, "compilation", "正在解析组件与连接关系")
|
||||
network = _compile_xml_document_or_422(document)
|
||||
emit(0, "initialization", "模型编译完成，正在准备求解器")
|
||||
emit(0, "initialization", "连接关系已解析，正在准备原生模型")
|
||||
|
||||
def report_system_progress(progress: float, phase: str) -> None:
|
||||
bounded_progress = min(1.0, max(0.0, progress))
|
||||
|
||||
@@ -1,23 +1,31 @@
|
||||
"""Reproducible native builds and a checked, model-specific executable cache."""
|
||||
"""On-demand native modules, checked object reuse and bounded executable caches."""
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from dataclasses import dataclass, field
|
||||
from hashlib import sha256
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
from typing import Any, Callable
|
||||
|
||||
from .compiler import NativeProgram
|
||||
from .modules import component_modules
|
||||
from .cache_storage import acquire_cache_lease, touch_cache_entry, prune_cache
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[3]
|
||||
NATIVE = ROOT / "native"
|
||||
CACHE = ROOT / "app/data/native-builds"
|
||||
LIBRARIES = ("cvode", "core", "nvecserial", "sunmatrixdense", "sunlinsoldense")
|
||||
COMPILER_FLAGS = ("-std=c11", "-O3", "-Wall", "-Wextra", "-Werror", "-ffp-contract=off", "-fno-fast-math")
|
||||
RUNTIME_SOURCES = ("runtime/main.c", "runtime/common.c", "runtime/rk45.c",
|
||||
"runtime/cvode_solver.c", "runtime/json_numbers.c", "encoding/ryu/d2s.c")
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
@@ -26,12 +34,115 @@ class NativeBuild:
|
||||
manifest: dict
|
||||
cache_hit: bool
|
||||
seconds: float
|
||||
details: dict = field(default_factory=dict)
|
||||
_lease: Any = field(default=None, repr=False, compare=False)
|
||||
_cache: Path | None = field(default=None, repr=False, compare=False)
|
||||
|
||||
def close(self) -> None:
|
||||
"""Release this caller's executable pin after its last use."""
|
||||
if self._lease is not None:
|
||||
self._lease.close()
|
||||
if self._cache is not None:
|
||||
_prune(self._cache)
|
||||
|
||||
|
||||
def _hash(path: Path) -> str:
|
||||
return sha256(path.read_bytes()).hexdigest()
|
||||
|
||||
|
||||
def _key(value: object) -> str:
|
||||
return sha256(json.dumps(value, sort_keys=True, separators=(",", ":")).encode()).hexdigest()
|
||||
|
||||
|
||||
def _runtime_sources(program: NativeProgram) -> list[Path]:
|
||||
return [NATIVE / name for name in RUNTIME_SOURCES] + [
|
||||
NATIVE / "components/modules" / f"{name}.c"
|
||||
for name in component_modules(program.source + "\n" + program.header)
|
||||
]
|
||||
|
||||
|
||||
def _budget(name: str, default: int) -> int:
|
||||
value = os.environ.get(name, str(default))
|
||||
if not re.fullmatch(r"[0-9]+", value):
|
||||
raise ValueError(f"{name} must be a nonnegative integer in MiB.")
|
||||
return int(value) * 1024**2
|
||||
|
||||
|
||||
def _prune(cache: Path) -> dict:
|
||||
return prune_cache(cache,
|
||||
model_limit_bytes=_budget("SIMULATION_NATIVE_MODEL_CACHE_MB", 256),
|
||||
object_limit_bytes=_budget("SIMULATION_NATIVE_OBJECT_CACHE_MB", 128))
|
||||
|
||||
|
||||
def _checked_manifest(target: Path, key_name: str, key: str, version: int, required: set[str]) -> dict | None:
|
||||
if not target.exists() and not target.is_symlink():
|
||||
return None
|
||||
message = f"Native cache integrity check failed: {target}"
|
||||
if target.is_symlink() or not target.is_dir():
|
||||
raise RuntimeError(message)
|
||||
try:
|
||||
path = target / "manifest.json"
|
||||
if path.is_symlink():
|
||||
raise ValueError("linked manifest")
|
||||
manifest = json.loads(path.read_text(encoding="utf-8"))
|
||||
if not isinstance(manifest, dict) or manifest.get(key_name) != key or manifest.get("cacheVersion") != version:
|
||||
raise ValueError("identity mismatch")
|
||||
artifacts = manifest.get("artifacts")
|
||||
if not isinstance(artifacts, dict) or set(artifacts) != required:
|
||||
raise ValueError("incomplete artifacts")
|
||||
if key_name == "objectKey":
|
||||
recipe = {name: manifest[name] for name in ("cacheVersion", "sourceName", "preprocessedSha256", "compiler")}
|
||||
if _key(recipe) != key:
|
||||
raise ValueError("object identity mismatch")
|
||||
else:
|
||||
recipe = manifest.get("buildIdentity")
|
||||
if not isinstance(recipe, dict) or _key(recipe) != key or manifest.get("objectKeys") != recipe["objectKeys"]:
|
||||
raise ValueError("model identity mismatch")
|
||||
if artifacts["model.c"] != recipe["sourceSha256"] or artifacts["model.h"] != recipe["headerSha256"]:
|
||||
raise ValueError("model source identity mismatch")
|
||||
if _key({name: manifest[name] for name in recipe["contractKeys"]}) != recipe["contractSha256"]:
|
||||
raise ValueError("model contract mismatch")
|
||||
if artifacts["THIRD_PARTY_NOTICES.txt"] != recipe["notice"]:
|
||||
raise ValueError("license artifact mismatch")
|
||||
for name in required:
|
||||
if name.lower().endswith(".dll") and artifacts[name] != recipe["dependencies"][name]:
|
||||
raise ValueError("runtime dependency artifact mismatch")
|
||||
for name, digest in artifacts.items():
|
||||
if not re.fullmatch(r"[A-Za-z0-9_.-]+", name) or name in (".", ".."):
|
||||
raise ValueError("invalid artifact name")
|
||||
path = target / name
|
||||
if path.is_symlink() or not path.is_file() or not isinstance(digest, str) or _hash(path) != digest:
|
||||
raise ValueError("artifact mismatch")
|
||||
return manifest
|
||||
except (OSError, ValueError, TypeError, KeyError) as exc:
|
||||
raise RuntimeError(message) from exc
|
||||
|
||||
|
||||
def _publish(stage: Path, target: Path, key_name: str, key: str, version: int, required: set[str]) -> dict:
|
||||
try:
|
||||
stage.rename(target)
|
||||
except OSError:
|
||||
# Another reader/build may publish the same content while we compile.
|
||||
# Its artifacts need not be byte-identical (e.g. PE timestamps), but
|
||||
# must satisfy the same complete identity and its own recorded hashes.
|
||||
winner = _checked_manifest(target, key_name, key, version, required)
|
||||
if winner is None:
|
||||
raise
|
||||
shutil.rmtree(stage)
|
||||
return winner
|
||||
manifest = _checked_manifest(target, key_name, key, version, required)
|
||||
assert manifest is not None
|
||||
return manifest
|
||||
|
||||
|
||||
def _command(command: list[str], *, log: list[str], timeout: float = 120) -> bytes:
|
||||
result = subprocess.run(command, capture_output=True, timeout=timeout)
|
||||
log.append(" ".join(command) + "\n" + result.stderr.decode("utf-8", errors="replace"))
|
||||
if result.returncode:
|
||||
raise RuntimeError("Native compilation failed: " + result.stderr.decode("utf-8", errors="replace")[-3000:])
|
||||
return result.stdout
|
||||
|
||||
|
||||
def toolchain() -> tuple[str, Path, str]:
|
||||
if os.name != "nt" and not sys.platform.startswith("linux"):
|
||||
raise RuntimeError("Native builds support Windows and Linux; this platform is not supported.")
|
||||
@@ -49,90 +160,190 @@ def toolchain() -> tuple[str, Path, str]:
|
||||
return compiler, base, version
|
||||
|
||||
|
||||
def build_native(program: NativeProgram, *, cache_dir: Path | None = None) -> NativeBuild:
|
||||
start = time.perf_counter()
|
||||
def build_native(program: NativeProgram, *, cache_dir: Path | None = None,
|
||||
progress_callback: Callable[[str], None] | None = None) -> NativeBuild:
|
||||
started = time.perf_counter()
|
||||
if progress_callback:
|
||||
progress_callback("native-cache-check")
|
||||
compiler, sundials, compiler_version = toolchain()
|
||||
runtime = sorted(NATIVE.rglob("*.c")) + sorted((NATIVE / "include").rglob("*.h"))
|
||||
flags = ["-std=c11", "-O3", "-Wall", "-Wextra", "-Werror", "-ffp-contract=off", "-fno-fast-math"]
|
||||
flags = list(COMPILER_FLAGS)
|
||||
executable_name = "model.exe" if os.name == "nt" else "model"
|
||||
if os.name == "nt":
|
||||
flags += ["-D__USE_MINGW_ANSI_STDIO=1", "-static-libgcc"]
|
||||
libraries = [sundials / "lib" / f"sundials_{name}.lib" for name in LIBRARIES]
|
||||
dlls = [sundials / "bin" / f"sundials_{name}.dll" for name in LIBRARIES]
|
||||
vc_runtime = sundials / "bin/vcruntime140.dll"
|
||||
if vc_runtime.is_file():
|
||||
dlls.append(vc_runtime)
|
||||
elif sys.platform.startswith("linux"):
|
||||
flags += ["-D_POSIX_C_SOURCE=200809L"]
|
||||
library_dirs = [sundials / "lib", sundials / "lib64", sundials / "lib/x86_64-linux-gnu"]
|
||||
library_dir = next((path for path in library_dirs if all(
|
||||
(path / f"libsundials_{name}.a").is_file() for name in LIBRARIES)), None)
|
||||
if library_dir is None:
|
||||
raise RuntimeError("SUNDIALS static libraries not found; build with BUILD_STATIC_LIBS=ON and set SUNDIALS_ROOT.")
|
||||
libraries = [library_dir / f"libsundials_{name}.a" for name in LIBRARIES]
|
||||
dlls = []
|
||||
if (sundials / "bin/vcruntime140.dll").is_file():
|
||||
dlls.append(sundials / "bin/vcruntime140.dll")
|
||||
else:
|
||||
raise RuntimeError("Native builds support Windows and Linux; this platform is not supported.")
|
||||
link_libraries = list(map(str, libraries))
|
||||
if sys.platform.startswith("linux"):
|
||||
link_libraries = ["-Wl,--start-group", *link_libraries, "-Wl,--end-group"]
|
||||
sources = {str(p.relative_to(ROOT)): _hash(p) for p in runtime}
|
||||
sources["native/THIRD_PARTY_NOTICES.txt"] = _hash(NATIVE / "THIRD_PARTY_NOTICES.txt")
|
||||
dependencies = {str(p.name): _hash(p) for p in libraries + dlls}
|
||||
# Header hashes include precision/index ABI settings as well as library APIs.
|
||||
for directory in ("sundials", "cvode", "nvector", "sunmatrix", "sunlinsol"):
|
||||
for path in sorted((sundials / "include" / directory).glob("*.h")):
|
||||
dependencies[f"{directory}/{path.name}"] = _hash(path)
|
||||
identity = dict(source=program.source, header=program.header, contract=program.manifest(), sources=sources,
|
||||
dependencies=dependencies, compiler=compiler_version, flags=flags,
|
||||
platform=sys.platform, abi=1)
|
||||
signature = sha256(json.dumps(identity, sort_keys=True).encode()).hexdigest()
|
||||
flags += ["-D_POSIX_C_SOURCE=200809L"]
|
||||
candidates = [sundials / "lib", sundials / "lib64", sundials / "lib/x86_64-linux-gnu"]
|
||||
directory = next((p for p in candidates if all((p / f"libsundials_{name}.a").is_file() for name in LIBRARIES)), None)
|
||||
if directory is None:
|
||||
raise RuntimeError("SUNDIALS static libraries not found; set SUNDIALS_ROOT.")
|
||||
libraries = [directory / f"libsundials_{name}.a" for name in LIBRARIES]
|
||||
dlls = []
|
||||
cache = (cache_dir or CACHE).resolve()
|
||||
target = cache / signature
|
||||
manifest_path = target / "manifest.json"
|
||||
if manifest_path.is_file():
|
||||
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
|
||||
if all((target / name).is_file() and _hash(target / name) == digest
|
||||
for name, digest in manifest["artifacts"].items()):
|
||||
return NativeBuild(target / executable_name, manifest, True, time.perf_counter()-start)
|
||||
raise RuntimeError(f"Native cache integrity check failed: {target}")
|
||||
cache.mkdir(parents=True, exist_ok=True)
|
||||
stage = Path(tempfile.mkdtemp(prefix="building-", dir=cache))
|
||||
(stage / "model.c").write_text(program.source, encoding="utf-8")
|
||||
(stage / "model.h").write_text(program.header, encoding="utf-8")
|
||||
command = [compiler, *flags, "-I", str(stage), "-I", str(NATIVE / "include"),
|
||||
"-I", str(sundials / "include"), str(stage / "model.c"),
|
||||
*[str(p) for p in runtime if p.suffix == ".c"],
|
||||
*link_libraries, "-lm", "-o", str(stage / executable_name)]
|
||||
result = subprocess.run(command, capture_output=True, text=True, timeout=120)
|
||||
(stage / "build.log").write_text(result.stdout + result.stderr, encoding="utf-8")
|
||||
if result.returncode:
|
||||
raise RuntimeError(f"Native compilation failed; see {stage / 'build.log'}: {result.stderr[-3000:]}")
|
||||
for library in dlls:
|
||||
shutil.copy2(library, stage / library.name)
|
||||
shutil.copy2(NATIVE / "THIRD_PARTY_NOTICES.txt", stage / "THIRD_PARTY_NOTICES.txt")
|
||||
manifest = {
|
||||
**program.manifest(), "buildKey": signature, "compiler": compiler_version,
|
||||
"compilerFlags": flags, "sourceHashes": sources, "dependencyHashes": dependencies,
|
||||
"artifacts": {p.name: _hash(p) for p in stage.iterdir() if p.name != "build.log"},
|
||||
}
|
||||
(stage / "manifest.json").write_text(json.dumps(manifest, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
for kind in ("models", "objects"):
|
||||
path = cache / kind
|
||||
if path.is_symlink():
|
||||
raise RuntimeError(f"Native cache directory must not be a symbolic link: {path}")
|
||||
path.mkdir(exist_ok=True)
|
||||
_budget("SIMULATION_NATIVE_MODEL_CACHE_MB", 256)
|
||||
_budget("SIMULATION_NATIVE_OBJECT_CACHE_MB", 128)
|
||||
work_key = sha256(os.urandom(32)).hexdigest()
|
||||
work_lease = acquire_cache_lease(cache, "models", work_key)
|
||||
try:
|
||||
stage.rename(target)
|
||||
except OSError:
|
||||
# A concurrent compiler may have published the identical cache first.
|
||||
if not manifest_path.is_file():
|
||||
raise
|
||||
published = json.loads(manifest_path.read_text(encoding="utf-8"))
|
||||
# PE linker timestamps can differ between concurrent equivalent builds.
|
||||
# Validate the winning build against its own hashes and our identity.
|
||||
if published.get("buildKey") != signature or not all(
|
||||
(target / name).is_file() and _hash(target / name) == digest
|
||||
for name, digest in published["artifacts"].items()
|
||||
):
|
||||
raise RuntimeError("Concurrent native build did not produce the expected artifacts.")
|
||||
manifest = published
|
||||
if stage.resolve().parent != cache:
|
||||
raise RuntimeError("Unexpected native build staging directory.")
|
||||
shutil.rmtree(stage)
|
||||
return NativeBuild(target / executable_name, manifest, False, time.perf_counter()-start)
|
||||
stage = Path(tempfile.mkdtemp(prefix=f"building-{work_key}-", dir=cache))
|
||||
except BaseException:
|
||||
work_lease.close()
|
||||
raise
|
||||
logs: list[str] = []
|
||||
model_lease = None
|
||||
try:
|
||||
(stage / "model.c").write_text(program.source, encoding="utf-8", newline="\n")
|
||||
(stage / "model.h").write_text(program.header, encoding="utf-8", newline="\n")
|
||||
compiler_path = Path(shutil.which(compiler) or compiler).resolve()
|
||||
compiler_identity = {"version": compiler_version, "binarySha256": _hash(compiler_path),
|
||||
"target": _command([compiler, "-dumpmachine"], log=logs).decode().strip(),
|
||||
"flags": flags, "platform": sys.platform}
|
||||
runtime = _runtime_sources(program)
|
||||
sources = [stage / "model.c", *runtime]
|
||||
native_headers = {p: _hash(p) for p in (NATIVE / "include").rglob("*.h")}
|
||||
dependency_headers = {p: _hash(p) for directory in ("sundials", "cvode", "nvector", "sunmatrix", "sunlinsol")
|
||||
for p in (sundials / "include" / directory).glob("*.h")}
|
||||
preprocessing_start = time.perf_counter()
|
||||
# Preprocessed bytes include the actual transitive headers and all
|
||||
# effective macros/ABI switches. Compile precisely these bytes, so a
|
||||
# header edit cannot race the cache identity and the compiler input.
|
||||
def preprocess(source: Path) -> dict:
|
||||
local_log: list[str] = []
|
||||
source_hash = _hash(source)
|
||||
data = _command([compiler, *flags, "-E", "-P", f"-fmacro-prefix-map={stage}=/generated",
|
||||
"-I", str(stage), "-I", str(NATIVE / "include"), "-I", str(sundials / "include"),
|
||||
str(source)], log=local_log)
|
||||
name = "model.c" if source == stage / "model.c" else "native/" + source.relative_to(NATIVE).as_posix()
|
||||
if _hash(source) != source_hash:
|
||||
raise RuntimeError(f"Native source changed during preprocessing: {source}")
|
||||
digest = sha256(data).hexdigest()
|
||||
identity = {"cacheVersion": 1, "sourceName": name, "preprocessedSha256": digest, "compiler": compiler_identity}
|
||||
return {"sourceName": name, "sourceSha256": source_hash, "data": data,
|
||||
"objectKey": _key(identity), "identity": identity, "log": local_log}
|
||||
with ThreadPoolExecutor(max_workers=min(4, len(sources))) as pool:
|
||||
units = list(pool.map(preprocess, sources))
|
||||
for unit in units:
|
||||
logs.extend(unit.pop("log"))
|
||||
preprocessing_seconds = time.perf_counter() - preprocessing_start
|
||||
if any(_hash(path) != digest for path, digest in {**native_headers, **dependency_headers}.items()):
|
||||
raise RuntimeError("Native headers changed during preprocessing; retry with stable sources.")
|
||||
dependencies = {p.name: _hash(p) for p in libraries + dlls}
|
||||
notice = (NATIVE / "THIRD_PARTY_NOTICES.txt").read_bytes()
|
||||
identity = {"cacheVersion": 2, "sourceSha256": sha256(program.source.encode()).hexdigest(),
|
||||
"headerSha256": sha256(program.header.encode()).hexdigest(),
|
||||
"contractSha256": _key(program.manifest()), "contractKeys": sorted(program.manifest()),
|
||||
"objectKeys": [unit["objectKey"] for unit in units],
|
||||
"compiler": compiler_identity, "dependencies": dependencies, "notice": sha256(notice).hexdigest()}
|
||||
signature = _key(identity)
|
||||
target = cache / "models" / signature
|
||||
required = {"model.c", "model.h", executable_name, "THIRD_PARTY_NOTICES.txt", *(p.name for p in dlls)}
|
||||
model_lease = acquire_cache_lease(cache, "models", signature)
|
||||
manifest = _checked_manifest(target, "buildKey", signature, 2, required)
|
||||
details = {"selectedModules": list(component_modules(program.source + "\n" + program.header)),
|
||||
"unitCount": len(units), "objectCacheHits": 0, "objectCompilations": 0,
|
||||
"preprocessSeconds": preprocessing_seconds, "compileSeconds": 0.0,
|
||||
"compileWallSeconds": 0.0, "linkSeconds": 0.0, "modelCacheHit": manifest is not None}
|
||||
if manifest is not None:
|
||||
if progress_callback:
|
||||
progress_callback("native-cache-hit")
|
||||
touch_cache_entry(cache, "models", signature)
|
||||
details["pruning"] = _prune(cache)
|
||||
result = NativeBuild(target / executable_name, manifest, True, time.perf_counter() - started,
|
||||
details, model_lease, cache)
|
||||
model_lease = None
|
||||
return result
|
||||
if progress_callback:
|
||||
progress_callback("native-compilation")
|
||||
compile_start = time.perf_counter()
|
||||
def compile_unit(index_unit: tuple[int, dict]) -> dict:
|
||||
index, unit = index_unit
|
||||
key = unit["objectKey"]
|
||||
destination = cache / "objects" / key
|
||||
with acquire_cache_lease(cache, "objects", key):
|
||||
object_manifest = _checked_manifest(destination, "objectKey", key, 1, {"unit.o"})
|
||||
hit = object_manifest is not None
|
||||
elapsed = 0.0
|
||||
local_log: list[str] = []
|
||||
if not hit:
|
||||
temporary = Path(tempfile.mkdtemp(prefix=f"building-{key}-", dir=cache / "objects"))
|
||||
try:
|
||||
(temporary / "unit.i").write_bytes(unit["data"])
|
||||
before = time.perf_counter()
|
||||
_command([compiler, *flags, "-x", "cpp-output", "-c", str(temporary / "unit.i"),
|
||||
"-o", str(temporary / "unit.o")], log=local_log)
|
||||
elapsed = time.perf_counter() - before
|
||||
(temporary / "unit.i").unlink()
|
||||
object_manifest = {**unit["identity"], "objectKey": key, "artifacts": {"unit.o": _hash(temporary / "unit.o")}}
|
||||
(temporary / "manifest.json").write_text(json.dumps(object_manifest, indent=2) + "\n")
|
||||
_publish(temporary, destination, "objectKey", key, 1, {"unit.o"})
|
||||
finally:
|
||||
if temporary.exists():
|
||||
shutil.rmtree(temporary)
|
||||
# Linking uses private copies, so another model's LRU pass can
|
||||
# safely reclaim the shared object after this lease is released.
|
||||
local_object = stage / f"object-{index}.o"
|
||||
shutil.copyfile(destination / "unit.o", local_object)
|
||||
touch_cache_entry(cache, "objects", key)
|
||||
return {"path": local_object, "hit": hit, "seconds": elapsed, "log": local_log}
|
||||
with ThreadPoolExecutor(max_workers=min(4, len(units))) as pool:
|
||||
objects = list(pool.map(compile_unit, enumerate(units)))
|
||||
details["compileWallSeconds"] = time.perf_counter() - compile_start
|
||||
details["compileSeconds"] = sum(item["seconds"] for item in objects)
|
||||
details["objectCacheHits"] = sum(item["hit"] for item in objects)
|
||||
details["objectCompilations"] = len(objects) - details["objectCacheHits"]
|
||||
for item in objects:
|
||||
logs.extend(item["log"])
|
||||
link_libraries = list(map(str, libraries))
|
||||
if sys.platform.startswith("linux"):
|
||||
link_libraries = ["-Wl,--start-group", *link_libraries, "-Wl,--end-group"]
|
||||
if progress_callback:
|
||||
progress_callback("native-linking")
|
||||
before_link = time.perf_counter()
|
||||
_command([compiler, *flags, *[str(item["path"]) for item in objects], *link_libraries,
|
||||
"-lm", "-o", str(stage / executable_name)], log=logs)
|
||||
details["linkSeconds"] = time.perf_counter() - before_link
|
||||
if {p.name: _hash(p) for p in libraries + dlls} != dependencies:
|
||||
raise RuntimeError("Native dependencies changed during linking; retry with a stable toolchain.")
|
||||
for item in objects:
|
||||
item["path"].unlink()
|
||||
for library in dlls:
|
||||
shutil.copyfile(library, stage / library.name)
|
||||
if _hash(stage / library.name) != dependencies[library.name]:
|
||||
raise RuntimeError("Native runtime dependency changed during copying; retry with a stable toolchain.")
|
||||
(stage / "THIRD_PARTY_NOTICES.txt").write_bytes(notice)
|
||||
(stage / "build.log").write_text("\n".join(logs), encoding="utf-8")
|
||||
manifest = {**program.manifest(), "cacheVersion": 2, "buildKey": signature, "buildIdentity": identity,
|
||||
"compiler": compiler_version, "compilerFlags": flags,
|
||||
"sourceHashes": {unit["sourceName"]: unit["sourceSha256"] for unit in units},
|
||||
"nativeHeaderHashes": {"native/" + p.relative_to(NATIVE).as_posix(): digest
|
||||
for p, digest in native_headers.items()},
|
||||
"dependencyHashes": dependencies, "selectedModules": details["selectedModules"],
|
||||
"objectKeys": [unit["objectKey"] for unit in units],
|
||||
"artifacts": {name: _hash(stage / name) for name in sorted(required)}}
|
||||
(stage / "manifest.json").write_text(json.dumps(manifest, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
manifest = _publish(stage, target, "buildKey", signature, 2, required)
|
||||
touch_cache_entry(cache, "models", signature)
|
||||
details["pruning"] = _prune(cache)
|
||||
result = NativeBuild(target / executable_name, manifest, False, time.perf_counter() - started,
|
||||
details, model_lease, cache)
|
||||
model_lease = None
|
||||
return result
|
||||
finally:
|
||||
if model_lease is not None:
|
||||
model_lease.close()
|
||||
# Failed compilations never publish partial entries or retain unlimited
|
||||
# model sources/preprocessed files. Error stderr is included in the exception.
|
||||
try:
|
||||
if stage.exists():
|
||||
shutil.rmtree(stage)
|
||||
finally:
|
||||
work_lease.close()
|
||||
@@ -0,0 +1,453 @@
|
||||
"""Leases and bounded LRU eviction for immutable native cache entries.
|
||||
|
||||
Builds publish and validate their own artifacts. Readers hold a shared lease
|
||||
before checking an entry and until they finish using it. Eviction only removes
|
||||
an entry while holding the corresponding nonblocking exclusive lease.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from contextlib import suppress
|
||||
from dataclasses import dataclass
|
||||
import errno
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import shutil
|
||||
import stat
|
||||
import weakref
|
||||
|
||||
|
||||
MODEL_LIMIT_BYTES = 256 * 1024**2
|
||||
OBJECT_LIMIT_BYTES = 128 * 1024**2
|
||||
LOCK_SHARDS = 128
|
||||
_KINDS = frozenset(("models", "objects"))
|
||||
_KEY = re.compile(r"[0-9a-f]{64}\Z")
|
||||
# tempfile.mkdtemp uses eight lowercase letters, digits or underscores. Older
|
||||
# stages lacking a lease key are deliberately outside automatic cleanup.
|
||||
_STAGE = re.compile(r"building-([0-9a-f]{64})-[a-z0-9_]{8}\Z")
|
||||
|
||||
if os.name == "nt":
|
||||
import ctypes
|
||||
from ctypes import wintypes
|
||||
import msvcrt
|
||||
|
||||
class _Offset(ctypes.Structure):
|
||||
_fields_ = [("Offset", wintypes.DWORD), ("OffsetHigh", wintypes.DWORD)]
|
||||
|
||||
class _OffsetUnion(ctypes.Union):
|
||||
_fields_ = [("offset", _Offset), ("Pointer", ctypes.c_void_p)]
|
||||
|
||||
class _Overlapped(ctypes.Structure):
|
||||
_fields_ = [
|
||||
("Internal", ctypes.c_size_t), ("InternalHigh", ctypes.c_size_t),
|
||||
("offset", _OffsetUnion), ("hEvent", wintypes.HANDLE),
|
||||
]
|
||||
|
||||
_kernel32 = ctypes.WinDLL("kernel32", use_last_error=True)
|
||||
_lock_file = _kernel32.LockFileEx
|
||||
_lock_file.argtypes = [
|
||||
wintypes.HANDLE, wintypes.DWORD, wintypes.DWORD,
|
||||
wintypes.DWORD, wintypes.DWORD, ctypes.POINTER(_Overlapped),
|
||||
]
|
||||
_lock_file.restype = wintypes.BOOL
|
||||
_unlock_file = _kernel32.UnlockFileEx
|
||||
_unlock_file.argtypes = [
|
||||
wintypes.HANDLE, wintypes.DWORD, wintypes.DWORD,
|
||||
wintypes.DWORD, ctypes.POINTER(_Overlapped),
|
||||
]
|
||||
_unlock_file.restype = wintypes.BOOL
|
||||
else:
|
||||
import fcntl
|
||||
|
||||
|
||||
def _validate(kind: str, key: str) -> None:
|
||||
if kind not in _KINDS or not isinstance(key, str) or _KEY.fullmatch(key) is None:
|
||||
raise ValueError("Native cache leases require models/objects and a lowercase SHA-256 key.")
|
||||
|
||||
|
||||
def _reparse(info: os.stat_result) -> bool:
|
||||
# Junctions as well as symbolic links are excluded on Windows.
|
||||
return stat.S_ISLNK(info.st_mode) or bool(
|
||||
getattr(info, "st_file_attributes", 0)
|
||||
& getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0x400)
|
||||
)
|
||||
|
||||
|
||||
def _directory(path: Path) -> os.stat_result | None:
|
||||
try:
|
||||
info = path.lstat()
|
||||
except FileNotFoundError:
|
||||
return None
|
||||
if _reparse(info) or not stat.S_ISDIR(info.st_mode):
|
||||
return None
|
||||
return info
|
||||
|
||||
|
||||
def _require_directory(path: Path, *, create: bool = False) -> None:
|
||||
if create:
|
||||
path.mkdir(parents=True, exist_ok=True)
|
||||
if _directory(path) is None:
|
||||
raise RuntimeError(f"Native cache directory must be a real directory: {path}")
|
||||
|
||||
|
||||
def _lock(fd: int, *, exclusive: bool, blocking: bool) -> bool:
|
||||
if os.name == "nt":
|
||||
# os.open supplies a synchronous handle. LockFileEx therefore really
|
||||
# waits without FAIL_IMMEDIATELY; unlike msvcrt.locking it also supports
|
||||
# overlapping shared leases, including multiple builds in one process.
|
||||
# The one-byte lock may extend beyond EOF, so lock files remain empty.
|
||||
overlapped = _Overlapped()
|
||||
flags = (2 if exclusive else 0) | (0 if blocking else 1)
|
||||
if _lock_file(msvcrt.get_osfhandle(fd), flags, 0, 1, 0, ctypes.byref(overlapped)):
|
||||
return True
|
||||
code = ctypes.get_last_error()
|
||||
if not blocking and code == 33: # ERROR_LOCK_VIOLATION
|
||||
return False
|
||||
raise ctypes.WinError(code)
|
||||
operation = fcntl.LOCK_EX if exclusive else fcntl.LOCK_SH
|
||||
if not blocking:
|
||||
operation |= fcntl.LOCK_NB
|
||||
while True:
|
||||
try:
|
||||
fcntl.flock(fd, operation)
|
||||
return True
|
||||
except InterruptedError:
|
||||
continue
|
||||
except OSError as exc:
|
||||
if not blocking and exc.errno in (errno.EACCES, errno.EAGAIN):
|
||||
return False
|
||||
raise
|
||||
|
||||
|
||||
def _release(fd: int, owner_pid: int) -> None:
|
||||
# A forked child's finalizer must not explicitly unlock its parent's flock.
|
||||
# Closing the inherited descriptor alone leaves the parent's lease intact.
|
||||
with suppress(OSError):
|
||||
if os.getpid() == owner_pid:
|
||||
if os.name == "nt":
|
||||
overlapped = _Overlapped()
|
||||
_unlock_file(msvcrt.get_osfhandle(fd), 0, 1, 0, ctypes.byref(overlapped))
|
||||
else:
|
||||
fcntl.flock(fd, fcntl.LOCK_UN)
|
||||
with suppress(OSError):
|
||||
os.close(fd)
|
||||
|
||||
|
||||
class CacheLease:
|
||||
"""An acquired cache-use lock; close explicitly or use as a context manager."""
|
||||
|
||||
__slots__ = ("path", "kind", "key", "exclusive", "_finalizer", "__weakref__")
|
||||
|
||||
def __init__(self, fd: int, path: Path, kind: str, key: str, exclusive: bool):
|
||||
self.path = path
|
||||
self.kind = kind
|
||||
self.key = key
|
||||
self.exclusive = exclusive
|
||||
self._finalizer = weakref.finalize(self, _release, fd, os.getpid())
|
||||
|
||||
@property
|
||||
def closed(self) -> bool:
|
||||
return not self._finalizer.alive
|
||||
|
||||
def close(self) -> None:
|
||||
self._finalizer()
|
||||
|
||||
def __enter__(self) -> CacheLease:
|
||||
if self.closed:
|
||||
raise RuntimeError("Cannot reuse a closed native cache lease.")
|
||||
return self
|
||||
|
||||
def __exit__(self, *_: object) -> None:
|
||||
self.close()
|
||||
|
||||
|
||||
def acquire_cache_lease(
|
||||
cache_dir: Path, kind: str, key: str, *, exclusive: bool = False,
|
||||
blocking: bool = True,
|
||||
) -> CacheLease | None:
|
||||
"""Acquire a shared-use/exclusive-eviction lease; return None only if busy.
|
||||
|
||||
Locks use 128 stable shards per namespace, bounding metadata as model keys
|
||||
accumulate. A collision can defer eviction but cannot admit unsafe deletion.
|
||||
Lock files must never be unlinked while the application is running: waiters
|
||||
may already hold the old inode. There are at most 256 empty lock files.
|
||||
"""
|
||||
_validate(kind, key)
|
||||
shard = int(key[:8], 16) % LOCK_SHARDS
|
||||
return _acquire_lock_file(
|
||||
Path(cache_dir).absolute(), kind, key, f"{kind}-{shard:03d}.lock",
|
||||
exclusive=exclusive, blocking=blocking,
|
||||
)
|
||||
|
||||
|
||||
def _acquire_lock_file(
|
||||
cache: Path, kind: str, key: str, lock_name: str, *, exclusive: bool,
|
||||
blocking: bool,
|
||||
) -> CacheLease | None:
|
||||
_require_directory(cache, create=True)
|
||||
lock_dir = cache / ".locks"
|
||||
_require_directory(lock_dir, create=True)
|
||||
lock_path = lock_dir / lock_name
|
||||
try:
|
||||
info = lock_path.lstat()
|
||||
except FileNotFoundError:
|
||||
info = None
|
||||
if info is not None and (_reparse(info) or not stat.S_ISREG(info.st_mode)):
|
||||
raise RuntimeError(f"Native cache lock must be a regular file: {lock_path}")
|
||||
flags = os.O_RDWR | os.O_CREAT | getattr(os, "O_NOFOLLOW", 0) | getattr(os, "O_BINARY", 0)
|
||||
fd = os.open(lock_path, flags, 0o600)
|
||||
try:
|
||||
if not stat.S_ISREG(os.fstat(fd).st_mode):
|
||||
raise RuntimeError(f"Native cache lock must be a regular file: {lock_path}")
|
||||
os.set_inheritable(fd, False)
|
||||
if not _lock(fd, exclusive=exclusive, blocking=blocking):
|
||||
os.close(fd)
|
||||
return None
|
||||
return CacheLease(fd, cache / kind / key, kind, key, exclusive)
|
||||
except BaseException:
|
||||
os.close(fd)
|
||||
raise
|
||||
|
||||
|
||||
def touch_cache_entry(cache_dir: Path, kind: str, key: str) -> None:
|
||||
"""Mark an immutable entry as recently used while its caller holds a lease."""
|
||||
_validate(kind, key)
|
||||
cache = Path(cache_dir).absolute()
|
||||
_require_directory(cache)
|
||||
_require_directory(cache / kind)
|
||||
entry = cache / kind / key
|
||||
_require_directory(entry)
|
||||
# Directory mtime is separate from the checksummed artifact/manifest bytes.
|
||||
os.utime(entry, None, follow_symlinks=False)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _Entry:
|
||||
path: Path
|
||||
size: int
|
||||
mtime_ns: int
|
||||
device: int
|
||||
inode: int
|
||||
|
||||
|
||||
def _tree_size(path: Path) -> int | None:
|
||||
"""Count regular file bytes without reading files or following any links."""
|
||||
size = 0
|
||||
try:
|
||||
with os.scandir(path) as children:
|
||||
for child in children:
|
||||
info = child.stat(follow_symlinks=False)
|
||||
if _reparse(info):
|
||||
return None
|
||||
if stat.S_ISREG(info.st_mode):
|
||||
size += info.st_size
|
||||
elif stat.S_ISDIR(info.st_mode):
|
||||
nested = _tree_size(Path(child.path))
|
||||
if nested is None:
|
||||
return None
|
||||
size += nested
|
||||
else:
|
||||
return None
|
||||
except FileNotFoundError:
|
||||
return None # A concurrent sweep may already have removed this entry.
|
||||
return size
|
||||
|
||||
|
||||
def _scan(namespace: Path) -> tuple[list[_Entry], int]:
|
||||
entries = []
|
||||
skipped = 0
|
||||
if _directory(namespace) is None:
|
||||
try:
|
||||
namespace.lstat()
|
||||
except FileNotFoundError:
|
||||
return entries, 0
|
||||
return entries, 1
|
||||
with os.scandir(namespace) as children:
|
||||
for child in children:
|
||||
if _KEY.fullmatch(child.name) is None:
|
||||
skipped += 1
|
||||
continue
|
||||
path = Path(child.path)
|
||||
info = _directory(path)
|
||||
if info is None:
|
||||
skipped += 1
|
||||
continue
|
||||
size = _tree_size(path)
|
||||
if size is None:
|
||||
skipped += 1
|
||||
continue
|
||||
entries.append(_Entry(path, size, info.st_mtime_ns, info.st_dev, info.st_ino))
|
||||
return entries, skipped
|
||||
|
||||
|
||||
def _empty_report(limit: int) -> dict:
|
||||
return {
|
||||
"limitBytes": limit, "beforeBytes": 0, "afterBytes": 0,
|
||||
"removedBytes": 0, "removedEntries": 0, "skippedInUse": 0,
|
||||
"skippedChanged": 0, "skippedUnmanaged": 0, "oversizedEntries": 0,
|
||||
"overLimitBytes": 0, "skippedConcurrentSweep": False, "errors": [],
|
||||
}
|
||||
|
||||
|
||||
def _prune_kind(cache: Path, kind: str, limit: int) -> dict:
|
||||
report = _empty_report(limit)
|
||||
namespace = cache / kind
|
||||
try:
|
||||
entries, skipped = _scan(namespace)
|
||||
except OSError as exc:
|
||||
report["errors"].append(str(exc))
|
||||
return report
|
||||
report["skippedUnmanaged"] = skipped
|
||||
remaining = report["beforeBytes"] = sum(entry.size for entry in entries)
|
||||
remaining_count = len(entries)
|
||||
for entry in sorted(entries, key=lambda item: (item.mtime_ns, item.path.name)):
|
||||
if remaining <= limit:
|
||||
break
|
||||
# Keep the last entry if it alone exceeds the budget. Older oversized
|
||||
# entries are still evictable, so they cannot accumulate unboundedly.
|
||||
if remaining_count == 1 and entry.size > limit:
|
||||
continue
|
||||
try:
|
||||
lease = acquire_cache_lease(cache, kind, entry.path.name, exclusive=True, blocking=False)
|
||||
if lease is None:
|
||||
report["skippedInUse"] += 1
|
||||
continue
|
||||
with lease:
|
||||
info = _directory(entry.path)
|
||||
if info is None:
|
||||
report["skippedChanged"] += 1
|
||||
continue
|
||||
if (info.st_dev, info.st_ino, info.st_mtime_ns) != (
|
||||
entry.device, entry.inode, entry.mtime_ns,
|
||||
):
|
||||
# It was republished or reused after sorting; defer it to
|
||||
# the next sweep instead of evicting a freshly used build.
|
||||
report["skippedChanged"] += 1
|
||||
continue
|
||||
size = _tree_size(entry.path)
|
||||
if size is None:
|
||||
report["skippedUnmanaged"] += 1
|
||||
continue
|
||||
shutil.rmtree(entry.path)
|
||||
remaining -= entry.size
|
||||
remaining_count -= 1
|
||||
report["removedBytes"] += size
|
||||
report["removedEntries"] += 1
|
||||
except (OSError, RuntimeError) as exc:
|
||||
# Cleanup is best effort. A locked executable, permissions or a
|
||||
# damaged unrelated cache entry must not fail a valid simulation.
|
||||
report["errors"].append(str(exc))
|
||||
try:
|
||||
current, _ = _scan(namespace)
|
||||
report["afterBytes"] = sum(entry.size for entry in current)
|
||||
report["oversizedEntries"] = sum(entry.size > limit for entry in current)
|
||||
except OSError as exc:
|
||||
report["errors"].append(str(exc))
|
||||
report["afterBytes"] = remaining
|
||||
report["overLimitBytes"] = max(0, report["afterBytes"] - limit)
|
||||
return report
|
||||
|
||||
|
||||
|
||||
def _empty_orphan_report() -> dict:
|
||||
return {
|
||||
"orphanStagesRemoved": 0, "orphanStagesBytes": 0,
|
||||
"skippedInUse": 0, "skippedUnmanaged": 0,
|
||||
"skippedConcurrentSweep": False, "errors": [],
|
||||
}
|
||||
|
||||
|
||||
def _prune_orphan_stages(cache: Path) -> dict:
|
||||
"""Remove only named stages whose builder's lease is no longer held.
|
||||
|
||||
Callers acquire the embedded key's shared lease BEFORE creating the stage
|
||||
and retain it until publication or cleanup. The OS releases that protection
|
||||
if the builder is killed; no age threshold or unreliable PID test is needed.
|
||||
Root/models stages use models leases; objects stages use objects leases.
|
||||
"""
|
||||
report = _empty_orphan_report()
|
||||
for namespace, kind in ((cache, "models"), (cache / "models", "models"),
|
||||
(cache / "objects", "objects")):
|
||||
try:
|
||||
if _directory(namespace) is None:
|
||||
continue
|
||||
with os.scandir(namespace) as children:
|
||||
candidates = []
|
||||
for child in children:
|
||||
match = _STAGE.fullmatch(child.name)
|
||||
if match is not None:
|
||||
candidates.append((Path(child.path), match.group(1)))
|
||||
elif child.name.startswith("building-"):
|
||||
report["skippedUnmanaged"] += 1
|
||||
except OSError as exc:
|
||||
report["errors"].append(f"{namespace}: {exc}")
|
||||
continue
|
||||
for stage, key in candidates:
|
||||
try:
|
||||
lease = acquire_cache_lease(cache, kind, key, exclusive=True, blocking=False)
|
||||
if lease is None:
|
||||
report["skippedInUse"] += 1
|
||||
continue
|
||||
with lease:
|
||||
if _directory(stage) is None:
|
||||
report["skippedUnmanaged"] += 1
|
||||
continue
|
||||
size = _tree_size(stage)
|
||||
if size is None:
|
||||
report["skippedUnmanaged"] += 1
|
||||
continue
|
||||
shutil.rmtree(stage)
|
||||
report["orphanStagesRemoved"] += 1
|
||||
report["orphanStagesBytes"] += size
|
||||
except (OSError, RuntimeError) as exc:
|
||||
report["errors"].append(f"{stage}: {exc}")
|
||||
return report
|
||||
|
||||
|
||||
def prune_cache(
|
||||
cache_dir: Path, *, model_limit_bytes: int = MODEL_LIMIT_BYTES,
|
||||
object_limit_bytes: int = OBJECT_LIMIT_BYTES,
|
||||
) -> dict:
|
||||
"""Apply separate soft LRU byte budgets to managed models and objects.
|
||||
|
||||
In-use entries (including conservative shard collisions), the final entry
|
||||
when it alone exceeds the budget, unknown names, links and legacy root-level
|
||||
builds are retained. Returned counts explain any remaining overage. The
|
||||
cache may transiently exceed its budgets while builds or solvers are active.
|
||||
Strictly named stages abandoned by killed builders are removed under their
|
||||
matching lease and counted separately in the orphanStages report.
|
||||
"""
|
||||
for limit in (model_limit_bytes, object_limit_bytes):
|
||||
if isinstance(limit, bool) or not isinstance(limit, int) or limit < 0:
|
||||
raise ValueError("Native cache byte limits must be nonnegative integers.")
|
||||
cache = Path(cache_dir).absolute()
|
||||
empty = {
|
||||
"models": _empty_report(model_limit_bytes),
|
||||
"objects": _empty_report(object_limit_bytes),
|
||||
"orphanStages": _empty_orphan_report(),
|
||||
}
|
||||
try:
|
||||
if _directory(cache) is None:
|
||||
# Do not create a missing cache or inspect an alias to a directory.
|
||||
return empty
|
||||
# One additional stable lock file (257 total) serializes sweep decisions
|
||||
# without serializing readers/builds. Otherwise simultaneous sweeps can
|
||||
# each remove a different entry and incorrectly discard the final one.
|
||||
sweep = _acquire_lock_file(
|
||||
cache, "models", "0" * 64, "prune.lock", exclusive=True, blocking=False,
|
||||
)
|
||||
if sweep is None:
|
||||
for report in empty.values():
|
||||
report["skippedConcurrentSweep"] = True
|
||||
return empty
|
||||
with sweep:
|
||||
orphan_stages = _prune_orphan_stages(cache)
|
||||
return {
|
||||
"models": _prune_kind(cache, "models", model_limit_bytes),
|
||||
"objects": _prune_kind(cache, "objects", object_limit_bytes),
|
||||
"orphanStages": orphan_stages,
|
||||
}
|
||||
except (OSError, RuntimeError) as exc:
|
||||
for report in empty.values():
|
||||
report["errors"].append(str(exc))
|
||||
return empty
|
||||
@@ -0,0 +1,34 @@
|
||||
"""Reviewed native component module exports and their compilation dependencies."""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
EXPORTS = {
|
||||
"properties": frozenset({
|
||||
"native_properties_init", "native_temperature_ph_context", "native_density_context",
|
||||
"native_gas_init", "native_gas", "native_gas_context", "native_medium_init",
|
||||
"native_density", "native_temperature_ph", "native_viscosity",
|
||||
"native_medium_gas", "native_medium_gas_context",
|
||||
}),
|
||||
"orifice": frozenset({"native_orifice", "native_orifice_context", "native_medium_orifice", "native_medium_orifice_context"}),
|
||||
"pipe": frozenset({"native_pipe_resistance", "native_pipe_flow", "native_pipe_flow_context",
|
||||
"native_pipe_flow_cached", "native_pipe_flow_cached_context",
|
||||
"native_pipe_diagnostics", "native_pipe_diagnostics_context"}),
|
||||
"mechanics": frozenset({"native_contact", "native_stop_motion", "native_limit_force"}),
|
||||
"signal": frozenset({"native_signal", "native_signal_break"}),
|
||||
}
|
||||
DEPENDENCIES = {"orifice": ("properties",), "pipe": ("properties",)}
|
||||
|
||||
|
||||
def component_modules(source: str) -> tuple[str, ...]:
|
||||
# Generated C refers to these functions directly. Include identifier uses
|
||||
# (also function pointers), conservatively including any comments/strings.
|
||||
identifiers = set(re.findall(r"\bnative_[A-Za-z0-9_]+\b", source))
|
||||
selected = {name for name, exports in EXPORTS.items() if identifiers & exports}
|
||||
pending = list(selected)
|
||||
while pending:
|
||||
for dependency in DEPENDENCIES.get(pending.pop(), ()):
|
||||
if dependency not in selected:
|
||||
selected.add(dependency)
|
||||
pending.append(dependency)
|
||||
return tuple(name for name in EXPORTS if name in selected)
|
||||
@@ -105,6 +105,7 @@ def execute_native(build: NativeBuild, config: SolveIVPConfig, sample_step: floa
|
||||
payload["buildKey"] = build.manifest["buildKey"]
|
||||
payload["cacheHit"] = build.cache_hit
|
||||
payload["buildSeconds"] = build.seconds
|
||||
payload["buildDetails"] = build.details
|
||||
if activity_tracker is not None:
|
||||
activity_tracker.record_native_progress(payload["simulatedUntil"], payload["nfev"], payload["acceptedSteps"])
|
||||
return payload
|
||||
@@ -115,13 +116,20 @@ def simulate_native(network, config, *, sample_step, progress_callback=None,
|
||||
if config.method not in ("RK45", "BDF"):
|
||||
raise NativeCapabilityError(f"Native v1 does not support method {config.method}.")
|
||||
if progress_callback:
|
||||
progress_callback(0.0, "initializing")
|
||||
progress_callback(0.0, "native-generation")
|
||||
program = compile_native_program(network)
|
||||
build = build_native(program)
|
||||
with tempfile.TemporaryDirectory(prefix="native-simulation-") as directory:
|
||||
data = execute_native(build, config, sample_step, run_dir=Path(directory),
|
||||
cancel_check=cancel_check, progress_callback=progress_callback,
|
||||
activity_tracker=activity_tracker, raw_series=raw_series)
|
||||
build = build_native(program, progress_callback=(
|
||||
(lambda phase: progress_callback(0.0, phase)) if progress_callback else None
|
||||
))
|
||||
try:
|
||||
with tempfile.TemporaryDirectory(prefix="native-simulation-") as directory:
|
||||
data = execute_native(build, config, sample_step, run_dir=Path(directory),
|
||||
cancel_check=cancel_check, progress_callback=progress_callback,
|
||||
activity_tracker=activity_tracker, raw_series=raw_series)
|
||||
finally:
|
||||
# Keep the executable pinned through process exit and result reading;
|
||||
# only then may cache eviction reclaim this completed model version.
|
||||
build.close()
|
||||
totals = {
|
||||
"nfev": data["nfev"], "njev": data["njev"], "nlu": data["nlu"],
|
||||
"acceptedStepCount": data["acceptedSteps"], "rejectedStepCount": data["rejectedSteps"],
|
||||
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,115 @@
|
||||
# 按需编译与分层缓存优化
|
||||
|
||||
日期：2026-09-12。基线提交 `aa4951b14ea6453138e3e877f1167a28100f486f`。本轮已接入正常构建入口，采用“可复用目标文件 + 完整模型缓存 + 容量预算/LRU”。同机八路首次构建中位 3.7742 → 2.4875 s，减少 34.09%；完整缓存命中的检查增加约 0.13 s。数值方程、牛顿求根、着色雅可比、`rtol=1e-8`、采样和结果编码保持当前已接受版本。
|
||||
|
||||
## 编译与复用方式
|
||||
|
||||
原来 `kernels.c` 中的实现拆为物性、孔口、管路、机械、信号五个功能模块。59 个原函数体逐字一致。构建器从生成代码中查找实际引用的导出函数，补齐依赖后选择模块；例如管路、孔口依赖物性。固定运行库仍需编译。粒度是功能模块，不是每个元件实例；同一模块内的相关函数会一起编译。
|
||||
|
||||
首次只编译所需单元，最多 4 个并行任务。每个单元预处理成确切的 C 内容，以预处理字节、逻辑文件名、编译器和编译选项计算对象键，再编译这份内容。缓存有效性覆盖实际使用的宏和间接头文件。完整模型另核对生成源码/头文件、组件合同、对象键、链接库和依赖声明。
|
||||
|
||||
系统改参数通常只影响生成的 `model.c`；拓扑或状态布局变化也可能使包含 `model.h` 的运行单元失效。完整模型仍在缓存时直接复用可执行文件，否则复用未变化的 `.o` 再链接。时间范围、步长、rtol 等运行选项不编入模型。缓存跨服务重启保留；服务重启本身不要求重新编译。
|
||||
|
||||
A/B/C/D 对应的模块对象可共同留在对象池，各系统的可执行文件也可分别留在模型池。容量回收后可能只保留部分对象或模型；再用到被清理内容时自动重建。这样能跨项目共享相同内容，也能限制长期增长。
|
||||
|
||||
## 文件位置与容量管理
|
||||
|
||||
所有新编译缓存都在 `app/data/native-builds/`，已被 Git 忽略：
|
||||
|
||||
| 位置 | 内容 | 默认容量预算 |
|
||||
| --- | --- | ---: |
|
||||
| `models/<SHA>/` | 完整模型可执行文件、生成源码、合同清单及必要 DLL | 256 MiB |
|
||||
| `objects/<SHA>/` | 可复用目标文件及校验清单 | 128 MiB |
|
||||
| `.locks/` | 固定分片的进程使用锁和清理锁 | 最多 257 个空锁文件 |
|
||||
|
||||
环境变量 `SIMULATION_NATIVE_MODEL_CACHE_MB`、`SIMULATION_NATIVE_OBJECT_CACHE_MB` 可设为非负整数 MiB。按最近使用时间淘汰较久未用内容；两池分别计量。模型淘汰不强制删除其共享对象，对象淘汰也不影响已链接模型。再次运行被淘汰的模型只是需要重新构建，不会丢失工程或仿真结果。
|
||||
|
||||
预算是**安全软上限**：正在构建/运行的条目跳过，锁分片碰撞也可能暂缓清理；单个条目已超过预算时保留最后一份并报告超额。计量为受管理文件的逻辑字节，不含文件系统元数据、锁、正在写入的临时目录以及旧格式缓存。因此不能将 384 MiB 理解为磁盘占用的绝对硬上限。
|
||||
|
||||
旧格式的根级 `<SHA>/` 未自动迁移或删除，本机检查时有 229 份、89,754,121 字节（85.60 MiB），不计入新预算，新代码也不会继续向旧格式追加。保留它们是为了不影响仍可能使用旧构建器的进程；它们的位置与新 `models/objects/` 分开，后续可在确认旧进程停止后单独整理。
|
||||
|
||||
普通失败清理临时目录；进程被强制终止遗留的新格式 `building-<SHA>-<随机后缀>/`，在后续清理时先取得对应排他锁，再回收。并发清理由全局锁串行化，避免重复淘汰。缓存缺文件、校验失败、符号链接或合同不一致时明确报错，不执行未经核验的缓存。
|
||||
|
||||
## 构建成本对照
|
||||
|
||||
使用冻结的基线源码和当前候选，各 1 轮预热、3 轮正式测量，每轮从独立空应用缓存开始；交替版本顺序，整个序列共 48 个案例。没有清空操作系统文件缓存。编译器、编译参数、链接库、各案例的生成 C/头文件/XML 均核对一致。统计是两组各自的中位数。
|
||||
|
||||
本表计时包含构建函数及紧邻的使用锁释放/容量清理，不含 Python 导入、XML/C 生成和求解。旧版没有对象阶段细分，缺失值保留为空，不用差值伪造。
|
||||
|
||||
| 场景 | 原版 | 新版 | 变化 |
|
||||
| --- | ---: | ---: | ---: |
|
||||
| 八路首次构建（空应用缓存） | 3.7742 s | 2.4875 s | 减少 34.09% |
|
||||
| 相同八路再次运行 | 0.0347 s | 0.1632 s | 增加 128.6 ms |
|
||||
| 八路改一个管长参数 | 3.9009 s | 2.4838 s | 减少 36.33% |
|
||||
| 改参数后切回原八路 | 0.0335 s | 0.1868 s | 增加 153.4 ms |
|
||||
| 八路切换到修正四路 | 2.4908 s | 1.1109 s | 减少 55.40% |
|
||||
| 四路切回已缓存八路 | 0.0332 s | 0.1580 s | 增加 124.9 ms |
|
||||
|
||||
参数案例仅在隔离副本中将 `amesim_pnl0001_1.le` 从 1 改为 1.001 m；`tests/data` 正式模型未改。切四路使用现有修正四路工程，最后重新用原八路。
|
||||
|
||||
新版首次编译 12 个单元；仅改管长时复用 11 个、重编 1 个；切四路复用 7 个、重编 5 个。八路和四路实际上都使用五类功能模块，所以这组速度改善来自并行构建和对象复用，不能归结为它们省掉了某个未用元件模块。未用模块不会触发编译/失效的行为另有专项测试。
|
||||
|
||||
| 新版场景 | 预处理墙钟 | 并行编译墙钟 | 链接墙钟 |
|
||||
| --- | ---: | ---: | ---: |
|
||||
| 八路首次 | 0.0965 s | 2.2163 s | 0.0490 s |
|
||||
| 八路改参数 | 0.0925 s | 2.2261 s | 0.0494 s |
|
||||
| 切换四路 | 0.0956 s | 0.8880 s | 0.0490 s |
|
||||
| 相同八路完整命中 | 0.0966 s | 0 | 0 |
|
||||
|
||||
阶段是嵌套观测值，各阶段的中位数不能相加后当作一次实测总时长。`compileSeconds` 是并行任务墙钟之和，不是 CPU 时间，也不能与 `compileWallSeconds` 累加。
|
||||
|
||||
仍有两点代价：
|
||||
|
||||
- 完整命中仍做预处理和内容校验，检查从 34.7 ms 增到 163.2 ms。保留这层检查可准确识别头文件/宏变化；本轮不把它描述为命中速度优化。
|
||||
- 八路生成的 `model.c` 单独编译仍约 2.23 s，占主要成本。改一个参数虽然少编 11 个单元，墙钟仍接近新版首次构建；本轮没有把模型参数改成运行时载入。
|
||||
|
||||
运行序列结束时，旧缓存逻辑文件 4,230,318 字节，新缓存 5,064,224 字节（其中共享对象 788,363 字节），增加约 19.7%。首次单模型分别为 1,638,937 / 2,034,665 字节。新增对象池用一些空间换取重用，容量预算负责控制后续增长。性能案例没有触及预算；实际淘汰在独立并发/LRU 测试中验证。
|
||||
|
||||
## 数值与网页验证
|
||||
|
||||
修正八路 0～10 s、BDF、`rtol=1e-8` 完整运行。原生 `series/final/finalState` 的 **1,790,486 个 binary64 值逐位一致**，包括正负零。参考来自上一轮已接受的着色雅可比结果。`nfev=22853`、`njev=433`、雅可比 RHS 12124、普通 CVODE RHS 10729、接受步 6660、拒绝步 359、事件 1、启动 4、雅可比回退 0，均未变化。
|
||||
|
||||
用生产前端静态资源和真实浏览器在临时 8024 服务执行 1 次预热 + 3 次测量。首次完整模型未命中，但已有 7 个对象可用，因此它不是上表的全空缓存案例。首次构建 2.5649 s、点击到可查看 6.1815 s。随后三次中位数：
|
||||
|
||||
| 网页过程 | 耗时 |
|
||||
| --- | ---: |
|
||||
| 后台构建缓存检查 | 0.1976 s |
|
||||
| C 求解 | 2.6489 s |
|
||||
| 点击运行 → 结果可查看 | 3.7586 s |
|
||||
| 点击运行 → 浏览器 IndexedDB 保存完成 | 3.8605 s |
|
||||
| CSV 点击 → 下载保存完成 | 1.0348 s |
|
||||
|
||||
可查看是完成状态和可用按钮的 DOM 观测；保存是事务完成后的索引发布观测；下载包含浏览器通知及自动化 `saveAs`。过程存在重叠，不能累加。此次是本机回环地址验证，未复测远程端口转发链路，也不拿历史网页样本计算本轮配对加速比。
|
||||
|
||||
4 次完整网页结果、CSV、刷新恢复均与参考逐数值一致，共核对 7,154,280 个 CSV 单元；所有 CSV 文件字节相同。前端只调整了准备阶段文案/进度识别，显示生成、检查缓存、编译及链接的实际阶段。
|
||||
|
||||
本轮未重新运行 Amesim 软件或给出新增 Amesim 速度结论。四路已有外部曲线基准仍纳入回归，门槛保持不变；八路数值结论是对已接受的 native 版本等价。
|
||||
|
||||
## 回归与适用范围
|
||||
|
||||
- 93 项后端回归通过（54.123 s）：元件/物性/管路、50 个网络冻结基准、四路曲线门槛、BDF/RK45、生成器、事件、取消、雅可比、结果接口等。
|
||||
- 收尾构建/存储专项 37 项通过（7.397 s，与前述部分重叠）：真实 GCC 构建、跨模型复用、参数/头文件/宏/编译选项/链接库失效、缓存完整性、编译链接失败、并发发布、活跃进程期间清理、LRU/软预算、符号链接、清理失败、被杀进程遗留目录回收。
|
||||
- 前端构建通过；仿真超时与活动看门狗 12 项通过；实际浏览器四轮下载与恢复核对通过。
|
||||
- 诊断工具已适配新的模块结构，避免聚合入口和模块重复编译；历史管路插桩工具做了语法/插桩准备检查，本轮未重跑其整套历史性能实验。
|
||||
|
||||
实际平台为 Linux x86_64、Python 3.12.3、GCC 13.3.0、SUNDIALS 7.4.0。Windows 使用 `LockFileEx` 的共享/排他锁实现，但没有 Windows 实机测试。最终补充生成 C/H 显式 LF 写出，避免 Windows 默认 CRLF 导致身份校验不一致；以模拟平台换行的真实构建回归验证。该补充在正式计时之后，对本次 Linux 写出字节没有影响，未为此重复 48 次计时。来源摘要分别保存计时版本与最终版本 SHA。
|
||||
|
||||
## 记录与复现
|
||||
|
||||
轻量 [机器可读摘要](assets/2026-09-12/native-build-cache-summary.json) 保存场景统计、来源 SHA、容量规则和网页核对。原始记录位于 Git 忽略的 `test/incremental-build-20260912/`：
|
||||
|
||||
- `baseline-source/`、`source-manifest.json`：原版本冻结源码与清单。
|
||||
- `benchmark/summary.json`、`benchmark/timings.csv`、各轮案例目录：48 次构建记录。
|
||||
- `browser/summary.json`、`browser-equality.json`、`native-bitwise-equality.json`：网页计时与一致性。
|
||||
- `regression.log`、`frontend-build.log`、`frontend-regression.log`：验证日志；最终换行专项结果另见摘要。
|
||||
|
||||
```sh
|
||||
.venv/bin/python tests/manual/benchmark_native_build_cache.py \
|
||||
--baseline-root test/incremental-build-20260912/baseline-source \
|
||||
--output-dir test/incremental-build-20260912/reproduction \
|
||||
--warmups 1 --repeats 3 --run
|
||||
|
||||
.venv/bin/python -m unittest tests.test_native_build_cache tests.test_native_cache_storage -v
|
||||
```
|
||||
|
||||
本轮没有安装新环境，环境/缓存/大结果均留在忽略目录。5173、8000 保持停止，临时 8024 已停止。此前结果传输压缩仍只在备份目录，未启用到生产。改动保留在工作区，未提交或推送 Git。
|
||||
@@ -0,0 +1,9 @@
|
||||
# 2026-09-12：按需编译与分层缓存
|
||||
|
||||
原生构建改为按实际引用选择功能模块，复用内容校验的目标文件，保留完整模型缓存。两池默认预算分别为 128 MiB 和 256 MiB，按最近使用时间清理；跨进程使用锁保护活跃构建/运行，超大单条目和在用内容允许暂时超额。旧根级缓存单独保留，新代码不继续追加旧格式。
|
||||
|
||||
冻结原版本同机对照，八路首次构建 3.7742→2.4875 s（-34.09%）、修改一个管长参数 3.9009→2.4838 s（-36.33%）、切四路 2.4908→1.1109 s（-55.40%）。完整模型命中检查增加约 0.13 s，已披露。八路原生 1,790,486 个 binary64 值逐位一致；网页三次中位可查看 3.7586 s、保存 3.8605 s、CSV 1.0348 s，四轮完整结果/下载/刷新一致。
|
||||
|
||||
93 项后端回归、收尾 37 项缓存专项（部分重叠）、前端构建和 12 项超时/看门狗测试通过。Windows 换行做了模拟回归，未实机验证 Windows 锁实现。正常 5173/8000 服务保持停止，临时验证服务已停止；传输压缩仍仅归档备份。本轮未安装环境、提交或推送 Git。
|
||||
|
||||
实现、缓存目录、容量边界、阶段成本和原始记录见 [完整报告](../other/按需编译与分层缓存优化-2026-09-12.md)。
|
||||
@@ -7749,7 +7749,7 @@ function FlowWorkbench() {
|
||||
"info",
|
||||
`开始仿真：${projectNameRef.current || "untitled"}，时间范围 ${project.simulation.t_start} - ${project.simulation.t_stop} s`,
|
||||
);
|
||||
await runBusyAction("正在编译模型并运行仿真...", async () => {
|
||||
await runBusyAction("正在准备并运行仿真...", async () => {
|
||||
const xml = buildSystemXml(project, componentDefinitions, nodesRef.current);
|
||||
const simulationId = createSimulationId();
|
||||
activeSimulationIdRef.current = simulationId;
|
||||
@@ -7774,6 +7774,11 @@ function FlowWorkbench() {
|
||||
"compilation",
|
||||
"initialization",
|
||||
"initializing",
|
||||
"native-generation",
|
||||
"native-cache-check",
|
||||
"native-cache-hit",
|
||||
"native-compilation",
|
||||
"native-linking",
|
||||
].includes(event.phase);
|
||||
const progressMessage =
|
||||
watchdog?.status === "slow-active"
|
||||
|
||||
+6
-3
@@ -66,15 +66,18 @@ BDF 默认使用结构着色差分，无需环境变量或网页选项。修正
|
||||
- 网页和 Python CLI 默认 `rtol=1e-8`；生成的状态绝对误差限为质量 `1e-14 kg`、内能 `1e-8 J`、速度/位移 `1e-12`（各自 SI 单位）。独立 C 程序默认 `rtol=1e-6`，对照时应显式传入。CLI 可覆盖 rtol；本版不支持自定义 atol 或 first_step。不同积分器相同局部容差不保证全局曲线误差完全相同。
|
||||
- 时间信号显式分段，塑性/反弹端挡用稠密插值定位并重启。试探 RHS 不修改已接受状态。柔性接触沿用现有分段力公式，不改变刚度或阻尼来提速。
|
||||
- 每任务独立进程，支持进度、取消及超时。进程崩溃不会作为成功返回，受控失败保留最后接受状态。
|
||||
- 编译缓存位于 `app/data/native-builds/`，校验模型、组件合同、源码、编译选项及依赖哈希。本版参数特化入 C，修改模型数值参数会重新编译；修改时间、步长、rtol、采样选项可复用 EXE。
|
||||
- 编译缓存位于 `app/data/native-builds/`：`models/<SHA>/` 保存完整模型，`objects/<SHA>/` 保存可跨模型复用的模块目标文件。按当前模型使用的元件函数及其依赖选择模块，最多并行编译 4 个缺失单元。预处理后的实际 C 内容、工具链和编译选项组成对象键；完整模型键另含生成源码、组件合同及链接依赖。模型数值参数仍特化入 C，但仅重编受到影响的单元；时间、步长、rtol、采样选项仍是运行参数。
|
||||
- 完整模型默认预算 256 MiB、对象预算 128 MiB，可分别设置非负整数环境变量 `SIMULATION_NATIVE_MODEL_CACHE_MB`、`SIMULATION_NATIVE_OBJECT_CACHE_MB`。按目录最近使用时间执行 LRU；活跃构建/运行及最后一个单独超额的条目保留并报告超额，因此是安全软上限。预算计算受管理文件的逻辑字节，不含旧版根级缓存、锁及文件系统元数据。
|
||||
- 缓存读写带跨进程使用锁，命中时核验工件和身份，损坏缓存明确失败。对象复制到构建私有目录后再链接；模型使用锁一直保留到进程退出和结果读回。失败正常清理临时文件；进程被杀遗留的新格式构建目录在下一次清理时按锁状态回收。旧格式缓存保留，不自动迁移或删除，避免影响旧服务。
|
||||
- Python 调用者若长期保留 `NativeBuild`，需在最后一次执行/打包后调用 `build.close()` 释放使用保护并触发清理；网页 runner 自动处理。返回的 `buildDetails` 区分完整命中、目标文件命中/编译次数、预处理、并行编译墙钟、编译任务耗时之和、链接和容量清理结果。
|
||||
|
||||
## 代码职责
|
||||
|
||||
- `app/simulation/native_codegen/input.py`：CLI 输入适配。
|
||||
- `contracts.py`：逐组件 C 实现版本白名单，新增模型或版本不会自动视为已支持。
|
||||
- `compiler.py` / `extended.py`：能力检查、状态/输出布局、连接分组、常系数约束消元、C 生成。保留已验证的简单拓扑快速生成路径，两条路径均只运行 C 数值代码。
|
||||
- `build.py` / `runner.py`：编译缓存、依赖打包、隔离执行与结果适配。
|
||||
- `native/components/kernels.c`：空气/氦气物性、阀与管路流量、信号、摩擦接触和端挡原语。
|
||||
- `build.py` / `modules.py` / `cache_storage.py`：按需模块、两层编译缓存、完整性校验与有界 LRU；`runner.py`：隔离执行、使用保护与结果适配。
|
||||
- `native/components/modules/`：物性、孔口、管路、机械和信号模块。`kernels.c` 仅供诊断的聚合入口，生产构建不再将聚合入口与模块重复编译。
|
||||
- `native/runtime/`：RK45、CVODE 适配、事件定位、采样与 CLI。
|
||||
- `app/simulation/backends.py` / `results.py`：共用执行入口与结果合同。
|
||||
|
||||
|
||||
+11
-485
@@ -1,486 +1,12 @@
|
||||
/* Helium Peng-Robinson and compressible-orifice kernels.
|
||||
* Ported from the project's Python physical equations; validated independently
|
||||
* against Python at ordinary, reverse-flow and contact trial states.
|
||||
/* Compatibility translation unit for standalone kernel diagnostics.
|
||||
* Production builds select individual module sources and compile each once;
|
||||
* never compile this entry together with those files. The internal-linkage
|
||||
* switch preserves the original single-translation-unit diagnostic behavior.
|
||||
* Paths also resolve through -I native/include when a harness copies this text.
|
||||
*/
|
||||
#include "kernels.h"
|
||||
#include <math.h>
|
||||
#include <stddef.h>
|
||||
|
||||
#define RU 8.31446261815324
|
||||
#define MOLAR_MASS 0.004002602
|
||||
#define TC 5.1953
|
||||
#define PC 227460.0
|
||||
#define OMEGA (-0.382)
|
||||
static const double PI=3.1415926535897932384626433832795;
|
||||
static const double pr_a=.457235583*RU*RU*TC*TC/PC;
|
||||
static const double pr_b=.07779607*RU*TC/PC;
|
||||
static const double kappa=.37464+1.54226*OMEGA-.26992*OMEGA*OMEGA;
|
||||
static const double rg=RU/MOLAR_MASS;
|
||||
static const NativeMedium helium_medium={1,RU/MOLAR_MASS,2.5*RU/MOLAR_MASS,293.15,0,1.96e-5,293.15,79.4};
|
||||
|
||||
static int same_medium(const NativeMedium *a,const NativeMedium *b) {
|
||||
return a->real_helium==b->real_helium && a->R==b->R && a->cp==b->cp &&
|
||||
a->Tref==b->Tref && a->slope==b->slope && a->mu==b->mu && a->muT==b->muT && a->S==b->S;
|
||||
}
|
||||
void native_properties_init(NativePropertyCache *cache,NativePropertyState *states,size_t capacity) {
|
||||
cache->states=states;cache->count=0;cache->capacity=states?capacity:0;
|
||||
}
|
||||
static NativePropertyState *property_new(NativePropertyCache *cache,const NativeMedium *m,
|
||||
double p,double T,NativePropertyState *scratch) {
|
||||
int valid=p>0 && T>0 && isfinite(p) && isfinite(T);
|
||||
NativePropertyState *s=valid && cache && cache->count<cache->capacity ? &cache->states[cache->count++] : scratch;
|
||||
*s=(NativePropertyState){0};s->medium=*m;s->p=p;s->T=T;s->valid=valid?NATIVE_PROPERTY_PT:0;
|
||||
return s;
|
||||
}
|
||||
static NativePropertyState *property_pt(NativePropertyCache *cache,const NativeMedium *m,
|
||||
double p,double T,NativePropertyState *scratch) {
|
||||
if(cache)for(size_t i=0;i<cache->count;i++) {
|
||||
NativePropertyState *s=&cache->states[i];
|
||||
if((s->valid&NATIVE_PROPERTY_PT) && s->p==p && s->T==T && same_medium(&s->medium,m))return s;
|
||||
}
|
||||
return property_new(cache,m,p,T,scratch);
|
||||
}
|
||||
static double property_density(NativePropertyState *s) {
|
||||
if(!(s->valid&NATIVE_PROPERTY_RHO)) {
|
||||
s->rho=native_density(&s->medium,s->p,s->T);
|
||||
if(s->rho>0 && isfinite(s->rho))s->valid|=NATIVE_PROPERTY_RHO;
|
||||
}
|
||||
return s->rho;
|
||||
}
|
||||
static double property_viscosity(NativePropertyState *s) {
|
||||
if(!(s->valid&NATIVE_PROPERTY_MU)) {
|
||||
s->mu=native_viscosity(&s->medium,s->T,0);
|
||||
if(s->mu>0 && isfinite(s->mu))s->valid|=NATIVE_PROPERTY_MU;
|
||||
}
|
||||
return s->mu;
|
||||
}
|
||||
static void remember_gas(NativePropertyCache *cache,const NativeMedium *m,const NativeGas *gas) {
|
||||
/* Below helium's critical temperature retain the existing vapor-root/PH
|
||||
selection. A future phase-aware medium contract can carry that state. */
|
||||
if(!cache || (m->real_helium && gas->T<=TC))return;
|
||||
NativePropertyState scratch,*s=property_pt(cache,m,gas->p,gas->T,&scratch);
|
||||
if(!(s->valid&NATIVE_PROPERTY_PT) || !isfinite(gas->h))return;
|
||||
if((s->valid&NATIVE_PROPERTY_H) && s->h!=gas->h)
|
||||
s=property_new(cache,m,gas->p,gas->T,&scratch);
|
||||
s->h=gas->h;s->valid|=NATIVE_PROPERTY_H;
|
||||
if(gas->rho>0 && isfinite(gas->rho)){s->rho=gas->rho;s->valid|=NATIVE_PROPERTY_RHO;}
|
||||
}
|
||||
double native_temperature_ph_context(NativePropertyCache *cache,const NativeMedium *m,double p,double h) {
|
||||
if(cache)for(size_t i=0;i<cache->count;i++) {
|
||||
NativePropertyState *s=&cache->states[i];
|
||||
if((s->valid&NATIVE_PROPERTY_H) && s->p==p && s->h==h && same_medium(&s->medium,m))return s->T;
|
||||
}
|
||||
double T=native_temperature_ph(m,p,h);
|
||||
if(cache && isfinite(h) && T>0 && isfinite(T) && p>0 && isfinite(p)) {
|
||||
NativePropertyState scratch,*s=property_pt(cache,m,p,T,&scratch);
|
||||
if((s->valid&NATIVE_PROPERTY_H) && s->h!=h)s=property_new(cache,m,p,T,&scratch);
|
||||
s->h=h;s->valid|=NATIVE_PROPERTY_H;
|
||||
}
|
||||
return T;
|
||||
}
|
||||
double native_density_context(NativePropertyCache *cache,const NativeMedium *m,double p,double T) {
|
||||
NativePropertyState scratch;return property_density(property_pt(cache,m,p,T,&scratch));
|
||||
}
|
||||
|
||||
static double cube_root(double x) { return x==0 ? 0 : copysign(pow(fabs(x),1.0/3.0),x); }
|
||||
static void attraction(double T, double *a, double *da, double *dda) {
|
||||
double tr=T/TC, sr=sqrt(tr), base=1+kappa*(1-sr);
|
||||
*a=pr_a*base*base;
|
||||
*da=pr_a*(-base*kappa/(TC*sr));
|
||||
*dda=pr_a*kappa/(2*TC*TC)*(kappa/tr+base/(tr*sr));
|
||||
}
|
||||
static double z_factor(double p,double T) {
|
||||
double a,da,dda; attraction(T,&a,&da,&dda);
|
||||
double A=a*p/(RU*RU*T*T), B=pr_b*p/(RU*T);
|
||||
double ca=-(1-B),cb=A-3*B*B-2*B,cc=-(A*B-B*B-B*B*B);
|
||||
double pp=cb-ca*ca/3,qq=2*ca*ca*ca/27-ca*cb/3+cc;
|
||||
double disc=pow(qq/2,2)+pow(pp/3,3),off=-ca/3,roots[3]; int n;
|
||||
if(disc>1e-14) { roots[0]=cube_root(-qq/2+sqrt(disc))+cube_root(-qq/2-sqrt(disc))+off;n=1; }
|
||||
else if(fabs(disc)<=1e-14) { double u=cube_root(-qq/2);roots[0]=2*u+off;roots[1]=-u+off;n=2; }
|
||||
else { if(pp>=0) return NAN; double radius=2*sqrt(-pp/3);
|
||||
double arg=(3*qq/(2*pp))*sqrt(-3/pp),theta=acos(fmax(-1,fmin(1,arg)))/3;
|
||||
for(int i=0;i<3;i++) { roots[i]=radius*cos(theta-2*PI*i/3)+off; }
|
||||
n=3;
|
||||
}
|
||||
double z=-INFINITY;for(int i=0;i<n;i++) if(roots[i]>B && isfinite(roots[i])) z=fmax(z,roots[i]);
|
||||
return isfinite(z)?z:NAN;
|
||||
}
|
||||
static double density(double p,double T) { return MOLAR_MASS/(z_factor(p,T)*RU*T/p); }
|
||||
static double log_volume(double rho) {
|
||||
double v=MOLAR_MASS/rho,sq=sqrt(2.0);
|
||||
return log((v+(1+sq)*pr_b)/(v+(1-sq)*pr_b))/(2*sq*pr_b*MOLAR_MASS);
|
||||
}
|
||||
static double u_departure(double T,double rho) {
|
||||
double a,da,dda;attraction(T,&a,&da,&dda);return (T*da-a)*log_volume(rho);
|
||||
}
|
||||
static double h_departure(double p,double T) {
|
||||
double a,da,dda;attraction(T,&a,&da,&dda);
|
||||
double z=z_factor(p,T),B=pr_b*p/(RU*T),sq=sqrt(2.0);
|
||||
double dep=RU*T*(z-1)+(T*da-a)*log((z+(1+sq)*B)/(z+(1-sq)*B))/(2*sq*pr_b);
|
||||
return dep/MOLAR_MASS;
|
||||
}
|
||||
static double u_ideal(double T) { return rg*(1.5*T-745.375); }
|
||||
static double h_ideal(double T) { return rg*(2.5*T-745.375); }
|
||||
static double temperature_u(double u) { return (u/rg+745.375)/1.5; }
|
||||
static double temperature_h(double h) { return (h/rg+745.375)/2.5; }
|
||||
static double pressure_rho(double T,double rho) {
|
||||
double v=MOLAR_MASS/rho,a,da,dda;attraction(T,&a,&da,&dda);
|
||||
if(v<=pr_b || T<=0) return NAN;
|
||||
return RU*T/(v-pr_b)-a/(v*(v+pr_b)+pr_b*(v-pr_b));
|
||||
}
|
||||
static NativeGas gas_properties(double m,double U,double V) {
|
||||
NativeGas g;g.rho=m/V;g.u=U/m;
|
||||
double T=fmax(temperature_u(g.u),2.2);
|
||||
for(int i=0;i<16;i++) { double next=fmax(temperature_u(g.u-u_departure(T,g.rho)),2.2);
|
||||
int done=fabs(next-T)<=1e-10*fmax(T,1);T=next;if(done) break; }
|
||||
g.T=T;g.p=pressure_rho(T,g.rho);
|
||||
/* Same energy reference as u_ideal/h_ideal. Avoid a redundant cubic solve
|
||||
in the single-root region; preserve vapor-root semantics below TC. */
|
||||
g.h=T>TC ? g.u+g.p/g.rho : h_ideal(T)+h_departure(g.p,T);return g;
|
||||
}
|
||||
static double temperature_ph(double p,double h) {
|
||||
double T=fmax(temperature_h(h),2.2);
|
||||
for(int i=0;i<16;i++) { double next=fmax(temperature_h(h-h_departure(p,T)),2.2);
|
||||
int done=fabs(next-T)<=1e-10*fmax(T,1);T=next;if(done) break; }return T;
|
||||
}
|
||||
static void local_isentropic(NativePropertyState *s) {
|
||||
if(s->valid&NATIVE_PROPERTY_ISENTROPIC)return;
|
||||
double p=s->p,T=s->T,rho=property_density(s),v=MOLAR_MASS/rho,a,da,dda;attraction(T,&a,&da,&dda);
|
||||
double d=v*(v+pr_b)+pr_b*(v-pr_b);
|
||||
double dpT=RU/(v-pr_b)-da/d;
|
||||
double dpR=(-RU*T/pow(v-pr_b,2)+a*2*(v+pr_b)/(d*d))*(-MOLAR_MASS/(rho*rho));
|
||||
double cv=1.5*rg+T*dda*log_volume(rho);
|
||||
double cp=cv+T*dpT*dpT/(rho*rho*dpR),gamma=cp/cv;
|
||||
s->isentropic_factor=p/(rho*dpR*gamma);s->isentropic_exponent=p*(gamma-1)/(gamma*T*dpT);
|
||||
if(isfinite(s->isentropic_factor) && isfinite(s->isentropic_exponent))s->valid|=NATIVE_PROPERTY_ISENTROPIC;
|
||||
}
|
||||
static double isentropic(NativePropertyCache *cache,NativePropertyState *up,double pd) {
|
||||
local_isentropic(up);if(pd>=up->p)return up->isentropic_factor;
|
||||
double Td=fmax(up->T*pow(fmax(pd/up->p,1e-12),up->isentropic_exponent),2.2);
|
||||
NativePropertyState scratch,*down=property_pt(cache,&up->medium,fmax(pd,1),Td,&scratch);
|
||||
local_isentropic(down);return .5*(up->isentropic_factor+down->isentropic_factor);
|
||||
}
|
||||
static double subsonic_cm(double r,double gamma,double rho,double T,double p) {
|
||||
return sqrt(fmax(2/(1-gamma)*rho*T/p*(pow(r,2*gamma)-pow(r,1+gamma)),0));
|
||||
}
|
||||
static void state_valve(NativePropertyCache *cache,NativePropertyState *up,double pd,double *cm,double *vel) {
|
||||
double p=up->p,T=up->T;pd=fmax(fmin(pd,p),0);
|
||||
const NativeMedium *m=&up->medium;
|
||||
double cp=m->cp+m->slope*(T-m->Tref);
|
||||
double factor=m->real_helium?isentropic(cache,up,pd):(cp-m->R)/cp;
|
||||
double g=fmax(1e-9,fmin(1-1e-9,factor)),rho=fmax(property_density(up),1e-12);
|
||||
double r=fmax(pd/p,0),critical=pow(2*g/(g+1),1/(1-g)),eff;
|
||||
if(r<=critical) { eff=critical;*cm=sqrt(2/(1+g)*rho*T/p)*pow(2*g/(g+1),g/(1-g));*vel=sqrt(2/(1+g)*p/rho); }
|
||||
else { eff=r;*cm=subsonic_cm(r,g,rho,T,p);*vel=sqrt(fmax(2/(1-g)*p/rho*(1-pow(r,1-g)),0)); }
|
||||
double ref=subsonic_cm(.9999,g,rho,T,p);
|
||||
if(*cm>0 && ref>0) { double smooth=tanh(fmax(12*fabs(*cm/ref)*log(eff)/log(.9999),0));*cm*=smooth;*vel*=smooth; }
|
||||
}
|
||||
static void medium_valve(NativePropertyCache *cache,const NativeMedium *m,double p,double pd,double T,double *cm,double *vel) {
|
||||
NativePropertyState scratch,*up=property_pt(cache,m,fmax(p,1),fmax(T,1),&scratch);
|
||||
state_valve(cache,up,pd,cm,vel);
|
||||
}
|
||||
|
||||
int native_gas_init(double p, double T, double volume, double *mU) {
|
||||
if (!(p > 0 && T >= 2.2 && volume > 0)) return 0;
|
||||
double rho = density(p, T);
|
||||
mU[0] = rho * volume;
|
||||
mU[1] = mU[0] * (u_ideal(T) + u_departure(T, rho));
|
||||
return isfinite(mU[0]) && isfinite(mU[1]) && mU[0] > 0;
|
||||
}
|
||||
|
||||
int native_gas(double m, double U, double volume, NativeGas *gas) {
|
||||
if (!(m > 0 && volume > 0) || !isfinite(U)) return 0;
|
||||
*gas = gas_properties(m, U, volume);
|
||||
return gas->p > 0 && isfinite(gas->p) && isfinite(gas->h);
|
||||
}
|
||||
int native_gas_context(NativePropertyCache *cache,double m,double U,double V,NativeGas *gas) {
|
||||
int ok=native_gas(m,U,V,gas);if(ok)remember_gas(cache,&helium_medium,gas);return ok;
|
||||
}
|
||||
|
||||
int native_orifice(double p1, double p2, double h1, double h2,
|
||||
double cq_area, double opening, double *flow,
|
||||
double *cm, double *velocity) {
|
||||
return native_orifice_context(NULL,p1,p2,h1,h2,cq_area,opening,flow,cm,velocity);
|
||||
}
|
||||
int native_orifice_context(NativePropertyCache *cache,double p1,double p2,double h1,double h2,
|
||||
double cq_area,double opening,double *flow,double *cm,double *velocity) {
|
||||
int forward = p1 >= p2;
|
||||
double p = forward ? p1 : p2, pd = forward ? p2 : p1;
|
||||
double T = native_temperature_ph_context(cache,&helium_medium,fmax(p,1),forward?h1:h2);
|
||||
medium_valve(cache,&helium_medium,p,pd,T,cm,velocity);
|
||||
double sign = forward ? 1 : -1;
|
||||
*velocity *= sign;
|
||||
*flow = opening == 0 || fabs(p1-p2) <= 1e-8 ? 0 :
|
||||
sign * cq_area * opening * fmax(p, 1) * *cm / sqrt(fmax(T, 1));
|
||||
if (opening == 0) *velocity = 0;
|
||||
return isfinite(*flow) && isfinite(*cm) && isfinite(*velocity);
|
||||
}
|
||||
|
||||
double native_contact(double penetration, double velocity, double stiffness,
|
||||
double damping, double pdis, int signed_force) {
|
||||
if (penetration <= 0) return 0;
|
||||
double fraction = pdis > 0 ? -expm1(-penetration / pdis) : 1;
|
||||
double force = stiffness * penetration + fraction * damping * velocity;
|
||||
return signed_force == 1 ? force : fmax(force, 0);
|
||||
}
|
||||
|
||||
void native_stop_motion(double x, double v, double lower, double upper,
|
||||
double *acceleration, double *velocity) {
|
||||
double vt = 1e-12 * fmax(fabs(v), 1);
|
||||
if ((x <= lower + 1e-12*fmax(fabs(lower),1) && v <= vt && *acceleration <= 0) ||
|
||||
(x >= upper - 1e-12*fmax(fabs(upper),1) && v >= -vt && *acceleration >= 0)) {
|
||||
*acceleration = 0;
|
||||
*velocity = 0;
|
||||
}
|
||||
}
|
||||
|
||||
double native_signal(double t, double start, int stages, int cyclic, const double *data) {
|
||||
double elapsed = fmax(t-start,0), duration = 0, offset = 0;
|
||||
for (int i=0;i<stages;i++) duration += data[16+i];
|
||||
if (cyclic && duration > 0) elapsed = fmod(elapsed,duration);
|
||||
for (int i=0;i<stages;i++) {
|
||||
double d = data[16+i];
|
||||
if (elapsed < offset+d || i == stages-1)
|
||||
return d <= 0 ? data[8+i] : data[i] + (elapsed-offset)/d*(data[8+i]-data[i]);
|
||||
offset += d;
|
||||
}
|
||||
return data[8+stages-1];
|
||||
}
|
||||
|
||||
double native_signal_break(double t, double end, double start, int stages,
|
||||
int cyclic, const double *data) {
|
||||
double duration=0, offset=0, result=end;
|
||||
for (int i=0;i<stages;i++) duration += data[16+i];
|
||||
for (int i=0;i<stages;i++) {
|
||||
double event=start+offset;
|
||||
if (cyclic && duration > 0 && event <= t) {
|
||||
double cycle=fmax(0,floor((t-event)/duration)+1);
|
||||
event += cycle*duration;
|
||||
if (event <= t) event += duration;
|
||||
}
|
||||
if (event > t) result=fmin(result,event);
|
||||
offset += data[16+i];
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
static double ideal_temperature(const NativeMedium *m, double energy, double c) {
|
||||
double delta = energy - c*m->Tref;
|
||||
if (fabs(m->slope) <= 1e-15) return m->Tref + delta/c;
|
||||
double root = sqrt(fmax(c*c + 2*m->slope*delta, 0));
|
||||
double a = (-c+root)/m->slope, b = (-c-root)/m->slope;
|
||||
return m->Tref + (fabs(a)<=fabs(b) ? a : b);
|
||||
}
|
||||
int native_medium_init(const NativeMedium *medium, double p, double T, double V,
|
||||
int legacy_ideal_initial, double *mU) {
|
||||
if (!(p>0 && T>0 && V>0)) return 0;
|
||||
if (medium->real_helium && !legacy_ideal_initial) return native_gas_init(p,T,V,mU);
|
||||
double mass=p*V/(medium->R*T),dt=T-medium->Tref;
|
||||
double u=medium->real_helium?u_ideal(T):(medium->cp-medium->R)*T+.5*medium->slope*dt*dt;
|
||||
mU[0]=mass;mU[1]=mass*u;
|
||||
return isfinite(mU[0]) && isfinite(mU[1]);
|
||||
}
|
||||
double native_density(const NativeMedium *m, double p, double T) {
|
||||
return m->real_helium ? density(p,T) : p/(m->R*T);
|
||||
}
|
||||
double native_temperature_ph(const NativeMedium *m, double p, double h) {
|
||||
return m->real_helium ? temperature_ph(p,h) : ideal_temperature(m,h,m->cp);
|
||||
}
|
||||
double native_viscosity(const NativeMedium *m, double T, int diagnostic) {
|
||||
/* Retain the ABI argument; flow and diagnostics use the same property. */
|
||||
(void)diagnostic;
|
||||
if (m->real_helium)
|
||||
return 1e-7*exp(.7501594*log(T)+35.76324/T-2212.129/(T*T)+.9212635);
|
||||
return m->mu*pow(T/m->muT,1.5)*(m->muT+m->S)/(T+m->S);
|
||||
}
|
||||
int native_medium_gas(const NativeMedium *medium, double m, double U, double V, NativeGas *g) {
|
||||
if (medium->real_helium) return native_gas(m,U,V,g);
|
||||
if (!(m>0 && V>0)) return 0;
|
||||
g->u=U/m; g->rho=m/V;
|
||||
g->T=ideal_temperature(medium,g->u,medium->cp-medium->R);
|
||||
g->p=g->rho*medium->R*g->T;
|
||||
double dt=g->T-medium->Tref;
|
||||
g->h=medium->cp*g->T+.5*medium->slope*dt*dt;
|
||||
return g->T>0 && isfinite(g->p) && isfinite(g->h);
|
||||
}
|
||||
int native_medium_gas_context(NativePropertyCache *cache,const NativeMedium *m,double mass,double U,double V,NativeGas *g) {
|
||||
int ok=native_medium_gas(m,mass,U,V,g);if(ok)remember_gas(cache,m,g);return ok;
|
||||
}
|
||||
int native_medium_orifice(const NativeMedium *m, double p1, double p2, double h1, double h2,
|
||||
double area, double opening, double *q, double *cm, double *v) {
|
||||
return native_medium_orifice_context(NULL,m,p1,p2,h1,h2,area,opening,q,cm,v);
|
||||
}
|
||||
int native_medium_orifice_context(NativePropertyCache *cache,const NativeMedium *m,double p1,double p2,double h1,double h2,
|
||||
double area,double opening,double *q,double *cm,double *v) {
|
||||
double p=fmax(p1,p2),pd=fmin(p1,p2),sign=p1>=p2?1:-1;
|
||||
double T=fmax(native_temperature_ph_context(cache,m,fmax(p,1),p1>=p2?h1:h2),1);
|
||||
medium_valve(cache,m,p,pd,T,cm,v);
|
||||
*q=fabs(p1-p2)<=1e-8?0:sign*area*opening*fmax(p,1)*(*cm)/sqrt(T);
|
||||
*v*=fabs(opening)<=1e-12?0:sign;
|
||||
return isfinite(*q) && isfinite(*cm) && isfinite(*v);
|
||||
}
|
||||
static double pipe_rough_limit(double rr) {
|
||||
return rr>0 ? 1/pow(-2*log10(rr/3.7),2) : 0;
|
||||
}
|
||||
static double pipe_friction_prepared(double re, double rr, double rough_limit) {
|
||||
if(re<=0) return 64000000;
|
||||
double lam=64/re;
|
||||
if(re<=89.96829989) return lam;
|
||||
double smooth=1/pow(-1.8*log10(6.9/re),2),turb=smooth;
|
||||
if(rr>0) { double r=re*rr,weight=r*r/(r*r+180*180);turb+=weight*(rough_limit-smooth); }
|
||||
double trans=pow((re-89.96829989)/2741.96700831,8.37293695);
|
||||
return lam+trans/(1+trans)*(turb-lam);
|
||||
}
|
||||
static double pipe_friction(double re, double rr) {
|
||||
return pipe_friction_prepared(re,rr,pipe_rough_limit(rr));
|
||||
}
|
||||
/* Derivative of the existing friction blend, local to this scalar solve. */
|
||||
static double pipe_friction_derivative(double re,double rr,double rough,double *df) {
|
||||
double lam=64/re,dl=-lam/re;
|
||||
if(re<=89.96829989){*df=dl;return lam;}
|
||||
double a=-1.8*log10(6.9/re),smooth=1/(a*a);
|
||||
double ds=-2*smooth/a*1.8/(log(10.0)*re),turb=smooth,dt=ds;
|
||||
if(rr>0){double r=re*rr,w=r*r/(r*r+180*180),dw=2*w*(1-w)/re;
|
||||
turb+=w*(rough-smooth);dt=ds*(1-w)+dw*(rough-smooth);}
|
||||
double z=pow((re-89.96829989)/2741.96700831,8.37293695),b=z/(1+z);
|
||||
double db=8.37293695*b*(1-b)/(re-89.96829989);
|
||||
*df=dl*(1-b)+b*dt+db*(turb-lam);return lam+b*(turb-lam);
|
||||
}
|
||||
static double pipe_checked_solution(double re,double f,double K,double rr,double rough,
|
||||
double flow_per_re,NativePipeSolve *status) {
|
||||
double target=sqrt(K/f),change=fabs(target-re);
|
||||
/* Separate the absolute and relative tests so a large flow scale cannot
|
||||
make both sides overflow and accidentally satisfy the stopping test. */
|
||||
if(!isfinite(target) || !isfinite(target*flow_per_re) ||
|
||||
!(change<=re*1e-10 || change*flow_per_re<=1e-13))return NAN;
|
||||
double value=target*target*pipe_friction_prepared(target,rr,rough);
|
||||
status->relative_residual=fabs(value/K-1);
|
||||
if(isfinite(value) && status->relative_residual<=1e-9) {
|
||||
status->converged=1;return target;
|
||||
}
|
||||
return NAN;
|
||||
}
|
||||
double native_pipe_resistance(double K,double rr,double flow_per_re,NativePipeSolve *status) {
|
||||
NativePipeSolve local={0,0,0,INFINITY};if(!status)status=&local;
|
||||
*status=local;
|
||||
if(!(K>=0 && rr>=0 && flow_per_re>0) || !isfinite(K) || !isfinite(rr) || !isfinite(flow_per_re))return NAN;
|
||||
if(K==0){status->converged=1;status->relative_residual=0;return 0;}
|
||||
if(K/64<=89.96829989) {
|
||||
double re=K/64;
|
||||
status->relative_residual=fabs(64*re/K-1);
|
||||
if(re>0 && isfinite(re*flow_per_re) && status->relative_residual<=1e-9) {
|
||||
status->converged=1;return re;
|
||||
}
|
||||
return NAN;
|
||||
}
|
||||
double rough=pipe_rough_limit(rr),lo=0,hi=fmax(sqrt(K/.02),1),df;
|
||||
if(!isfinite(rough) || !isfinite(hi))return NAN;
|
||||
int bracketed=0;
|
||||
for(int i=0;i<128;i++) {
|
||||
double value=hi*hi*pipe_friction_prepared(hi,rr,rough);
|
||||
if(!isfinite(value))return NAN;
|
||||
if(value>=K){bracketed=1;break;}
|
||||
lo=hi;hi*=2;
|
||||
if(!isfinite(hi))return NAN;
|
||||
}
|
||||
/* F(0)=-K; each positive lower endpoint was checked while expanding.
|
||||
Never start iteration with an unchecked or non-finite upper residual. */
|
||||
if(!bracketed)return NAN;
|
||||
double re=fmax(lo,fmin(sqrt(K/.02),hi)),previous_residual=INFINITY;
|
||||
int previous_newton=0;
|
||||
for(int i=0;i<128;i++) {
|
||||
status->iterations=i+1;
|
||||
double f=pipe_friction_derivative(re,rr,rough,&df),value=re*re*f,F=value-K;
|
||||
if(!(f>0) || !isfinite(f) || !isfinite(value))return NAN;
|
||||
status->relative_residual=fabs(value/K-1);
|
||||
double target=pipe_checked_solution(re,f,K,rr,rough,flow_per_re,status);
|
||||
if(status->converged)return target;
|
||||
if(F>0)hi=re;else lo=re;
|
||||
double slope=2*re*f+re*re*df;
|
||||
double next=slope>0 && isfinite(slope) ? re-F/slope : NAN;
|
||||
/* An in-bracket Newton step is useful only if it reduces |F|.
|
||||
After a step that fails to halve it, bisect the retained bracket;
|
||||
successive Newton steps therefore cannot stagnate near an endpoint. */
|
||||
int bisect=(previous_newton && fabs(F)>.5*previous_residual) ||
|
||||
!(slope>0) || !isfinite(slope) || !isfinite(next) || next<=lo || next>=hi;
|
||||
if(bisect) {
|
||||
next=lo+.5*(hi-lo);status->bisections++;
|
||||
if(next<=lo || next>=hi) {
|
||||
/* Adjacent floating-point endpoints: inspect both actual
|
||||
candidates, then fail if neither meets the original tests. */
|
||||
double endpoints[]={lo,hi};
|
||||
for(int j=0;j<2;j++) {
|
||||
double endpoint=endpoints[j];
|
||||
target=pipe_checked_solution(endpoint,pipe_friction_prepared(endpoint,rr,rough),
|
||||
K,rr,rough,flow_per_re,status);
|
||||
if(status->converged)return target;
|
||||
}
|
||||
return NAN;
|
||||
}
|
||||
}
|
||||
previous_newton=!bisect;previous_residual=fabs(F);
|
||||
re=next;
|
||||
}
|
||||
return NAN;
|
||||
}
|
||||
double native_pipe_flow(const NativeMedium *m, double p1, double p2, double T,
|
||||
double d, double length, double rr, int kind) {
|
||||
return native_pipe_flow_context(NULL,m,p1,p2,T,d,length,rr,kind);
|
||||
}
|
||||
double native_pipe_flow_context(NativePropertyCache *cache,const NativeMedium *m,double p1,double p2,double T,
|
||||
double d,double length,double rr,int kind) {
|
||||
if(fabs(p1-p2)<=1e-8) return 0;
|
||||
double p=fmax(fmax(p1,p2),1),pd=fmin(p1,p2),sign=p1>p2?1:-1;
|
||||
T=fmax(T,1);
|
||||
NativePropertyState scratch,*up=property_pt(cache,m,p,T,&scratch);
|
||||
double area=PI*d*d/4,mu=property_viscosity(up),den=PI*d*mu;
|
||||
/* PNL0001/2/3 share compressible flow and its near-equilibrium smoothing.
|
||||
PNL0003 differs in storage placement, not in the resistance law. */
|
||||
double cm,vel;state_valve(cache,up,pd,&cm,&vel);
|
||||
if(kind==0) {
|
||||
double lam=pow(area*p*cm,2)/(16*PI*mu*length*T);
|
||||
if(4*lam/den<=1000) return sign*lam;
|
||||
}
|
||||
double base=area*p*cm/sqrt(T),K=pow(4*base/den,2)*d/length;
|
||||
return sign*native_pipe_resistance(K,rr,den/4,NULL)*den/4;
|
||||
}
|
||||
double native_pipe_flow_cached(NativePipeCache *cache, const NativeMedium *m,
|
||||
double p1, double p2, double T, double d,
|
||||
double length, double rr, int kind) {
|
||||
return native_pipe_flow_cached_context(NULL,cache,m,p1,p2,T,d,length,rr,kind);
|
||||
}
|
||||
double native_pipe_flow_cached_context(NativePropertyCache *properties,NativePipeCache *cache,const NativeMedium *m,
|
||||
double p1,double p2,double T,double d,double length,double rr,int kind) {
|
||||
if(cache->valid && cache->p1==p1 && cache->p2==p2 && cache->T==T &&
|
||||
cache->diameter==d && cache->length==length && cache->roughness==rr && cache->kind==kind &&
|
||||
same_medium(&cache->medium,m))
|
||||
return cache->flow;
|
||||
double result=properties?native_pipe_flow_context(properties,m,p1,p2,T,d,length,rr,kind):
|
||||
native_pipe_flow(m,p1,p2,T,d,length,rr,kind);
|
||||
cache->valid=0;
|
||||
if(isfinite(result)) {
|
||||
cache->medium=*m;cache->p1=p1;cache->p2=p2;cache->T=T;
|
||||
cache->diameter=d;cache->length=length;cache->roughness=rr;cache->kind=kind;
|
||||
cache->flow=result;cache->valid=1;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
void native_pipe_diagnostics(const NativeMedium *m, double q, double p, double T,
|
||||
double d, double length, double rr, int diagnostic, double *r) {
|
||||
native_pipe_diagnostics_context(NULL,m,q,p,T,d,length,rr,diagnostic,r);
|
||||
}
|
||||
void native_pipe_diagnostics_context(NativePropertyCache *cache,const NativeMedium *m,double q,double p,double T,
|
||||
double d,double length,double rr,int diagnostic,double *r) {
|
||||
(void)diagnostic;
|
||||
NativePropertyState scratch,*state=property_pt(cache,m,p,T,&scratch);
|
||||
double area=PI*d*d/4,re=4*fabs(q)/(PI*d*property_viscosity(state)),ff=pipe_friction(re,rr);
|
||||
r[0]=re;
|
||||
r[1]=fabs(q)*sqrt(T)/fmax(sqrt(d/(length*ff))*area*p,1e-18);
|
||||
r[2]=q/(fmax(property_density(state),1e-12)*area);r[3]=ff;
|
||||
}
|
||||
double native_limit_force(double penetration, double velocity, double stiffness,
|
||||
double damping, double depth, int signed_force) {
|
||||
if(penetration<=0) return 0;
|
||||
double force=stiffness*penetration+(depth>0?fmin(penetration/depth,1):1)*damping*velocity;
|
||||
return signed_force==1?force:fmax(force,0);
|
||||
}
|
||||
#define NATIVE_COMPONENT_AMALGAMATION 1
|
||||
#include "../components/modules/properties.c"
|
||||
#include "../components/modules/orifice.c"
|
||||
#include "../components/modules/mechanics.c"
|
||||
#include "../components/modules/signal.c"
|
||||
#include "../components/modules/pipe.c"
|
||||
@@ -0,0 +1,27 @@
|
||||
#include "kernels.h"
|
||||
#include <math.h>
|
||||
|
||||
double native_contact(double penetration, double velocity, double stiffness,
|
||||
double damping, double pdis, int signed_force) {
|
||||
if (penetration <= 0) return 0;
|
||||
double fraction = pdis > 0 ? -expm1(-penetration / pdis) : 1;
|
||||
double force = stiffness * penetration + fraction * damping * velocity;
|
||||
return signed_force == 1 ? force : fmax(force, 0);
|
||||
}
|
||||
|
||||
void native_stop_motion(double x, double v, double lower, double upper,
|
||||
double *acceleration, double *velocity) {
|
||||
double vt = 1e-12 * fmax(fabs(v), 1);
|
||||
if ((x <= lower + 1e-12*fmax(fabs(lower),1) && v <= vt && *acceleration <= 0) ||
|
||||
(x >= upper - 1e-12*fmax(fabs(upper),1) && v >= -vt && *acceleration >= 0)) {
|
||||
*acceleration = 0;
|
||||
*velocity = 0;
|
||||
}
|
||||
}
|
||||
|
||||
double native_limit_force(double penetration, double velocity, double stiffness,
|
||||
double damping, double depth, int signed_force) {
|
||||
if(penetration<=0) return 0;
|
||||
double force=stiffness*penetration+(depth>0?fmin(penetration/depth,1):1)*damping*velocity;
|
||||
return signed_force==1?force:fmax(force,0);
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
#include "component_properties_internal.h"
|
||||
#include <math.h>
|
||||
|
||||
int native_orifice(double p1, double p2, double h1, double h2,
|
||||
double cq_area, double opening, double *flow,
|
||||
double *cm, double *velocity) {
|
||||
return native_orifice_context(NULL,p1,p2,h1,h2,cq_area,opening,flow,cm,velocity);
|
||||
}
|
||||
int native_orifice_context(NativePropertyCache *cache,double p1,double p2,double h1,double h2,
|
||||
double cq_area,double opening,double *flow,double *cm,double *velocity) {
|
||||
int forward = p1 >= p2;
|
||||
double p = forward ? p1 : p2, pd = forward ? p2 : p1;
|
||||
double T = native_temperature_ph_context(cache,&helium_medium,fmax(p,1),forward?h1:h2);
|
||||
medium_valve(cache,&helium_medium,p,pd,T,cm,velocity);
|
||||
double sign = forward ? 1 : -1;
|
||||
*velocity *= sign;
|
||||
*flow = opening == 0 || fabs(p1-p2) <= 1e-8 ? 0 :
|
||||
sign * cq_area * opening * fmax(p, 1) * *cm / sqrt(fmax(T, 1));
|
||||
if (opening == 0) *velocity = 0;
|
||||
return isfinite(*flow) && isfinite(*cm) && isfinite(*velocity);
|
||||
}
|
||||
|
||||
int native_medium_orifice(const NativeMedium *m, double p1, double p2, double h1, double h2,
|
||||
double area, double opening, double *q, double *cm, double *v) {
|
||||
return native_medium_orifice_context(NULL,m,p1,p2,h1,h2,area,opening,q,cm,v);
|
||||
}
|
||||
int native_medium_orifice_context(NativePropertyCache *cache,const NativeMedium *m,double p1,double p2,double h1,double h2,
|
||||
double area,double opening,double *q,double *cm,double *v) {
|
||||
double p=fmax(p1,p2),pd=fmin(p1,p2),sign=p1>=p2?1:-1;
|
||||
double T=fmax(native_temperature_ph_context(cache,m,fmax(p,1),p1>=p2?h1:h2),1);
|
||||
medium_valve(cache,m,p,pd,T,cm,v);
|
||||
*q=fabs(p1-p2)<=1e-8?0:sign*area*opening*fmax(p,1)*(*cm)/sqrt(T);
|
||||
*v*=fabs(opening)<=1e-12?0:sign;
|
||||
return isfinite(*q) && isfinite(*cm) && isfinite(*v);
|
||||
}
|
||||
@@ -0,0 +1,163 @@
|
||||
#include "component_properties_internal.h"
|
||||
#include "component_constants_internal.h"
|
||||
#include <math.h>
|
||||
|
||||
static double pipe_rough_limit(double rr) {
|
||||
return rr>0 ? 1/pow(-2*log10(rr/3.7),2) : 0;
|
||||
}
|
||||
static double pipe_friction_prepared(double re, double rr, double rough_limit) {
|
||||
if(re<=0) return 64000000;
|
||||
double lam=64/re;
|
||||
if(re<=89.96829989) return lam;
|
||||
double smooth=1/pow(-1.8*log10(6.9/re),2),turb=smooth;
|
||||
if(rr>0) { double r=re*rr,weight=r*r/(r*r+180*180);turb+=weight*(rough_limit-smooth); }
|
||||
double trans=pow((re-89.96829989)/2741.96700831,8.37293695);
|
||||
return lam+trans/(1+trans)*(turb-lam);
|
||||
}
|
||||
static double pipe_friction(double re, double rr) {
|
||||
return pipe_friction_prepared(re,rr,pipe_rough_limit(rr));
|
||||
}
|
||||
/* Derivative of the existing friction blend, local to this scalar solve. */
|
||||
static double pipe_friction_derivative(double re,double rr,double rough,double *df) {
|
||||
double lam=64/re,dl=-lam/re;
|
||||
if(re<=89.96829989){*df=dl;return lam;}
|
||||
double a=-1.8*log10(6.9/re),smooth=1/(a*a);
|
||||
double ds=-2*smooth/a*1.8/(log(10.0)*re),turb=smooth,dt=ds;
|
||||
if(rr>0){double r=re*rr,w=r*r/(r*r+180*180),dw=2*w*(1-w)/re;
|
||||
turb+=w*(rough-smooth);dt=ds*(1-w)+dw*(rough-smooth);}
|
||||
double z=pow((re-89.96829989)/2741.96700831,8.37293695),b=z/(1+z);
|
||||
double db=8.37293695*b*(1-b)/(re-89.96829989);
|
||||
*df=dl*(1-b)+b*dt+db*(turb-lam);return lam+b*(turb-lam);
|
||||
}
|
||||
static double pipe_checked_solution(double re,double f,double K,double rr,double rough,
|
||||
double flow_per_re,NativePipeSolve *status) {
|
||||
double target=sqrt(K/f),change=fabs(target-re);
|
||||
/* Separate the absolute and relative tests so a large flow scale cannot
|
||||
make both sides overflow and accidentally satisfy the stopping test. */
|
||||
if(!isfinite(target) || !isfinite(target*flow_per_re) ||
|
||||
!(change<=re*1e-10 || change*flow_per_re<=1e-13))return NAN;
|
||||
double value=target*target*pipe_friction_prepared(target,rr,rough);
|
||||
status->relative_residual=fabs(value/K-1);
|
||||
if(isfinite(value) && status->relative_residual<=1e-9) {
|
||||
status->converged=1;return target;
|
||||
}
|
||||
return NAN;
|
||||
}
|
||||
double native_pipe_resistance(double K,double rr,double flow_per_re,NativePipeSolve *status) {
|
||||
NativePipeSolve local={0,0,0,INFINITY};if(!status)status=&local;
|
||||
*status=local;
|
||||
if(!(K>=0 && rr>=0 && flow_per_re>0) || !isfinite(K) || !isfinite(rr) || !isfinite(flow_per_re))return NAN;
|
||||
if(K==0){status->converged=1;status->relative_residual=0;return 0;}
|
||||
if(K/64<=89.96829989) {
|
||||
double re=K/64;
|
||||
status->relative_residual=fabs(64*re/K-1);
|
||||
if(re>0 && isfinite(re*flow_per_re) && status->relative_residual<=1e-9) {
|
||||
status->converged=1;return re;
|
||||
}
|
||||
return NAN;
|
||||
}
|
||||
double rough=pipe_rough_limit(rr),lo=0,hi=fmax(sqrt(K/.02),1),df;
|
||||
if(!isfinite(rough) || !isfinite(hi))return NAN;
|
||||
int bracketed=0;
|
||||
for(int i=0;i<128;i++) {
|
||||
double value=hi*hi*pipe_friction_prepared(hi,rr,rough);
|
||||
if(!isfinite(value))return NAN;
|
||||
if(value>=K){bracketed=1;break;}
|
||||
lo=hi;hi*=2;
|
||||
if(!isfinite(hi))return NAN;
|
||||
}
|
||||
/* F(0)=-K; each positive lower endpoint was checked while expanding.
|
||||
Never start iteration with an unchecked or non-finite upper residual. */
|
||||
if(!bracketed)return NAN;
|
||||
double re=fmax(lo,fmin(sqrt(K/.02),hi)),previous_residual=INFINITY;
|
||||
int previous_newton=0;
|
||||
for(int i=0;i<128;i++) {
|
||||
status->iterations=i+1;
|
||||
double f=pipe_friction_derivative(re,rr,rough,&df),value=re*re*f,F=value-K;
|
||||
if(!(f>0) || !isfinite(f) || !isfinite(value))return NAN;
|
||||
status->relative_residual=fabs(value/K-1);
|
||||
double target=pipe_checked_solution(re,f,K,rr,rough,flow_per_re,status);
|
||||
if(status->converged)return target;
|
||||
if(F>0)hi=re;else lo=re;
|
||||
double slope=2*re*f+re*re*df;
|
||||
double next=slope>0 && isfinite(slope) ? re-F/slope : NAN;
|
||||
/* An in-bracket Newton step is useful only if it reduces |F|.
|
||||
After a step that fails to halve it, bisect the retained bracket;
|
||||
successive Newton steps therefore cannot stagnate near an endpoint. */
|
||||
int bisect=(previous_newton && fabs(F)>.5*previous_residual) ||
|
||||
!(slope>0) || !isfinite(slope) || !isfinite(next) || next<=lo || next>=hi;
|
||||
if(bisect) {
|
||||
next=lo+.5*(hi-lo);status->bisections++;
|
||||
if(next<=lo || next>=hi) {
|
||||
/* Adjacent floating-point endpoints: inspect both actual
|
||||
candidates, then fail if neither meets the original tests. */
|
||||
double endpoints[]={lo,hi};
|
||||
for(int j=0;j<2;j++) {
|
||||
double endpoint=endpoints[j];
|
||||
target=pipe_checked_solution(endpoint,pipe_friction_prepared(endpoint,rr,rough),
|
||||
K,rr,rough,flow_per_re,status);
|
||||
if(status->converged)return target;
|
||||
}
|
||||
return NAN;
|
||||
}
|
||||
}
|
||||
previous_newton=!bisect;previous_residual=fabs(F);
|
||||
re=next;
|
||||
}
|
||||
return NAN;
|
||||
}
|
||||
double native_pipe_flow(const NativeMedium *m, double p1, double p2, double T,
|
||||
double d, double length, double rr, int kind) {
|
||||
return native_pipe_flow_context(NULL,m,p1,p2,T,d,length,rr,kind);
|
||||
}
|
||||
double native_pipe_flow_context(NativePropertyCache *cache,const NativeMedium *m,double p1,double p2,double T,
|
||||
double d,double length,double rr,int kind) {
|
||||
if(fabs(p1-p2)<=1e-8) return 0;
|
||||
double p=fmax(fmax(p1,p2),1),pd=fmin(p1,p2),sign=p1>p2?1:-1;
|
||||
T=fmax(T,1);
|
||||
NativePropertyState scratch,*up=property_pt(cache,m,p,T,&scratch);
|
||||
double area=PI*d*d/4,mu=property_viscosity(up),den=PI*d*mu;
|
||||
/* PNL0001/2/3 share compressible flow and its near-equilibrium smoothing.
|
||||
PNL0003 differs in storage placement, not in the resistance law. */
|
||||
double cm,vel;state_valve(cache,up,pd,&cm,&vel);
|
||||
if(kind==0) {
|
||||
double lam=pow(area*p*cm,2)/(16*PI*mu*length*T);
|
||||
if(4*lam/den<=1000) return sign*lam;
|
||||
}
|
||||
double base=area*p*cm/sqrt(T),K=pow(4*base/den,2)*d/length;
|
||||
return sign*native_pipe_resistance(K,rr,den/4,NULL)*den/4;
|
||||
}
|
||||
double native_pipe_flow_cached(NativePipeCache *cache, const NativeMedium *m,
|
||||
double p1, double p2, double T, double d,
|
||||
double length, double rr, int kind) {
|
||||
return native_pipe_flow_cached_context(NULL,cache,m,p1,p2,T,d,length,rr,kind);
|
||||
}
|
||||
double native_pipe_flow_cached_context(NativePropertyCache *properties,NativePipeCache *cache,const NativeMedium *m,
|
||||
double p1,double p2,double T,double d,double length,double rr,int kind) {
|
||||
if(cache->valid && cache->p1==p1 && cache->p2==p2 && cache->T==T &&
|
||||
cache->diameter==d && cache->length==length && cache->roughness==rr && cache->kind==kind &&
|
||||
same_medium(&cache->medium,m))
|
||||
return cache->flow;
|
||||
double result=properties?native_pipe_flow_context(properties,m,p1,p2,T,d,length,rr,kind):
|
||||
native_pipe_flow(m,p1,p2,T,d,length,rr,kind);
|
||||
cache->valid=0;
|
||||
if(isfinite(result)) {
|
||||
cache->medium=*m;cache->p1=p1;cache->p2=p2;cache->T=T;
|
||||
cache->diameter=d;cache->length=length;cache->roughness=rr;cache->kind=kind;
|
||||
cache->flow=result;cache->valid=1;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
void native_pipe_diagnostics(const NativeMedium *m, double q, double p, double T,
|
||||
double d, double length, double rr, int diagnostic, double *r) {
|
||||
native_pipe_diagnostics_context(NULL,m,q,p,T,d,length,rr,diagnostic,r);
|
||||
}
|
||||
void native_pipe_diagnostics_context(NativePropertyCache *cache,const NativeMedium *m,double q,double p,double T,
|
||||
double d,double length,double rr,int diagnostic,double *r) {
|
||||
(void)diagnostic;
|
||||
NativePropertyState scratch,*state=property_pt(cache,m,p,T,&scratch);
|
||||
double area=PI*d*d/4,re=4*fabs(q)/(PI*d*property_viscosity(state)),ff=pipe_friction(re,rr);
|
||||
r[0]=re;
|
||||
r[1]=fabs(q)*sqrt(T)/fmax(sqrt(d/(length*ff))*area*p,1e-18);
|
||||
r[2]=q/(fmax(property_density(state),1e-12)*area);r[3]=ff;
|
||||
}
|
||||
@@ -0,0 +1,241 @@
|
||||
/* Helium Peng-Robinson and compressible-orifice kernels.
|
||||
* Ported from the project's Python physical equations; validated independently
|
||||
* against Python at ordinary, reverse-flow and contact trial states.
|
||||
*/
|
||||
#include "component_properties_internal.h"
|
||||
#include "component_constants_internal.h"
|
||||
#include <math.h>
|
||||
#include <stddef.h>
|
||||
|
||||
#define RU 8.31446261815324
|
||||
#define MOLAR_MASS 0.004002602
|
||||
#define TC 5.1953
|
||||
#define PC 227460.0
|
||||
#define OMEGA (-0.382)
|
||||
static const double pr_a=.457235583*RU*RU*TC*TC/PC;
|
||||
static const double pr_b=.07779607*RU*TC/PC;
|
||||
static const double kappa=.37464+1.54226*OMEGA-.26992*OMEGA*OMEGA;
|
||||
static const double rg=RU/MOLAR_MASS;
|
||||
NATIVE_COMPONENT_INTERNAL const NativeMedium helium_medium={1,RU/MOLAR_MASS,2.5*RU/MOLAR_MASS,293.15,0,1.96e-5,293.15,79.4};
|
||||
|
||||
NATIVE_COMPONENT_INTERNAL int same_medium(const NativeMedium *a,const NativeMedium *b) {
|
||||
return a->real_helium==b->real_helium && a->R==b->R && a->cp==b->cp &&
|
||||
a->Tref==b->Tref && a->slope==b->slope && a->mu==b->mu && a->muT==b->muT && a->S==b->S;
|
||||
}
|
||||
void native_properties_init(NativePropertyCache *cache,NativePropertyState *states,size_t capacity) {
|
||||
cache->states=states;cache->count=0;cache->capacity=states?capacity:0;
|
||||
}
|
||||
static NativePropertyState *property_new(NativePropertyCache *cache,const NativeMedium *m,
|
||||
double p,double T,NativePropertyState *scratch) {
|
||||
int valid=p>0 && T>0 && isfinite(p) && isfinite(T);
|
||||
NativePropertyState *s=valid && cache && cache->count<cache->capacity ? &cache->states[cache->count++] : scratch;
|
||||
*s=(NativePropertyState){0};s->medium=*m;s->p=p;s->T=T;s->valid=valid?NATIVE_PROPERTY_PT:0;
|
||||
return s;
|
||||
}
|
||||
NATIVE_COMPONENT_INTERNAL NativePropertyState *property_pt(NativePropertyCache *cache,const NativeMedium *m,
|
||||
double p,double T,NativePropertyState *scratch) {
|
||||
if(cache)for(size_t i=0;i<cache->count;i++) {
|
||||
NativePropertyState *s=&cache->states[i];
|
||||
if((s->valid&NATIVE_PROPERTY_PT) && s->p==p && s->T==T && same_medium(&s->medium,m))return s;
|
||||
}
|
||||
return property_new(cache,m,p,T,scratch);
|
||||
}
|
||||
NATIVE_COMPONENT_INTERNAL double property_density(NativePropertyState *s) {
|
||||
if(!(s->valid&NATIVE_PROPERTY_RHO)) {
|
||||
s->rho=native_density(&s->medium,s->p,s->T);
|
||||
if(s->rho>0 && isfinite(s->rho))s->valid|=NATIVE_PROPERTY_RHO;
|
||||
}
|
||||
return s->rho;
|
||||
}
|
||||
NATIVE_COMPONENT_INTERNAL double property_viscosity(NativePropertyState *s) {
|
||||
if(!(s->valid&NATIVE_PROPERTY_MU)) {
|
||||
s->mu=native_viscosity(&s->medium,s->T,0);
|
||||
if(s->mu>0 && isfinite(s->mu))s->valid|=NATIVE_PROPERTY_MU;
|
||||
}
|
||||
return s->mu;
|
||||
}
|
||||
static void remember_gas(NativePropertyCache *cache,const NativeMedium *m,const NativeGas *gas) {
|
||||
/* Below helium's critical temperature retain the existing vapor-root/PH
|
||||
selection. A future phase-aware medium contract can carry that state. */
|
||||
if(!cache || (m->real_helium && gas->T<=TC))return;
|
||||
NativePropertyState scratch,*s=property_pt(cache,m,gas->p,gas->T,&scratch);
|
||||
if(!(s->valid&NATIVE_PROPERTY_PT) || !isfinite(gas->h))return;
|
||||
if((s->valid&NATIVE_PROPERTY_H) && s->h!=gas->h)
|
||||
s=property_new(cache,m,gas->p,gas->T,&scratch);
|
||||
s->h=gas->h;s->valid|=NATIVE_PROPERTY_H;
|
||||
if(gas->rho>0 && isfinite(gas->rho)){s->rho=gas->rho;s->valid|=NATIVE_PROPERTY_RHO;}
|
||||
}
|
||||
double native_temperature_ph_context(NativePropertyCache *cache,const NativeMedium *m,double p,double h) {
|
||||
if(cache)for(size_t i=0;i<cache->count;i++) {
|
||||
NativePropertyState *s=&cache->states[i];
|
||||
if((s->valid&NATIVE_PROPERTY_H) && s->p==p && s->h==h && same_medium(&s->medium,m))return s->T;
|
||||
}
|
||||
double T=native_temperature_ph(m,p,h);
|
||||
if(cache && isfinite(h) && T>0 && isfinite(T) && p>0 && isfinite(p)) {
|
||||
NativePropertyState scratch,*s=property_pt(cache,m,p,T,&scratch);
|
||||
if((s->valid&NATIVE_PROPERTY_H) && s->h!=h)s=property_new(cache,m,p,T,&scratch);
|
||||
s->h=h;s->valid|=NATIVE_PROPERTY_H;
|
||||
}
|
||||
return T;
|
||||
}
|
||||
double native_density_context(NativePropertyCache *cache,const NativeMedium *m,double p,double T) {
|
||||
NativePropertyState scratch;return property_density(property_pt(cache,m,p,T,&scratch));
|
||||
}
|
||||
|
||||
static double cube_root(double x) { return x==0 ? 0 : copysign(pow(fabs(x),1.0/3.0),x); }
|
||||
static void attraction(double T, double *a, double *da, double *dda) {
|
||||
double tr=T/TC, sr=sqrt(tr), base=1+kappa*(1-sr);
|
||||
*a=pr_a*base*base;
|
||||
*da=pr_a*(-base*kappa/(TC*sr));
|
||||
*dda=pr_a*kappa/(2*TC*TC)*(kappa/tr+base/(tr*sr));
|
||||
}
|
||||
static double z_factor(double p,double T) {
|
||||
double a,da,dda; attraction(T,&a,&da,&dda);
|
||||
double A=a*p/(RU*RU*T*T), B=pr_b*p/(RU*T);
|
||||
double ca=-(1-B),cb=A-3*B*B-2*B,cc=-(A*B-B*B-B*B*B);
|
||||
double pp=cb-ca*ca/3,qq=2*ca*ca*ca/27-ca*cb/3+cc;
|
||||
double disc=pow(qq/2,2)+pow(pp/3,3),off=-ca/3,roots[3]; int n;
|
||||
if(disc>1e-14) { roots[0]=cube_root(-qq/2+sqrt(disc))+cube_root(-qq/2-sqrt(disc))+off;n=1; }
|
||||
else if(fabs(disc)<=1e-14) { double u=cube_root(-qq/2);roots[0]=2*u+off;roots[1]=-u+off;n=2; }
|
||||
else { if(pp>=0) return NAN; double radius=2*sqrt(-pp/3);
|
||||
double arg=(3*qq/(2*pp))*sqrt(-3/pp),theta=acos(fmax(-1,fmin(1,arg)))/3;
|
||||
for(int i=0;i<3;i++) { roots[i]=radius*cos(theta-2*PI*i/3)+off; }
|
||||
n=3;
|
||||
}
|
||||
double z=-INFINITY;for(int i=0;i<n;i++) if(roots[i]>B && isfinite(roots[i])) z=fmax(z,roots[i]);
|
||||
return isfinite(z)?z:NAN;
|
||||
}
|
||||
static double density(double p,double T) { return MOLAR_MASS/(z_factor(p,T)*RU*T/p); }
|
||||
static double log_volume(double rho) {
|
||||
double v=MOLAR_MASS/rho,sq=sqrt(2.0);
|
||||
return log((v+(1+sq)*pr_b)/(v+(1-sq)*pr_b))/(2*sq*pr_b*MOLAR_MASS);
|
||||
}
|
||||
static double u_departure(double T,double rho) {
|
||||
double a,da,dda;attraction(T,&a,&da,&dda);return (T*da-a)*log_volume(rho);
|
||||
}
|
||||
static double h_departure(double p,double T) {
|
||||
double a,da,dda;attraction(T,&a,&da,&dda);
|
||||
double z=z_factor(p,T),B=pr_b*p/(RU*T),sq=sqrt(2.0);
|
||||
double dep=RU*T*(z-1)+(T*da-a)*log((z+(1+sq)*B)/(z+(1-sq)*B))/(2*sq*pr_b);
|
||||
return dep/MOLAR_MASS;
|
||||
}
|
||||
static double u_ideal(double T) { return rg*(1.5*T-745.375); }
|
||||
static double h_ideal(double T) { return rg*(2.5*T-745.375); }
|
||||
static double temperature_u(double u) { return (u/rg+745.375)/1.5; }
|
||||
static double temperature_h(double h) { return (h/rg+745.375)/2.5; }
|
||||
static double pressure_rho(double T,double rho) {
|
||||
double v=MOLAR_MASS/rho,a,da,dda;attraction(T,&a,&da,&dda);
|
||||
if(v<=pr_b || T<=0) return NAN;
|
||||
return RU*T/(v-pr_b)-a/(v*(v+pr_b)+pr_b*(v-pr_b));
|
||||
}
|
||||
static NativeGas gas_properties(double m,double U,double V) {
|
||||
NativeGas g;g.rho=m/V;g.u=U/m;
|
||||
double T=fmax(temperature_u(g.u),2.2);
|
||||
for(int i=0;i<16;i++) { double next=fmax(temperature_u(g.u-u_departure(T,g.rho)),2.2);
|
||||
int done=fabs(next-T)<=1e-10*fmax(T,1);T=next;if(done) break; }
|
||||
g.T=T;g.p=pressure_rho(T,g.rho);
|
||||
/* Same energy reference as u_ideal/h_ideal. Avoid a redundant cubic solve
|
||||
in the single-root region; preserve vapor-root semantics below TC. */
|
||||
g.h=T>TC ? g.u+g.p/g.rho : h_ideal(T)+h_departure(g.p,T);return g;
|
||||
}
|
||||
static double temperature_ph(double p,double h) {
|
||||
double T=fmax(temperature_h(h),2.2);
|
||||
for(int i=0;i<16;i++) { double next=fmax(temperature_h(h-h_departure(p,T)),2.2);
|
||||
int done=fabs(next-T)<=1e-10*fmax(T,1);T=next;if(done) break; }return T;
|
||||
}
|
||||
static void local_isentropic(NativePropertyState *s) {
|
||||
if(s->valid&NATIVE_PROPERTY_ISENTROPIC)return;
|
||||
double p=s->p,T=s->T,rho=property_density(s),v=MOLAR_MASS/rho,a,da,dda;attraction(T,&a,&da,&dda);
|
||||
double d=v*(v+pr_b)+pr_b*(v-pr_b);
|
||||
double dpT=RU/(v-pr_b)-da/d;
|
||||
double dpR=(-RU*T/pow(v-pr_b,2)+a*2*(v+pr_b)/(d*d))*(-MOLAR_MASS/(rho*rho));
|
||||
double cv=1.5*rg+T*dda*log_volume(rho);
|
||||
double cp=cv+T*dpT*dpT/(rho*rho*dpR),gamma=cp/cv;
|
||||
s->isentropic_factor=p/(rho*dpR*gamma);s->isentropic_exponent=p*(gamma-1)/(gamma*T*dpT);
|
||||
if(isfinite(s->isentropic_factor) && isfinite(s->isentropic_exponent))s->valid|=NATIVE_PROPERTY_ISENTROPIC;
|
||||
}
|
||||
static double isentropic(NativePropertyCache *cache,NativePropertyState *up,double pd) {
|
||||
local_isentropic(up);if(pd>=up->p)return up->isentropic_factor;
|
||||
double Td=fmax(up->T*pow(fmax(pd/up->p,1e-12),up->isentropic_exponent),2.2);
|
||||
NativePropertyState scratch,*down=property_pt(cache,&up->medium,fmax(pd,1),Td,&scratch);
|
||||
local_isentropic(down);return .5*(up->isentropic_factor+down->isentropic_factor);
|
||||
}
|
||||
static double subsonic_cm(double r,double gamma,double rho,double T,double p) {
|
||||
return sqrt(fmax(2/(1-gamma)*rho*T/p*(pow(r,2*gamma)-pow(r,1+gamma)),0));
|
||||
}
|
||||
NATIVE_COMPONENT_INTERNAL void state_valve(NativePropertyCache *cache,NativePropertyState *up,double pd,double *cm,double *vel) {
|
||||
double p=up->p,T=up->T;pd=fmax(fmin(pd,p),0);
|
||||
const NativeMedium *m=&up->medium;
|
||||
double cp=m->cp+m->slope*(T-m->Tref);
|
||||
double factor=m->real_helium?isentropic(cache,up,pd):(cp-m->R)/cp;
|
||||
double g=fmax(1e-9,fmin(1-1e-9,factor)),rho=fmax(property_density(up),1e-12);
|
||||
double r=fmax(pd/p,0),critical=pow(2*g/(g+1),1/(1-g)),eff;
|
||||
if(r<=critical) { eff=critical;*cm=sqrt(2/(1+g)*rho*T/p)*pow(2*g/(g+1),g/(1-g));*vel=sqrt(2/(1+g)*p/rho); }
|
||||
else { eff=r;*cm=subsonic_cm(r,g,rho,T,p);*vel=sqrt(fmax(2/(1-g)*p/rho*(1-pow(r,1-g)),0)); }
|
||||
double ref=subsonic_cm(.9999,g,rho,T,p);
|
||||
if(*cm>0 && ref>0) { double smooth=tanh(fmax(12*fabs(*cm/ref)*log(eff)/log(.9999),0));*cm*=smooth;*vel*=smooth; }
|
||||
}
|
||||
NATIVE_COMPONENT_INTERNAL void medium_valve(NativePropertyCache *cache,const NativeMedium *m,double p,double pd,double T,double *cm,double *vel) {
|
||||
NativePropertyState scratch,*up=property_pt(cache,m,fmax(p,1),fmax(T,1),&scratch);
|
||||
state_valve(cache,up,pd,cm,vel);
|
||||
}
|
||||
|
||||
int native_gas_init(double p, double T, double volume, double *mU) {
|
||||
if (!(p > 0 && T >= 2.2 && volume > 0)) return 0;
|
||||
double rho = density(p, T);
|
||||
mU[0] = rho * volume;
|
||||
mU[1] = mU[0] * (u_ideal(T) + u_departure(T, rho));
|
||||
return isfinite(mU[0]) && isfinite(mU[1]) && mU[0] > 0;
|
||||
}
|
||||
|
||||
int native_gas(double m, double U, double volume, NativeGas *gas) {
|
||||
if (!(m > 0 && volume > 0) || !isfinite(U)) return 0;
|
||||
*gas = gas_properties(m, U, volume);
|
||||
return gas->p > 0 && isfinite(gas->p) && isfinite(gas->h);
|
||||
}
|
||||
int native_gas_context(NativePropertyCache *cache,double m,double U,double V,NativeGas *gas) {
|
||||
int ok=native_gas(m,U,V,gas);if(ok)remember_gas(cache,&helium_medium,gas);return ok;
|
||||
}
|
||||
|
||||
static double ideal_temperature(const NativeMedium *m, double energy, double c) {
|
||||
double delta = energy - c*m->Tref;
|
||||
if (fabs(m->slope) <= 1e-15) return m->Tref + delta/c;
|
||||
double root = sqrt(fmax(c*c + 2*m->slope*delta, 0));
|
||||
double a = (-c+root)/m->slope, b = (-c-root)/m->slope;
|
||||
return m->Tref + (fabs(a)<=fabs(b) ? a : b);
|
||||
}
|
||||
int native_medium_init(const NativeMedium *medium, double p, double T, double V,
|
||||
int legacy_ideal_initial, double *mU) {
|
||||
if (!(p>0 && T>0 && V>0)) return 0;
|
||||
if (medium->real_helium && !legacy_ideal_initial) return native_gas_init(p,T,V,mU);
|
||||
double mass=p*V/(medium->R*T),dt=T-medium->Tref;
|
||||
double u=medium->real_helium?u_ideal(T):(medium->cp-medium->R)*T+.5*medium->slope*dt*dt;
|
||||
mU[0]=mass;mU[1]=mass*u;
|
||||
return isfinite(mU[0]) && isfinite(mU[1]);
|
||||
}
|
||||
double native_density(const NativeMedium *m, double p, double T) {
|
||||
return m->real_helium ? density(p,T) : p/(m->R*T);
|
||||
}
|
||||
double native_temperature_ph(const NativeMedium *m, double p, double h) {
|
||||
return m->real_helium ? temperature_ph(p,h) : ideal_temperature(m,h,m->cp);
|
||||
}
|
||||
double native_viscosity(const NativeMedium *m, double T, int diagnostic) {
|
||||
/* Retain the ABI argument; flow and diagnostics use the same property. */
|
||||
(void)diagnostic;
|
||||
if (m->real_helium)
|
||||
return 1e-7*exp(.7501594*log(T)+35.76324/T-2212.129/(T*T)+.9212635);
|
||||
return m->mu*pow(T/m->muT,1.5)*(m->muT+m->S)/(T+m->S);
|
||||
}
|
||||
int native_medium_gas(const NativeMedium *medium, double m, double U, double V, NativeGas *g) {
|
||||
if (medium->real_helium) return native_gas(m,U,V,g);
|
||||
if (!(m>0 && V>0)) return 0;
|
||||
g->u=U/m; g->rho=m/V;
|
||||
g->T=ideal_temperature(medium,g->u,medium->cp-medium->R);
|
||||
g->p=g->rho*medium->R*g->T;
|
||||
double dt=g->T-medium->Tref;
|
||||
g->h=medium->cp*g->T+.5*medium->slope*dt*dt;
|
||||
return g->T>0 && isfinite(g->p) && isfinite(g->h);
|
||||
}
|
||||
int native_medium_gas_context(NativePropertyCache *cache,const NativeMedium *m,double mass,double U,double V,NativeGas *g) {
|
||||
int ok=native_medium_gas(m,mass,U,V,g);if(ok)remember_gas(cache,m,g);return ok;
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
#include "kernels.h"
|
||||
#include <math.h>
|
||||
|
||||
double native_signal(double t, double start, int stages, int cyclic, const double *data) {
|
||||
double elapsed = fmax(t-start,0), duration = 0, offset = 0;
|
||||
for (int i=0;i<stages;i++) duration += data[16+i];
|
||||
if (cyclic && duration > 0) elapsed = fmod(elapsed,duration);
|
||||
for (int i=0;i<stages;i++) {
|
||||
double d = data[16+i];
|
||||
if (elapsed < offset+d || i == stages-1)
|
||||
return d <= 0 ? data[8+i] : data[i] + (elapsed-offset)/d*(data[8+i]-data[i]);
|
||||
offset += d;
|
||||
}
|
||||
return data[8+stages-1];
|
||||
}
|
||||
|
||||
double native_signal_break(double t, double end, double start, int stages,
|
||||
int cyclic, const double *data) {
|
||||
double duration=0, offset=0, result=end;
|
||||
for (int i=0;i<stages;i++) duration += data[16+i];
|
||||
for (int i=0;i<stages;i++) {
|
||||
double event=start+offset;
|
||||
if (cyclic && duration > 0 && event <= t) {
|
||||
double cycle=fmax(0,floor((t-event)/duration)+1);
|
||||
event += cycle*duration;
|
||||
if (event <= t) event += duration;
|
||||
}
|
||||
if (event > t) result=fmin(result,event);
|
||||
offset += data[16+i];
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,10 @@
|
||||
#ifndef NATIVE_COMPONENT_CONSTANTS_INTERNAL_H
|
||||
#define NATIVE_COMPONENT_CONSTANTS_INTERNAL_H
|
||||
|
||||
/* Include only in modules using PI, so each standalone translation unit can
|
||||
* fold the unchanged double constant without an external data dependency.
|
||||
* The include guard also keeps the diagnostic amalgamation valid.
|
||||
*/
|
||||
static const double PI=3.1415926535897932384626433832795;
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,31 @@
|
||||
#ifndef NATIVE_COMPONENT_PROPERTIES_INTERNAL_H
|
||||
#define NATIVE_COMPONENT_PROPERTIES_INTERNAL_H
|
||||
|
||||
#include "kernels.h"
|
||||
|
||||
/* Private interfaces shared by the flow modules. Gas/EOS formulas, property
|
||||
* reuse and isentropic calculations stay together in properties.c. This
|
||||
* switch is only for the diagnostic amalgamation in components/kernels.c;
|
||||
* separately compiled production modules share ordinary external symbols.
|
||||
*/
|
||||
#ifdef NATIVE_COMPONENT_AMALGAMATION
|
||||
#define NATIVE_COMPONENT_INTERNAL static
|
||||
static const NativeMedium helium_medium;
|
||||
#else
|
||||
#define NATIVE_COMPONENT_INTERNAL
|
||||
extern const NativeMedium helium_medium;
|
||||
#endif
|
||||
|
||||
NATIVE_COMPONENT_INTERNAL int same_medium(const NativeMedium *, const NativeMedium *);
|
||||
NATIVE_COMPONENT_INTERNAL NativePropertyState *property_pt(
|
||||
NativePropertyCache *, const NativeMedium *, double p, double T,
|
||||
NativePropertyState *scratch);
|
||||
NATIVE_COMPONENT_INTERNAL double property_density(NativePropertyState *);
|
||||
NATIVE_COMPONENT_INTERNAL double property_viscosity(NativePropertyState *);
|
||||
NATIVE_COMPONENT_INTERNAL void state_valve(
|
||||
NativePropertyCache *, NativePropertyState *, double pd, double *cm, double *velocity);
|
||||
NATIVE_COMPONENT_INTERNAL void medium_valve(
|
||||
NativePropertyCache *, const NativeMedium *, double p, double pd, double T,
|
||||
double *cm, double *velocity);
|
||||
|
||||
#endif
|
||||
@@ -291,7 +291,11 @@ def main():
|
||||
'mode':'plain' if args.plain else 'profile','port':args.port,
|
||||
'python':sys.version,'platform':sys.platform,
|
||||
'frontendFiles':{str(p.relative_to(out/'frontend')):sha256(p.read_bytes()).hexdigest() for p in (out/'frontend').rglob('*') if p.is_file()},
|
||||
'productionKernelSha256':sha256((ROOT/'native/components/kernels.c').read_bytes()).hexdigest()}
|
||||
# Retain the historical entry hash; module/header hashes now
|
||||
# identify the numerical sources behind that diagnostic entry.
|
||||
'productionKernelSha256':sha256((ROOT/'native/components/kernels.c').read_bytes()).hexdigest(),
|
||||
'productionNativeSourceHashes':{p.relative_to(ROOT/'native').as_posix():sha256(p.read_bytes()).hexdigest()
|
||||
for p in sorted((ROOT/'native').rglob('*')) if p.is_file() and p.suffix in ('.c','.h')}}
|
||||
(out/'environment.json').write_text(json.dumps(metadata,indent=2)+'\n')
|
||||
import uvicorn
|
||||
uvicorn.run(profile.app(api.app) if profile else api.app,host='127.0.0.1',port=args.port)
|
||||
|
||||
@@ -0,0 +1,382 @@
|
||||
"""Prepare or serially measure frozen/current native builds; never run a model.
|
||||
|
||||
Example (fresh output directory; omit --run for preparation only)::
|
||||
|
||||
.venv/bin/python tests/manual/benchmark_native_build_cache.py \
|
||||
--baseline-root test/incremental-build-20260912/baseline-source \
|
||||
--output-dir test/incremental-build-20260912/benchmark \
|
||||
--warmups 1 --repeats 3 --run
|
||||
|
||||
Each version/round gets an empty application cache, followed by A/full hit,
|
||||
A-prime/changed pipe length, A/full hit, four branches/first build, A/full hit.
|
||||
Application-cache cold does not mean OS page-cache cold. Versions run in separate
|
||||
Python processes, with alternating version order by round. Preparation, imports,
|
||||
hashing and recording are outside build timing. NativeBuild.close() is measured
|
||||
separately and as part of an adjacent build-and-release wall interval. Compilation
|
||||
retains each production builder's concurrency and flags. No solver, frontend,
|
||||
strace or previous diagnostic timings are invoked/combined.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from copy import deepcopy
|
||||
import csv
|
||||
from datetime import datetime, timezone
|
||||
from hashlib import sha256
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
from pathlib import Path
|
||||
import platform
|
||||
import shutil
|
||||
import statistics
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
SCRIPT = Path(__file__).resolve()
|
||||
SCENARIOS = (
|
||||
("cold", "a", False), ("same_model", "a", True),
|
||||
("parameter_change", "a_prime", False), ("return_to_a", "a", True),
|
||||
("four_branches", "four", False), ("return_from_four", "a", True),
|
||||
)
|
||||
METRICS = ("buildSeconds", "buildCallWallSeconds", "releaseWallSeconds",
|
||||
"buildAndReleaseWallSeconds", "executableBytes", "cacheBytes",
|
||||
"preprocessSeconds", "compileSeconds", "compileWallSeconds",
|
||||
"linkSeconds", "unitCount", "objectCacheHits", "objectCompilations")
|
||||
ENVIRONMENT = ("SIMULATION_NATIVE_CC", "SUNDIALS_ROOT",
|
||||
"SIMULATION_NATIVE_MODEL_CACHE_MB", "SIMULATION_NATIVE_OBJECT_CACHE_MB")
|
||||
|
||||
|
||||
def digest(path: Path) -> str:
|
||||
with path.open("rb") as stream:
|
||||
value = sha256()
|
||||
for block in iter(lambda: stream.read(1024 * 1024), b""):
|
||||
value.update(block)
|
||||
return value.hexdigest()
|
||||
|
||||
|
||||
def json_digest(value) -> str:
|
||||
return sha256(json.dumps(value, sort_keys=True, separators=(",", ":")).encode()).hexdigest()
|
||||
|
||||
|
||||
def write_json(path: Path, value) -> None:
|
||||
path.write_text(json.dumps(value, ensure_ascii=False, indent=2, allow_nan=False) + "\n", encoding="utf-8")
|
||||
|
||||
|
||||
def source_files(root: Path) -> dict:
|
||||
paths = list((root / "app").rglob("*.py"))
|
||||
paths += [p for directory in ("native", "schemas") for p in (root / directory).rglob("*") if p.is_file()]
|
||||
return {p.relative_to(root).as_posix(): digest(p) for p in sorted(paths)
|
||||
if "__pycache__" not in p.parts and not p.relative_to(root).as_posix().startswith("app/data/")}
|
||||
|
||||
|
||||
def source_record(root: Path) -> dict:
|
||||
if not (root / "app/simulation/native_codegen/build.py").is_file() or not (root / "native").is_dir():
|
||||
raise ValueError(f"Incomplete source root: {root}")
|
||||
frozen = root / "source-manifest.json"
|
||||
if frozen.exists():
|
||||
manifest = json.loads(frozen.read_text())
|
||||
for name, expected in manifest["trackedFiles"].items():
|
||||
if digest(root / name) != expected:
|
||||
raise ValueError(f"Frozen source integrity failure: {root / name}")
|
||||
revision = {"gitHead": manifest["head"], "frozenManifestSha256": digest(frozen), "worktreeStatus": None}
|
||||
else:
|
||||
revision = {"gitHead": subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=root, text=True).strip(),
|
||||
"worktreeStatus": subprocess.check_output(["git", "status", "--porcelain", "--", "app", "native", "schemas"], cwd=root, text=True)}
|
||||
files = source_files(root)
|
||||
return {"root": str(root), **revision, "sourceFiles": files, "sourceFilesSha256": json_digest(files)}
|
||||
|
||||
|
||||
def assert_source(record: dict) -> None:
|
||||
if source_files(Path(record["root"])) != record["sourceFiles"]:
|
||||
raise RuntimeError(f"Source changed after preparation: {record['root']}")
|
||||
|
||||
|
||||
def make_inputs(args, output: Path) -> tuple[dict, dict]:
|
||||
directory = output / "inputs"
|
||||
directory.mkdir()
|
||||
paths = {"a": directory / "model-a.json", "a_prime": directory / "model-a-prime.json", "four": directory / "model-four.json"}
|
||||
shutil.copyfile(args.input, paths["a"])
|
||||
shutil.copyfile(args.four_input, paths["four"])
|
||||
original = json.loads(paths["a"].read_text(encoding="utf-8"))
|
||||
variant = deepcopy(original)
|
||||
matches = [(i, node) for i, node in enumerate(variant["nodes"]) if node["id"] == args.variant_node]
|
||||
if len(matches) != 1:
|
||||
raise ValueError(f"Expected exactly one variant node: {args.variant_node}")
|
||||
index, node = matches[0]
|
||||
data = node["data"]
|
||||
if data["modelType"] != "amesim_pnl0001":
|
||||
raise ValueError("The controlled length variant requires an amesim_pnl0001 pipe")
|
||||
old = data["parameters"]["le"]
|
||||
if type(old) not in (int, float) or not math.isfinite(old) or old <= 0:
|
||||
raise ValueError("Pipe le must be a positive numeric SI value")
|
||||
new = old * 1.001
|
||||
if not math.isfinite(new) or new == old:
|
||||
raise ValueError("Pipe length variant did not produce a finite changed value")
|
||||
data["parameters"]["le"] = new
|
||||
write_json(paths["a_prime"], variant)
|
||||
data["parameters"]["le"] = old
|
||||
if variant != original:
|
||||
raise RuntimeError("Variant changed fields beyond the selected pipe length")
|
||||
inputs = {key: {"path": str(path), "sha256": digest(path)} for key, path in paths.items()}
|
||||
change = {"nodeId": node["id"], "modelType": data["modelType"], "jsonPointer": f"/nodes/{index}/data/parameters/le",
|
||||
"before": old, "after": new, "storedUnit": "m", "factor": 1.001,
|
||||
"note": "Stored numeric values are SI; editor parameterUnits do not rescale them."}
|
||||
return inputs, change
|
||||
|
||||
|
||||
def generate(config: dict, version: str, *, save: bool):
|
||||
source = config["versions"][version]
|
||||
assert_source(source)
|
||||
root = Path(source["root"])
|
||||
# Every worker starts a fresh interpreter; neither version imports the other.
|
||||
sys.path.insert(0, str(root))
|
||||
from app.main import compile_system_xml_network
|
||||
from app.simulation.native_codegen.compiler import compile_native_program
|
||||
from app.simulation.native_codegen.input import load_input
|
||||
from app.simulation.native_codegen import build as builder
|
||||
for name, module in tuple(sys.modules.items()):
|
||||
location = getattr(module, "__file__", None)
|
||||
if (name == "app" or name.startswith("app.")) and location and not Path(location).resolve().is_relative_to(root):
|
||||
raise RuntimeError(f"Mixed source imports: {name}: {location}")
|
||||
if builder.NATIVE.resolve() != root / "native":
|
||||
raise RuntimeError("Builder native source root differs from the selected version")
|
||||
generated = Path(config["outputDir"]) / version / "generated"
|
||||
if save:
|
||||
generated.mkdir(parents=True)
|
||||
programs, records = {}, {}
|
||||
for key, item in config["inputs"].items():
|
||||
path = Path(item["path"])
|
||||
if digest(path) != item["sha256"]:
|
||||
raise RuntimeError(f"Input changed after preparation: {path}")
|
||||
started = time.perf_counter()
|
||||
xml, document = load_input(path)
|
||||
program = compile_native_program(compile_system_xml_network(document))
|
||||
preparation = time.perf_counter() - started
|
||||
contract = program.manifest()
|
||||
record = {"inputSha256": item["sha256"], "xmlSha256": sha256(xml).hexdigest(),
|
||||
"sourceSha256": sha256(program.source.encode()).hexdigest(), "headerSha256": sha256(program.header.encode()).hexdigest(),
|
||||
"contractSha256": json_digest(contract), "stateCount": len(program.state_keys),
|
||||
"variableCount": len(program.variables)}
|
||||
if save:
|
||||
directory = generated / key
|
||||
directory.mkdir()
|
||||
(directory / "input.xml").write_bytes(xml)
|
||||
(directory / "model.c").write_text(program.source, encoding="utf-8")
|
||||
(directory / "model.h").write_text(program.header, encoding="utf-8")
|
||||
write_json(directory / "contract.json", contract)
|
||||
write_json(directory / "preparation.json", {**record, "generationSeconds": preparation})
|
||||
programs[key], records[key] = program, record
|
||||
if programs["a"].header != programs["a_prime"].header or programs["a"].state_keys != programs["a_prime"].state_keys:
|
||||
raise RuntimeError("Length variant changed model header/state layout")
|
||||
if programs["a"].source == programs["a_prime"].source:
|
||||
raise RuntimeError("Length variant did not change generated C")
|
||||
if not save and records != config["generated"][version]:
|
||||
raise RuntimeError("Regenerated model differs from prepared artifacts")
|
||||
assert_source(source)
|
||||
return builder, programs, records
|
||||
|
||||
|
||||
def cache_size(cache: Path) -> dict:
|
||||
files = [p for p in cache.rglob("*") if p.is_file()]
|
||||
return {"logicalFileBytes": sum(p.stat().st_size for p in files), "fileCount": len(files),
|
||||
"modelsBytes": sum(p.stat().st_size for p in files if p.relative_to(cache).parts[0] == "models"),
|
||||
"objectsBytes": sum(p.stat().st_size for p in files if p.relative_to(cache).parts[0] == "objects")}
|
||||
|
||||
|
||||
def worker(args) -> None:
|
||||
config = json.loads(args.worker_config.read_text())
|
||||
if digest(SCRIPT) != config["scriptSha256"]:
|
||||
raise RuntimeError("Benchmark script changed after preparation")
|
||||
if {key: os.environ.get(key) for key in ENVIRONMENT} != config["environment"]:
|
||||
raise RuntimeError("Build environment changed after preparation")
|
||||
builder, programs, records = generate(config, args.version, save=args.worker == "prepare")
|
||||
if args.worker == "prepare":
|
||||
write_json(Path(config["outputDir"]) / args.version / "prepared-models.json", records)
|
||||
return
|
||||
directory = Path(config["outputDir"]) / args.version / args.label
|
||||
directory.mkdir()
|
||||
cache = directory / "cache"
|
||||
if cache.exists():
|
||||
raise RuntimeError("Cold build requires a previously nonexistent cache directory")
|
||||
rows = []
|
||||
for scenario, model, expected_hit in SCENARIOS:
|
||||
case = directory / scenario
|
||||
case.mkdir()
|
||||
before = time.perf_counter()
|
||||
built = builder.build_native(programs[model], cache_dir=cache)
|
||||
returned = time.perf_counter()
|
||||
release_start = time.perf_counter()
|
||||
if hasattr(built, "close"):
|
||||
built.close()
|
||||
released = time.perf_counter()
|
||||
manifest = built.manifest
|
||||
copied_manifest = case / "build-manifest.json"
|
||||
write_json(copied_manifest, manifest)
|
||||
details = getattr(built, "details", {})
|
||||
size = cache_size(cache)
|
||||
row = {"version": args.version, "run": args.label, "warmup": args.label.startswith("warmup-"),
|
||||
"scenario": scenario, "model": model, "buildSeconds": built.seconds,
|
||||
"buildCallWallSeconds": returned - before, "releaseWallSeconds": released - release_start,
|
||||
"buildAndReleaseWallSeconds": released - before, "cacheHit": built.cache_hit,
|
||||
"buildKey": manifest["buildKey"], "details": details,
|
||||
"executable": str(built.executable), "executableSha256": digest(built.executable),
|
||||
"executableBytes": built.executable.stat().st_size, "cacheBytes": size["logicalFileBytes"], "cacheSize": size,
|
||||
"sourceFilesSha256": config["versions"][args.version]["sourceFilesSha256"],
|
||||
"generated": records[model], "compiler": manifest["compiler"], "compilerFlags": manifest["compilerFlags"],
|
||||
"dependencyHashes": manifest["dependencyHashes"], "sourceHashes": manifest["sourceHashes"],
|
||||
"nativeHeaderHashes": manifest.get("nativeHeaderHashes"),
|
||||
"manifest": str(copied_manifest.relative_to(Path(config["outputDir"]))), "manifestSha256": digest(copied_manifest)}
|
||||
row["checks"] = {"expectedCacheHit": built.cache_hit == expected_hit,
|
||||
"parameterChangeReusesOtherObjects": (details.get("objectCompilations") == 1 and
|
||||
details.get("objectCacheHits") == details.get("unitCount", 0) - 1)
|
||||
if scenario == "parameter_change" and "objectCompilations" in details else None}
|
||||
write_json(case / "measurement.json", row)
|
||||
rows.append(row)
|
||||
if any(value is False for value in row["checks"].values()):
|
||||
raise RuntimeError(f"Unexpected cache behavior in {args.version}/{args.label}/{scenario}; measurement preserved")
|
||||
assert_source(config["versions"][args.version])
|
||||
write_json(directory / "measurements.json", {"complete": True, "solverExecuted": False, "rows": rows})
|
||||
|
||||
|
||||
def invoke(config: dict, mode: str, version: str, label: str | None = None) -> dict:
|
||||
output = Path(config["outputDir"])
|
||||
command = [sys.executable, str(SCRIPT), "--worker", mode, "--worker-config", str(output / "prepared.json"), "--version", version]
|
||||
if label:
|
||||
command += ["--label", label]
|
||||
log = output / "logs" / (version + "-" + (label or mode))
|
||||
start = time.perf_counter()
|
||||
with log.with_suffix(".stdout.log").open("wb") as stdout, log.with_suffix(".stderr.log").open("wb") as stderr:
|
||||
process = subprocess.run(command, cwd=config["versions"][version]["root"], stdout=stdout, stderr=stderr, timeout=config["workerTimeout"])
|
||||
observed = {"command": command, "processWallSeconds": time.perf_counter() - start, "exitCode": process.returncode,
|
||||
"version": version, "run": label, "scope": "Whole worker including import/generation/checks/recording; not build latency."}
|
||||
write_json(log.with_suffix(".process.json"), observed)
|
||||
if process.returncode:
|
||||
raise RuntimeError(f"Worker failed ({process.returncode}); see {log.with_suffix('.stderr.log')}")
|
||||
return observed
|
||||
|
||||
|
||||
def stat(values: list) -> dict:
|
||||
return {"n": len(values), "min": min(values), "median": statistics.median(values), "max": max(values)} if values else {"n": 0, "min": None, "median": None, "max": None}
|
||||
|
||||
|
||||
def metric(row: dict, name: str):
|
||||
return row[name] if name in row else row["details"].get(name)
|
||||
|
||||
|
||||
def summarize(config: dict, rows: list, processes: list) -> dict:
|
||||
expected = 2 * (config["warmups"] + config["repeats"]) * len(SCENARIOS)
|
||||
if len(rows) != expected:
|
||||
raise RuntimeError(f"Incomplete benchmark: expected {expected} rows, found {len(rows)}")
|
||||
compilers = {(r["compiler"], tuple(r["compilerFlags"])) for r in rows}
|
||||
libraries = {json_digest({k: v for k, v in r["dependencyHashes"].items() if "/" not in k}) for r in rows}
|
||||
if len(compilers) != 1 or len(libraries) != 1:
|
||||
raise RuntimeError("Compiler/flags or linked library bytes differ across benchmark runs")
|
||||
statistics_by_version = {}
|
||||
for version in config["versions"]:
|
||||
statistics_by_version[version] = {}
|
||||
for scenario, _, _ in SCENARIOS:
|
||||
selected = [r for r in rows if r["version"] == version and r["scenario"] == scenario and not r["warmup"]]
|
||||
if len(selected) != config["repeats"]:
|
||||
raise RuntimeError(f"Incomplete formal group: {version}/{scenario}")
|
||||
statistics_by_version[version][scenario] = {key: stat([metric(r, key) for r in selected if metric(r, key) is not None]) for key in METRICS}
|
||||
changes = {}
|
||||
for scenario, _, _ in SCENARIOS:
|
||||
changes[scenario] = {}
|
||||
for key in ("buildSeconds", "buildCallWallSeconds", "buildAndReleaseWallSeconds"):
|
||||
old = statistics_by_version["baseline"][scenario][key]["median"]
|
||||
new = statistics_by_version["candidate"][scenario][key]["median"]
|
||||
changes[scenario][key] = {"baselineMedian": old, "candidateMedian": new,
|
||||
"reductionPercent": 100 * (1 - new / old) if old else None,
|
||||
"speedup": old / new if new else None, "method": "ratio of group medians"}
|
||||
return {"complete": True, "solverExecuted": False, "prepared": "prepared.json", "scriptSha256": config["scriptSha256"],
|
||||
"contract": config["contract"], "checks": {"compilerAndFlagsIdentical": True, "linkedLibrariesIdentical": True,
|
||||
"generatedModelsIdenticalAcrossVersions": True, "variantHeaderAndStateLayoutUnchanged": True},
|
||||
"statistics": statistics_by_version, "changes": changes, "rows": rows, "workers": processes}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
parser.add_argument("--baseline-root", type=Path, default=ROOT / "test/incremental-build-20260912/baseline-source")
|
||||
parser.add_argument("--candidate-root", type=Path, default=ROOT)
|
||||
parser.add_argument("--input", type=Path, default=ROOT / "tests/data/test-mql-8-corrected.json")
|
||||
parser.add_argument("--four-input", type=Path, default=ROOT / "tests/data/test-mql-4-corrected.json")
|
||||
parser.add_argument("--variant-node", default="amesim_pnl0001_1")
|
||||
parser.add_argument("--output-dir", type=Path)
|
||||
parser.add_argument("--warmups", type=int, default=1)
|
||||
parser.add_argument("--repeats", type=int, default=3)
|
||||
parser.add_argument("--worker-timeout", type=float, default=300)
|
||||
parser.add_argument("--run", action="store_true")
|
||||
parser.add_argument("--worker", choices=("prepare", "round"), help=argparse.SUPPRESS)
|
||||
parser.add_argument("--worker-config", type=Path, help=argparse.SUPPRESS)
|
||||
parser.add_argument("--version", choices=("baseline", "candidate"), help=argparse.SUPPRESS)
|
||||
parser.add_argument("--label", help=argparse.SUPPRESS)
|
||||
args = parser.parse_args()
|
||||
if args.worker:
|
||||
if not args.worker_config or not args.version or (args.worker == "round" and not args.label):
|
||||
parser.error("Internal worker arguments are incomplete")
|
||||
worker(args)
|
||||
return
|
||||
if not args.output_dir or args.warmups < 1 or args.repeats < 1 or not math.isfinite(args.worker_timeout) or args.worker_timeout <= 0:
|
||||
parser.error("A fresh --output-dir, warmups/repeats >= 1 and a positive worker timeout are required")
|
||||
output = args.output_dir.resolve()
|
||||
if not output.is_relative_to(ROOT / "test") or output.exists():
|
||||
parser.error("Choose a previously nonexistent directory under the ignored test/ tree")
|
||||
versions = {"baseline": source_record(args.baseline_root.resolve()), "candidate": source_record(args.candidate_root.resolve())}
|
||||
output.mkdir(parents=True)
|
||||
(output / "logs").mkdir()
|
||||
try:
|
||||
inputs, change = make_inputs(args, output)
|
||||
config = {"createdAtUtc": datetime.now(timezone.utc).isoformat(), "outputDir": str(output), "versions": versions,
|
||||
"script": str(SCRIPT), "scriptSha256": digest(SCRIPT), "python": sys.version, "pythonExecutable": sys.executable,
|
||||
"platform": platform.platform(), "cpuCount": os.cpu_count(), "environment": {key: os.environ.get(key) for key in ENVIRONMENT},
|
||||
"inputs": inputs, "variant": change, "warmups": args.warmups, "repeats": args.repeats,
|
||||
"workerTimeout": args.worker_timeout, "runRequested": args.run,
|
||||
"contract": {"cold": "Independent empty application cache per version/round; OS caches are not flushed.",
|
||||
"sequence": [name for name, _, _ in SCENARIOS], "solverExecuted": False,
|
||||
"fourBranchCase": "Switch from the full eight-branch project to the corrected four-branch project, then restore A; no cached files are manually deleted.",
|
||||
"wallTiming": "Builder call and adjacent release are measured separately; generation, hashing, imports and recording are outside these spans.",
|
||||
"overlap": "buildSeconds is inside buildCallWallSeconds, which is inside buildAndReleaseWallSeconds. Detail spans are nested; compileSeconds sums concurrent TU walls and is not additive to compileWallSeconds.",
|
||||
"unavailable": "Old builder has no object/stage details; absent measurements are null, never inferred from differences.",
|
||||
"comparison": "Formal group-median ratios; warmups excluded. No old strace timings or solver performance claims."}}
|
||||
write_json(output / "prepared.json", config)
|
||||
for version in versions:
|
||||
invoke(config, "prepare", version)
|
||||
config["generated"] = {version: json.loads((output / version / "prepared-models.json").read_text()) for version in versions}
|
||||
if config["generated"]["baseline"] != config["generated"]["candidate"]:
|
||||
raise RuntimeError("Generated model/XML/header/contract differs between source versions")
|
||||
write_json(output / "prepared.json", config)
|
||||
if not args.run:
|
||||
print(json.dumps({"preparedOnly": True, "outputDir": str(output), "compiled": False, "solverExecuted": False}))
|
||||
return
|
||||
rows, processes = [], []
|
||||
for index in range(args.warmups + args.repeats):
|
||||
label = f"warmup-{index + 1}" if index < args.warmups else f"run-{index - args.warmups + 1}"
|
||||
order = ("baseline", "candidate") if index % 2 == 0 else ("candidate", "baseline")
|
||||
for version in order:
|
||||
processes.append(invoke(config, "round", version, label))
|
||||
rows.extend(json.loads((output / version / label / "measurements.json").read_text())["rows"])
|
||||
for record in versions.values():
|
||||
assert_source(record)
|
||||
summary = summarize(config, rows, processes)
|
||||
write_json(output / "summary.json", summary)
|
||||
with (output / "timings.csv").open("w", newline="", encoding="utf-8") as stream:
|
||||
writer = csv.DictWriter(stream, fieldnames=["version", "run", "warmup", "scenario", "metric", "value", "n", "min", "median", "max"])
|
||||
writer.writeheader()
|
||||
for row in rows:
|
||||
for key in METRICS:
|
||||
writer.writerow({**{k: row[k] for k in ("version", "run", "warmup", "scenario")}, "metric": key, "value": metric(row, key)})
|
||||
for version, groups in summary["statistics"].items():
|
||||
for scenario, values in groups.items():
|
||||
for key, value in values.items():
|
||||
writer.writerow({"version": version, "run": "formal-statistics", "warmup": False, "scenario": scenario, "metric": key, **value})
|
||||
print(json.dumps({"complete": True, "summary": str(output / "summary.json"), "solverExecuted": False}))
|
||||
except Exception as exc:
|
||||
write_json(output / "failure.json", {"complete": False, "error": str(exc), "type": type(exc).__name__, "solverExecuted": False})
|
||||
raise
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -33,6 +33,76 @@ NEW_CALL = '''double base=area*p*cm/sqrt(T),K=pow(4*base/den,2)*d/length;
|
||||
return sign*native_pipe_resistance(K,rr,den/4,NULL)*den/4;'''
|
||||
|
||||
|
||||
def pipe_source_path(native: Path) -> Path:
|
||||
"""Locate executable pipe formulas, never a modular aggregation entry."""
|
||||
module = native / 'components/modules/pipe.c'
|
||||
legacy = native / 'components/kernels.c'
|
||||
if module.is_file():
|
||||
target = module
|
||||
elif (native / 'components/modules').exists():
|
||||
raise ValueError(f'Incomplete modular native snapshot: {module} is missing')
|
||||
else:
|
||||
target = legacy
|
||||
if not target.is_file():
|
||||
raise ValueError(f'Native snapshot has no audited pipe source: {native}')
|
||||
source = target.read_text()
|
||||
if source.count('double native_pipe_resistance(') != 1 or source.count('double native_pipe_flow(') != 1:
|
||||
raise ValueError(f'Unsupported pipe source layout: {target}; refusing an unmodified or uninstrumented run')
|
||||
return target
|
||||
|
||||
|
||||
def working_native_paths() -> list[str]:
|
||||
return sorted('native/' + path.relative_to(ROOT / 'native').as_posix()
|
||||
for path in (ROOT / 'native').rglob('*') if path.is_file())
|
||||
|
||||
|
||||
def build_snapshot(program, native: Path, cache: Path, *, required_sources=()):
|
||||
"""Build exact snapshot sources and verify the modified files were compiled.
|
||||
|
||||
Historical monolithic trees retain their original set of C translation
|
||||
units. This temporary selector override is confined to the standalone
|
||||
manual process; production source selection is always restored afterwards.
|
||||
An incompatible old model/runtime ABI fails compilation explicitly.
|
||||
"""
|
||||
native = native.resolve()
|
||||
pipe = pipe_source_path(native)
|
||||
required = (pipe, *required_sources)
|
||||
old_native = builder.NATIVE
|
||||
old_selector = getattr(builder, '_runtime_sources', None)
|
||||
legacy = pipe.name == 'kernels.c'
|
||||
try:
|
||||
builder.NATIVE = native
|
||||
if legacy and old_selector is not None:
|
||||
builder._runtime_sources = lambda _: sorted(native.rglob('*.c'))
|
||||
elif not legacy and old_selector is None:
|
||||
raise ValueError('A modular snapshot requires the incremental native source selector; use its matching checkout')
|
||||
selected = (builder._runtime_sources(program) if old_selector is not None
|
||||
else sorted(native.rglob('*.c')))
|
||||
if any(path.resolve() not in {source.resolve() for source in selected} for path in required):
|
||||
raise ValueError('The model/build selector does not compile the modified pipe/RHS source; refusing a misleading run')
|
||||
try:
|
||||
build = builder.build_native(program, cache_dir=cache)
|
||||
except RuntimeError as exc:
|
||||
raise RuntimeError(
|
||||
f'Cannot build this native snapshot ({native}). Its source layout or model/runtime ABI may be incompatible; '
|
||||
'use matching compiler/runtime revisions. No benchmark was executed.'
|
||||
) from exc
|
||||
recorded = build.manifest.get('sourceHashes', {})
|
||||
for path in required:
|
||||
suffix = 'native/' + path.resolve().relative_to(native).as_posix()
|
||||
matches = [digest for name, digest in recorded.items()
|
||||
if name == suffix or name.replace('\\', '/').endswith('/' + suffix)]
|
||||
if matches != [sha256(path.read_bytes()).hexdigest()]:
|
||||
if hasattr(build, 'close'):
|
||||
build.close()
|
||||
raise ValueError(f'Build manifest does not prove the modified source was compiled: {suffix}')
|
||||
return build
|
||||
finally:
|
||||
builder.NATIVE = old_native
|
||||
if old_selector is not None:
|
||||
builder._runtime_sources = old_selector
|
||||
|
||||
|
||||
def main():
|
||||
parser=argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('input',type=Path)
|
||||
@@ -42,7 +112,8 @@ def main():
|
||||
args=parser.parse_args()
|
||||
if args.runs<1:parser.error('--runs must be positive')
|
||||
out=args.output_dir.resolve();out.mkdir(parents=True,exist_ok=True)
|
||||
if (out/'summary.json').exists():parser.error('Choose a fresh output directory')
|
||||
if (out/'summary.json').exists() or any((out/name).exists() for name in ('fixed-point','previous-newton','guarded-newton')):
|
||||
parser.error('Choose a fresh output directory; existing variant sources must not be mixed')
|
||||
revision=subprocess.check_output(['git','rev-parse',args.baseline_ref],cwd=ROOT,text=True).strip()
|
||||
xml,doc=load_input(args.input)
|
||||
program=compile_native_program(compile_system_xml_network(doc))
|
||||
@@ -55,16 +126,16 @@ def main():
|
||||
try:
|
||||
for variant in ('fixed-point','previous-newton','guarded-newton'):
|
||||
directory=out/variant
|
||||
for name in paths:
|
||||
variant_paths = working_native_paths() if variant == 'guarded-newton' else paths
|
||||
for name in variant_paths:
|
||||
if variant=='guarded-newton':data=(ROOT/name).read_bytes()
|
||||
else:data=subprocess.check_output(['git','show',f'{revision}:{name}'],cwd=ROOT)
|
||||
target=directory/name;target.parent.mkdir(parents=True,exist_ok=True);target.write_bytes(data)
|
||||
if variant=='fixed-point':
|
||||
target=directory/'native/components/kernels.c';source=target.read_text()
|
||||
target=pipe_source_path(directory/'native');source=target.read_text()
|
||||
if source.count(NEW_CALL)!=1:raise ValueError('Baseline pipe flow layout does not match the audited fixed-point substitution')
|
||||
target.write_text(source.replace(NEW_CALL,OLD_ITERATION))
|
||||
builder.NATIVE=directory/'native'
|
||||
variants[variant]=builder.build_native(program,cache_dir=out/'cache')
|
||||
variants[variant]=build_snapshot(program,directory/'native',out/'cache')
|
||||
finally:builder.NATIVE=original_native
|
||||
rows=[]
|
||||
for index in range(args.runs+1):
|
||||
@@ -80,7 +151,9 @@ def main():
|
||||
summary={'baselineRef':revision,'input':str(args.input.resolve()),
|
||||
'inputSha256':sha256(args.input.read_bytes()).hexdigest(),'xmlSha256':sha256(xml).hexdigest(),
|
||||
'settings':vars(config),'sampleStep':doc.simulation.sample_step,'rows':rows,
|
||||
'variants':{name:{'sourceSha256':sha256((out/name/'native/components/kernels.c').read_bytes()).hexdigest(),
|
||||
'variants':{name:{'sourceSha256':sha256(pipe_source_path(out/name/'native').read_bytes()).hexdigest(),
|
||||
'pipeSource':str(pipe_source_path(out/name/'native').relative_to(out/name)),
|
||||
'nativeSourceHashes':build.manifest['sourceHashes'],
|
||||
'buildKey':build.manifest['buildKey'],
|
||||
'medianSolveSeconds':statistics.median(row['solveSeconds'] for row in rows if row['variant']==name and row['run']>0),
|
||||
'medianProcessSeconds':statistics.median(row['processWallSeconds'] for row in rows if row['variant']==name and row['run']>0)}
|
||||
|
||||
@@ -220,9 +220,15 @@ def prepare(args: argparse.Namespace) -> dict:
|
||||
# Reject numerical/runtime drift; a sparse timing-only cached main is allowed
|
||||
# because control and our current writer share the numeric model contract.
|
||||
differences = []
|
||||
for name, expected in manifest["sourceHashes"].items():
|
||||
recorded_sources = dict(manifest["sourceHashes"])
|
||||
if manifest.get("cacheVersion", 1) >= 2:
|
||||
headers = manifest.get("nativeHeaderHashes")
|
||||
if not isinstance(headers, dict) or not headers:
|
||||
raise RuntimeError("Cached build lacks native header hashes; rebuild before profiling")
|
||||
recorded_sources.update(headers)
|
||||
for name, expected in recorded_sources.items():
|
||||
relative = name.split("native/", 1)[-1]
|
||||
current = ROOT / "native" / relative
|
||||
current = cache / relative if relative == "model.c" else ROOT / "native" / relative
|
||||
if digest(current) != expected:
|
||||
differences.append(relative)
|
||||
if any(name != "runtime/main.c" for name in differences):
|
||||
@@ -243,7 +249,16 @@ def prepare(args: argparse.Namespace) -> dict:
|
||||
for name in ("model.c", "model.h", "manifest.json"):
|
||||
shutil.copy2(cache / name, output / name)
|
||||
instrument(native)
|
||||
command = [compiler, *manifest["compilerFlags"], "-I", str(output), "-I", str(native / "include"), "-I", str(sundials / "include"), str(output / "model.c"), *map(str, sorted(native.rglob("*.c"))), "-Wl,--start-group", *map(str, libraries), "-Wl,--end-group", "-lm", "-o", str(output / "profiled-model")]
|
||||
# Match the cached translation units. Compiling the compatibility entry
|
||||
# together with its new modules would define every component twice.
|
||||
sources = [native / name.split("native/", 1)[-1]
|
||||
for name in manifest["sourceHashes"]
|
||||
if name.endswith(".c") and name != "model.c"]
|
||||
if not sources or (native / "components/kernels.c" in sources and
|
||||
any(path.parent == native / "components/modules" for path in sources)):
|
||||
raise RuntimeError("Cached translation units are missing or mix amalgamation and modules")
|
||||
sources.append(native / "runtime/compute_profile.c")
|
||||
command = [compiler, *manifest["compilerFlags"], "-I", str(output), "-I", str(native / "include"), "-I", str(sundials / "include"), str(output / "model.c"), *map(str, sources), "-Wl,--start-group", *map(str, libraries), "-Wl,--end-group", "-lm", "-o", str(output / "profiled-model")]
|
||||
prepared = {"controlExecutable": str(cache / "model"), "profiledExecutable": str(output / "profiled-model"), "buildCommand": command, "runtimeArguments": numerical_arguments, "verifyJacobian": "--verify-jacobian" in numerical_arguments, "algorithm": "production-automatic", "cacheDir": str(cache), "cachedMainDifference": differences, "stateCount": len(manifest["stateKeys"]), "modelSha256": digest(output / "model.c"), "compiler": compiler_version, "warmups": args.warmups, "repeats": args.repeats, "scope": "Inclusive times overlap. Exclusive scope times partition integration including instrumentation overhead. No component or libc sub-cost inference.", "sourceHashes": {str(p.relative_to(output)): digest(p) for p in sorted(native.rglob("*")) if p.is_file()}}
|
||||
write_json(output / "prepared.json", prepared)
|
||||
return prepared
|
||||
|
||||
@@ -24,7 +24,9 @@ from app.simulation.backends import simulation_config
|
||||
from app.simulation.native_codegen import build as builder
|
||||
from app.simulation.native_codegen.compiler import compile_native_program
|
||||
from app.simulation.native_codegen.input import load_input
|
||||
from tests.manual.benchmark_native_pipe_solver import build_snapshot, pipe_source_path, working_native_paths
|
||||
|
||||
SNAPSHOT_HELPER = ROOT / 'tests/manual/benchmark_native_pipe_solver.py'
|
||||
VARIANTS = ('guarded-newton', 'previous-newton', 'fixed-point')
|
||||
FIELDS = '''cache_requests cache_hits cache_misses flow_calls zero_pressure_calls
|
||||
pnl00r_analytic_calls resistance_calls scalar_analytic_calls iterative_calls
|
||||
@@ -214,7 +216,7 @@ def instrument(source: str, variant: str) -> str:
|
||||
limit = '(kind==0?64:16)' if fixed else '80' if variant == 'previous-newton' else '128'
|
||||
prefix = PROFILE_PREFIX.replace('@VARIANT@', variant).replace('@FIXED@', str(int(fixed)))
|
||||
prefix = prefix.replace('@LIMIT@', limit).replace('@FIELDS@', ' '.join(f'X({name})' for name in FIELDS))
|
||||
source = replace_once(source, '#include <stddef.h>', '#include <stddef.h>\n' + prefix)
|
||||
source = replace_once(source, '#include <math.h>', '#include <math.h>\n' + prefix)
|
||||
# Exhaustion is marked at the actual loop fall-through, not inferred from
|
||||
# visiting the last allowed iteration (which can still converge).
|
||||
start = source.index('double native_pipe_resistance(')
|
||||
@@ -259,6 +261,19 @@ def instrument(source: str, variant: str) -> str:
|
||||
return source
|
||||
|
||||
|
||||
|
||||
def instrument_common(source: str) -> str:
|
||||
"""Count both ordinary and canonical Jacobian RHS evaluations exactly once."""
|
||||
calls = [('model_eval', ' return model_eval(t,y,dy,w);')]
|
||||
if 'int native_jacobian_rhs(' in source:
|
||||
calls.append(('model_eval_jacobian', ' return model_eval_jacobian(t,y,dy,w);'))
|
||||
for function, fragment in calls:
|
||||
source = replace_once(source, fragment,
|
||||
' extern void pipe_profile_rhs_enter(void),pipe_profile_rhs_leave(void);\n'
|
||||
f' pipe_profile_rhs_enter();int ok={function}(t,y,dy,w);pipe_profile_rhs_leave();return ok;')
|
||||
return source
|
||||
|
||||
|
||||
def git(*arguments: str) -> str:
|
||||
return subprocess.check_output(['git', *arguments], cwd=ROOT, text=True).strip()
|
||||
|
||||
@@ -268,11 +283,14 @@ def prepare(args, out: Path) -> dict:
|
||||
metadata = json.loads((out / 'prepared.json').read_text())
|
||||
if metadata['input_sha256'] != sha256(args.input.read_bytes()).hexdigest():
|
||||
raise ValueError('Prepared model no longer matches the input file')
|
||||
if metadata['script_sha256'] != sha256(Path(__file__).read_bytes()).hexdigest():
|
||||
if (metadata['script_sha256'] != sha256(Path(__file__).read_bytes()).hexdigest()
|
||||
or metadata.get('snapshot_helper_sha256') != sha256(SNAPSHOT_HELPER.read_bytes()).hexdigest()):
|
||||
raise ValueError('Diagnostic helper changed; choose a fresh output directory')
|
||||
return metadata
|
||||
if any((out / variant).exists() for variant in VARIANTS):
|
||||
raise ValueError('Partially prepared variant directories exist; choose a fresh output directory')
|
||||
previous = git('rev-parse', args.previous_ref)
|
||||
current = git('rev-parse', args.current_ref)
|
||||
current = 'working-tree' if args.current_ref == 'working-tree' else git('rev-parse', args.current_ref)
|
||||
xml, doc = load_input(args.input)
|
||||
config = simulation_config(doc.simulation)
|
||||
if config.rtol != 1e-8 or config.t_start != 0 or config.t_stop != 10:
|
||||
@@ -281,37 +299,40 @@ def prepare(args, out: Path) -> dict:
|
||||
out.mkdir(parents=True, exist_ok=True)
|
||||
(out / 'input.xml').write_bytes(xml)
|
||||
(out / 'input.json').write_bytes(args.input.read_bytes())
|
||||
paths = git('ls-tree', '-r', '--name-only', current, 'native').splitlines()
|
||||
metadata = dict(input=str(args.input.resolve()), input_sha256=sha256(args.input.read_bytes()).hexdigest(),
|
||||
xml_sha256=sha256(xml).hexdigest(), script_sha256=sha256(Path(__file__).read_bytes()).hexdigest(),
|
||||
snapshot_helper_sha256=sha256(SNAPSHOT_HELPER.read_bytes()).hexdigest(),
|
||||
previous_revision=previous, current_revision=current, settings=vars(config),
|
||||
sample_step=doc.simulation.sample_step, variants={},
|
||||
measurement_note='Instrumented times are diagnostic overhead and MUST NOT be used as production benchmark results.',
|
||||
counter_scope='rhs counts native_rhs/model_eval calls including rejected trials and Jacobian differences; non_rhs includes initialization/output/probe evaluations.',
|
||||
counter_scope='rhs counts native_rhs/model_eval and, where present, native_jacobian_rhs/model_eval_jacobian calls including rejected trials and Jacobian differences; non_rhs includes initialization/output/probe evaluations.',
|
||||
replay_scope='All guarded RHS resistance calls, including scalar analytic low-Re cases; no sampling or deduplication. Cache hits, zero pressure difference and direct PNL00R analytic calls are excluded and counted separately.',
|
||||
capture_format='Native-endian IEEE-754 binary64 records: base, diameter, length, relative roughness, den=pi*d*mu, kind (six doubles, 48 bytes). Replay on the same host.',
|
||||
residual_test='At actual returned q, independently factored long-double f(Re) evaluates abs(Re^2*f(Re)/K-1)<=1e-9. Separate from each algorithm stopping rule.')
|
||||
original_native = builder.NATIVE
|
||||
prepared_builds = [] # Pin earlier variants while later variants are built.
|
||||
try:
|
||||
for variant in VARIANTS:
|
||||
directory = out / variant
|
||||
revision = current if variant == 'guarded-newton' else previous
|
||||
paths = (working_native_paths() if revision == 'working-tree'
|
||||
else git('ls-tree', '-r', '--name-only', revision, 'native').splitlines())
|
||||
for name in paths:
|
||||
target = directory / name
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
target.write_bytes(subprocess.check_output(['git', 'show', f'{revision}:{name}'], cwd=ROOT))
|
||||
kernel = directory / 'native/components/kernels.c'
|
||||
data = ((ROOT / name).read_bytes() if revision == 'working-tree'
|
||||
else subprocess.check_output(['git', 'show', f'{revision}:{name}'], cwd=ROOT))
|
||||
target.write_bytes(data)
|
||||
kernel = pipe_source_path(directory / 'native')
|
||||
original_hash = sha256(kernel.read_bytes()).hexdigest()
|
||||
kernel.write_text(instrument(kernel.read_text(), variant))
|
||||
common = directory / 'native/runtime/common.c'
|
||||
common.write_text(replace_once(common.read_text(), ' return model_eval(t,y,dy,w);',
|
||||
''' extern void pipe_profile_rhs_enter(void),pipe_profile_rhs_leave(void);
|
||||
pipe_profile_rhs_enter();int ok=model_eval(t,y,dy,w);pipe_profile_rhs_leave();return ok;'''))
|
||||
common.write_text(instrument_common(common.read_text()))
|
||||
main = directory / 'native/runtime/main.c'
|
||||
main.write_text(replace_once(main.read_text(), 'int main(int argc, char **argv) {',
|
||||
'int main(int argc, char **argv) {\n extern void pipe_profile_install(void);pipe_profile_install();'))
|
||||
builder.NATIVE = directory / 'native'
|
||||
build = builder.build_native(program, cache_dir=out / 'cache')
|
||||
build = build_snapshot(program, directory / 'native', out / 'cache', required_sources=(common, main))
|
||||
prepared_builds.append(build)
|
||||
replay_source = directory / 'replay.c'
|
||||
replay_source.write_text(REPLAY_MAIN)
|
||||
replay = directory / ('replay.exe' if os.name == 'nt' else 'replay')
|
||||
@@ -323,7 +344,8 @@ def prepare(args, out: Path) -> dict:
|
||||
if compiled.returncode:
|
||||
raise RuntimeError(f'Replay compilation failed: {compiled.stderr}')
|
||||
metadata['variants'][variant] = dict(executable=str(build.executable), replay=str(replay),
|
||||
build_key=build.manifest['buildKey'], original_kernel_sha256=original_hash,
|
||||
build_key=build.manifest['buildKey'], pipe_source=str(kernel.relative_to(directory)),
|
||||
compiled_source_hashes=build.manifest['sourceHashes'], original_kernel_sha256=original_hash,
|
||||
instrumented_kernel_sha256=sha256(kernel.read_bytes()).hexdigest())
|
||||
print(f'Prepared {variant}', flush=True)
|
||||
finally:
|
||||
@@ -423,7 +445,8 @@ def main():
|
||||
parser.add_argument('--input', type=Path, default=ROOT / 'tests/data/test-mql-8-corrected.json')
|
||||
parser.add_argument('--output-dir', type=Path, required=True)
|
||||
parser.add_argument('--previous-ref', default='5d5a2e1')
|
||||
parser.add_argument('--current-ref', default='808c484')
|
||||
parser.add_argument('--current-ref', default='808c484',
|
||||
help='Audited guarded-solver revision, or working-tree for the current modular sources')
|
||||
parser.add_argument('--timeout', type=float, default=120)
|
||||
parser.add_argument('--prepare-only', action='store_true')
|
||||
args = parser.parse_args()
|
||||
|
||||
@@ -0,0 +1,25 @@
|
||||
"""Read the diagnostic kernel translation unit before test-only injection.
|
||||
|
||||
Only component module C includes are expanded. Header includes and the
|
||||
amalgamation linkage switch remain for the ordinary C preprocessor to handle.
|
||||
An older, unsplit kernels.c is returned unchanged.
|
||||
"""
|
||||
from pathlib import Path
|
||||
import re
|
||||
|
||||
|
||||
NATIVE = Path(__file__).resolve().parents[1] / "native"
|
||||
_SOURCE_INCLUDE = re.compile(r'^[ \t]*#[ \t]*include[ \t]+"([^"\n]+\.c)"[ \t]*$', re.MULTILINE)
|
||||
|
||||
|
||||
def native_kernel_source(native: Path = NATIVE) -> str:
|
||||
entry = native / "components/kernels.c"
|
||||
modules = (native / "components/modules").resolve()
|
||||
|
||||
def expand(match: re.Match[str]) -> str:
|
||||
source = (entry.parent / match[1]).resolve()
|
||||
if not source.is_relative_to(modules):
|
||||
return match[0]
|
||||
return source.read_text(encoding="utf-8")
|
||||
|
||||
return _SOURCE_INCLUDE.sub(expand, entry.read_text(encoding="utf-8"))
|
||||
@@ -0,0 +1,528 @@
|
||||
"""Exercise the native build cache with real, tiny C translation units.
|
||||
|
||||
These tests isolate their sources, libraries and caches in a temporary directory.
|
||||
They execute the linked programs to check behavior, and observe compiler commands
|
||||
to distinguish actual object reuse from a reported cache hit.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from contextlib import ExitStack
|
||||
from dataclasses import replace
|
||||
from hashlib import sha256
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
from app.simulation.native_codegen import build as native_build
|
||||
from app.simulation.native_codegen.compiler import NativeProgram
|
||||
from app.simulation.native_codegen.cache_storage import prune_cache
|
||||
from app.simulation.native_codegen.runner import execute_native
|
||||
from app.simulation.config import SolveIVPConfig
|
||||
|
||||
|
||||
@unittest.skipUnless(sys.platform.startswith("linux") and shutil.which("gcc"),
|
||||
"Small real-compiler fixtures currently require Linux gcc")
|
||||
class NativeBuildCacheTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.temporary = tempfile.TemporaryDirectory(prefix="native-build-cache-tests-")
|
||||
self.addCleanup(self.temporary.cleanup)
|
||||
self.root = Path(self.temporary.name)
|
||||
self.native = self.root / "native"
|
||||
self.cache = self.root / "cache"
|
||||
self.sundials = self.root / "sundials"
|
||||
self.compiler = str(Path(shutil.which("gcc")).resolve())
|
||||
self.real_run = subprocess.run
|
||||
self.compiler_version = self.real_run(
|
||||
[self.compiler, "--version"], capture_output=True, text=True,
|
||||
check=True, timeout=10,
|
||||
).stdout.splitlines()[0]
|
||||
self.commands: list[tuple[str, ...]] = []
|
||||
self.command_lock = threading.Lock()
|
||||
self._write("native/THIRD_PARTY_NOTICES.txt", "Fixture code belongs to this test.\n")
|
||||
self._write("native/include/values.h", '#include "nested/value.h"\n')
|
||||
self._write("native/include/nested/value.h", "#define COMMON_VALUE 3\n")
|
||||
self._write("native/include/signal_config.h", "#define SIGNAL_VALUE 5\n")
|
||||
self._write("native/runtime/main.c", '''#include "model.h"
|
||||
#include <stdio.h>
|
||||
int common_value(void);
|
||||
int main(void) {
|
||||
printf("%.17g\\n", model_value() + common_value() + HEADER_OFFSET);
|
||||
return 0;
|
||||
}
|
||||
''')
|
||||
self._write("native/runtime/common.c", '''#include "values.h"
|
||||
int common_value(void) { return COMMON_VALUE; }
|
||||
''')
|
||||
for name in ("rk45", "cvode_solver", "json_numbers"):
|
||||
self._write(f"native/runtime/{name}.c", "/* unused runtime fixture */\n")
|
||||
self._write("native/encoding/ryu/d2s.c", "/* unused encoder fixture */\n")
|
||||
self._write("native/components/modules/signal.c", '''#include "signal_config.h"
|
||||
double native_signal(void) { return SIGNAL_VALUE; }
|
||||
''')
|
||||
for name in ("properties", "pipe", "orifice", "mechanics"):
|
||||
self._write(f"native/components/modules/{name}.c", "/* unneeded component */\n")
|
||||
self._write("sundials/include/cvode/cvode.h", "/* fixture */\n")
|
||||
self._write("sundials/include/sundials/sundials_config.h", "/* fixture */\n")
|
||||
# An empty archive is sufficient: the fixture does not call SUNDIALS.
|
||||
for name in native_build.LIBRARIES:
|
||||
self._write(f"sundials/lib/libsundials_{name}.a", b"!<arch>\n")
|
||||
self.program = NativeProgram(
|
||||
source='#include "model.h"\ndouble model_value(void) { return native_signal() + 1; }\n',
|
||||
header="#define HEADER_OFFSET 0\ndouble model_value(void);\ndouble native_signal(void);\n",
|
||||
state_keys=(), variables=(), component_types=(),
|
||||
)
|
||||
self.patches = ExitStack()
|
||||
self.addCleanup(self.patches.close)
|
||||
self.patches.enter_context(patch.object(native_build, "ROOT", self.root))
|
||||
self.patches.enter_context(patch.object(native_build, "NATIVE", self.native))
|
||||
self.patches.enter_context(patch.object(
|
||||
native_build, "toolchain", return_value=(self.compiler, self.sundials, self.compiler_version),
|
||||
))
|
||||
self.patches.enter_context(patch.object(native_build.subprocess, "run", side_effect=self._run))
|
||||
|
||||
def _write(self, relative: str, content: str | bytes) -> Path:
|
||||
path = self.root / relative
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
if isinstance(content, bytes):
|
||||
path.write_bytes(content)
|
||||
else:
|
||||
path.write_text(content, encoding="utf-8")
|
||||
return path
|
||||
|
||||
def _run(self, command, *args, **kwargs):
|
||||
with self.command_lock:
|
||||
self.commands.append(tuple(map(str, command)))
|
||||
return self.real_run(command, *args, **kwargs)
|
||||
|
||||
def _build(self, program=None):
|
||||
return native_build.build_native(program or self.program, cache_dir=self.cache)
|
||||
|
||||
def _compiled_count(self) -> int:
|
||||
return sum("-c" in command for command in self.commands)
|
||||
|
||||
def _linked_count(self) -> int:
|
||||
return sum("-o" in command and "-c" not in command and "-E" not in command
|
||||
for command in self.commands)
|
||||
|
||||
def _output(self, build) -> float:
|
||||
completed = self.real_run([str(build.executable)], capture_output=True, text=True,
|
||||
check=True, timeout=10)
|
||||
return float(completed.stdout.strip())
|
||||
|
||||
def _parameter_program(self, value: int):
|
||||
return replace(self.program, source=self.program.source.replace(" + 1;", f" + {value};"))
|
||||
|
||||
def _objects(self):
|
||||
return sorted(path for path in (self.cache / "objects").iterdir()
|
||||
if len(path.name) == 64 and (path / "manifest.json").is_file())
|
||||
|
||||
def _object_for(self, suffix: str) -> Path:
|
||||
matches = [path for path in self._objects()
|
||||
if json.loads((path / "manifest.json").read_text())["sourceName"].endswith(suffix)]
|
||||
self.assertEqual(len(matches), 1, suffix)
|
||||
return matches[0]
|
||||
|
||||
def test_complete_hit_executes_same_program_without_compilation_or_linking(self):
|
||||
first = self._build()
|
||||
self.assertFalse(first.cache_hit)
|
||||
self.assertEqual(self._output(first), 9)
|
||||
self.assertGreater(self._compiled_count(), 1)
|
||||
self.commands.clear()
|
||||
second = self._build()
|
||||
self.assertTrue(second.cache_hit)
|
||||
self.assertEqual(first.manifest["buildKey"], second.manifest["buildKey"])
|
||||
self.assertEqual(self._output(second), 9)
|
||||
self.assertEqual(self._compiled_count(), 0)
|
||||
self.assertEqual(self._linked_count(), 0)
|
||||
|
||||
def test_parameter_edit_reuses_shared_objects_and_recompiles_only_model(self):
|
||||
first = self._build()
|
||||
self.commands.clear()
|
||||
second = self._build(self._parameter_program(11))
|
||||
self.assertFalse(second.cache_hit)
|
||||
self.assertNotEqual(first.manifest["buildKey"], second.manifest["buildKey"])
|
||||
self.assertEqual(self._output(second), 19)
|
||||
self.assertEqual(self._compiled_count(), 1)
|
||||
self.assertEqual(self._linked_count(), 1)
|
||||
self.commands.clear()
|
||||
self.assertTrue(self._build().cache_hit)
|
||||
self.assertEqual(self._compiled_count(), 0)
|
||||
|
||||
def test_model_header_edit_invalidates_only_translation_units_that_include_it(self):
|
||||
self._build()
|
||||
self.commands.clear()
|
||||
changed = replace(self.program, header=self.program.header.replace("OFFSET 0", "OFFSET 7"))
|
||||
result = self._build(changed)
|
||||
self.assertEqual(self._output(result), 16)
|
||||
# Both include model.h, but the model TU's preprocessed tokens are
|
||||
# unchanged because it does not use HEADER_OFFSET. Only main must rebuild.
|
||||
self.assertEqual(self._compiled_count(), 1)
|
||||
|
||||
def test_model_header_declaration_edit_rebuilds_all_actual_dependents(self):
|
||||
self._build()
|
||||
self.commands.clear()
|
||||
changed = replace(self.program, header=self.program.header + "extern int unused_model_state;\n")
|
||||
result = self._build(changed)
|
||||
self.assertEqual(self._output(result), 9)
|
||||
self.assertEqual(self._compiled_count(), 2)
|
||||
|
||||
def test_indirect_header_change_invalidates_the_actual_runtime_object(self):
|
||||
first = self._build()
|
||||
self.commands.clear()
|
||||
self._write("native/include/nested/value.h", "#define COMMON_VALUE 13\n")
|
||||
second = self._build()
|
||||
self.assertFalse(second.cache_hit)
|
||||
self.assertNotEqual(first.manifest["buildKey"], second.manifest["buildKey"])
|
||||
self.assertEqual(self._output(second), 19)
|
||||
self.assertEqual(self._compiled_count(), 1)
|
||||
|
||||
def test_unneeded_modules_and_headers_do_not_invalidate_or_enter_the_build(self):
|
||||
first = self._build()
|
||||
self.commands.clear()
|
||||
self._write("native/components/modules/pipe.c", "this is deliberately invalid C\n")
|
||||
self._write("native/components/modules/new_unused.c", "another deliberately invalid unit\n")
|
||||
self._write("native/include/unused_new_header.h", "#error should never be included\n")
|
||||
second = self._build()
|
||||
self.assertTrue(second.cache_hit)
|
||||
self.assertEqual(first.manifest["buildKey"], second.manifest["buildKey"])
|
||||
self.assertEqual(self._output(second), 9)
|
||||
self.assertEqual(self._compiled_count(), 0)
|
||||
|
||||
def test_used_module_change_rebuilds_one_object_and_changes_executable(self):
|
||||
first = self._build()
|
||||
self.commands.clear()
|
||||
self._write("native/include/signal_config.h", "#define SIGNAL_VALUE 15\n")
|
||||
second = self._build()
|
||||
self.assertNotEqual(first.manifest["buildKey"], second.manifest["buildKey"])
|
||||
self.assertEqual(self._output(second), 19)
|
||||
self.assertEqual(self._compiled_count(), 1)
|
||||
|
||||
def test_compile_flags_and_compiler_identity_cannot_reuse_old_objects(self):
|
||||
first = self._build()
|
||||
original_count = self._compiled_count()
|
||||
self.commands.clear()
|
||||
with patch.object(native_build, "COMPILER_FLAGS", (*native_build.COMPILER_FLAGS, "-DTEST_CACHE_VARIANT=1")):
|
||||
flags_build = self._build()
|
||||
self.assertNotEqual(first.manifest["buildKey"], flags_build.manifest["buildKey"])
|
||||
self.assertEqual(self._compiled_count(), original_count)
|
||||
self.assertEqual(self._output(flags_build), 9)
|
||||
self.commands.clear()
|
||||
with patch.object(native_build, "toolchain", return_value=(self.compiler, self.sundials, self.compiler_version + " test revision")):
|
||||
compiler_build = self._build()
|
||||
self.assertNotEqual(first.manifest["buildKey"], compiler_build.manifest["buildKey"])
|
||||
self.assertEqual(self._compiled_count(), original_count)
|
||||
self.assertEqual(self._output(compiler_build), 9)
|
||||
|
||||
def test_library_change_relinks_but_reuses_compiled_objects(self):
|
||||
first = self._build()
|
||||
self.commands.clear()
|
||||
# Valid ar symbol-table member with no entries, changing bytes only.
|
||||
empty_object = self.root / "library-member.c"
|
||||
empty_object.write_text("int library_fixture_symbol(void) { return 1; }\n")
|
||||
object_path = self.root / "library-member.o"
|
||||
self.real_run([self.compiler, "-c", str(empty_object), "-o", str(object_path)], check=True)
|
||||
self.real_run(["ar", "r", str(self.sundials / "lib/libsundials_core.a"), str(object_path)],
|
||||
capture_output=True, check=True)
|
||||
second = self._build()
|
||||
self.assertNotEqual(first.manifest["buildKey"], second.manifest["buildKey"])
|
||||
self.assertEqual(self._compiled_count(), 0)
|
||||
self.assertEqual(self._linked_count(), 1)
|
||||
self.assertEqual(self._output(second), 9)
|
||||
|
||||
def test_model_manifest_requires_identity_and_complete_safe_artifacts(self):
|
||||
built = self._build()
|
||||
manifest_path = built.executable.parent / "manifest.json"
|
||||
original = manifest_path.read_text()
|
||||
manifest = json.loads(original)
|
||||
mutations = {
|
||||
"empty artifacts": lambda value: value.update(artifacts={}),
|
||||
"missing executable": lambda value: value["artifacts"].pop(built.executable.name),
|
||||
"missing generated header": lambda value: value["artifacts"].pop("model.h"),
|
||||
"wrong build key": lambda value: value.update(buildKey="0" * 64),
|
||||
"wrong cache version": lambda value: value.update(cacheVersion=-1),
|
||||
"wrong object identities": lambda value: value.update(objectKeys=["0" * 64]),
|
||||
"wrong result contract": lambda value: value.update(stateKeys=["unexpected.state"]),
|
||||
"parent traversal": lambda value: value["artifacts"].update({"../outside": "0" * 64}),
|
||||
"absolute path": lambda value: value["artifacts"].update({str(self.root / "outside"): "0" * 64}),
|
||||
}
|
||||
for label, mutate in mutations.items():
|
||||
with self.subTest(label=label):
|
||||
modified = json.loads(original)
|
||||
mutate(modified)
|
||||
manifest_path.write_text(json.dumps(modified))
|
||||
try:
|
||||
with self.assertRaises(RuntimeError):
|
||||
self._build()
|
||||
finally:
|
||||
manifest_path.write_text(original)
|
||||
self.assertEqual(manifest["cacheVersion"], 2)
|
||||
self.assertTrue(self._build().cache_hit)
|
||||
|
||||
def test_model_source_cannot_be_replaced_by_updating_its_artifact_digest(self):
|
||||
built = self._build()
|
||||
source = built.executable.parent / "model.c"
|
||||
source.write_text(self._parameter_program(12).source)
|
||||
manifest_path = built.executable.parent / "manifest.json"
|
||||
manifest = json.loads(manifest_path.read_text())
|
||||
manifest["artifacts"]["model.c"] = sha256(source.read_bytes()).hexdigest()
|
||||
manifest_path.write_text(json.dumps(manifest))
|
||||
with self.assertRaises(RuntimeError):
|
||||
self._build()
|
||||
|
||||
def test_changed_executable_and_artifact_symlink_are_rejected(self):
|
||||
built = self._build()
|
||||
executable = built.executable
|
||||
original = executable.read_bytes()
|
||||
executable.write_bytes(original + b"corrupt")
|
||||
with self.assertRaises(RuntimeError):
|
||||
self._build()
|
||||
executable.write_bytes(original)
|
||||
target = self.root / "outside-executable"
|
||||
target.write_bytes(original)
|
||||
executable.unlink()
|
||||
executable.symlink_to(target)
|
||||
with self.assertRaises(RuntimeError):
|
||||
self._build()
|
||||
|
||||
def test_object_bytes_and_object_manifest_are_checked_when_reused(self):
|
||||
first = self._build()
|
||||
object_dir = self._object_for("signal.c")
|
||||
unit = object_dir / "unit.o"
|
||||
manifest_path = object_dir / "manifest.json"
|
||||
original_object = unit.read_bytes()
|
||||
original_manifest = manifest_path.read_text()
|
||||
changed_program = self._parameter_program(2)
|
||||
unit.write_bytes(original_object + b"corrupt")
|
||||
# Complete executable reuse need not inspect disposable intermediate objects.
|
||||
self.assertTrue(self._build().cache_hit)
|
||||
with self.assertRaises(RuntimeError):
|
||||
self._build(changed_program)
|
||||
unit.write_bytes(original_object)
|
||||
for label, mutate in {
|
||||
"empty artifacts": lambda value: value.update(artifacts={}),
|
||||
"wrong key": lambda value: value.update(objectKey="0" * 64),
|
||||
"wrong version": lambda value: value.update(cacheVersion=-1),
|
||||
"wrong source": lambda value: value.update(sourceName="native/another.c"),
|
||||
"wrong preprocessing": lambda value: value.update(preprocessedSha256="0" * 64),
|
||||
"wrong compiler": lambda value: value.update(compiler={}),
|
||||
"parent traversal": lambda value: value["artifacts"].update({"../unit.o": "0" * 64}),
|
||||
}.items():
|
||||
with self.subTest(label=label):
|
||||
modified = json.loads(original_manifest)
|
||||
mutate(modified)
|
||||
manifest_path.write_text(json.dumps(modified))
|
||||
try:
|
||||
with self.assertRaises(RuntimeError):
|
||||
self._build(changed_program)
|
||||
finally:
|
||||
manifest_path.write_text(original_manifest)
|
||||
target = self.root / "outside-unit.o"
|
||||
target.write_bytes(original_object)
|
||||
unit.unlink()
|
||||
unit.symlink_to(target)
|
||||
with self.assertRaises(RuntimeError):
|
||||
self._build(changed_program)
|
||||
self.assertEqual(self._output(first), 9)
|
||||
|
||||
def test_library_change_during_link_is_rejected_before_publication(self):
|
||||
first = self._build()
|
||||
before = {path.name for path in (self.cache / "models").iterdir() if len(path.name) == 64}
|
||||
source = self._write("replacement-library.c", "int replacement_library(void) { return 7; }\n")
|
||||
object_path = self.root / "replacement-library.o"
|
||||
alternate = self.root / "alternate.a"
|
||||
self.real_run([self.compiler, "-c", str(source), "-o", str(object_path)], check=True)
|
||||
self.real_run(["ar", "rc", str(alternate), str(object_path)], capture_output=True, check=True)
|
||||
library = self.sundials / "lib/libsundials_core.a"
|
||||
original = library.read_bytes()
|
||||
linked_successfully = []
|
||||
|
||||
def replace_library_at_link(command, *args, **kwargs):
|
||||
is_link = "-o" in command and "-c" not in command and "-E" not in command
|
||||
if is_link:
|
||||
library.write_bytes(alternate.read_bytes())
|
||||
result = self._run(command, *args, **kwargs)
|
||||
if is_link:
|
||||
linked_successfully.append(result.returncode == 0)
|
||||
return result
|
||||
|
||||
try:
|
||||
with patch.object(native_build.subprocess, "run", side_effect=replace_library_at_link):
|
||||
with self.assertRaises(RuntimeError):
|
||||
self._build(self._parameter_program(2))
|
||||
self.assertEqual(linked_successfully, [True])
|
||||
published = {path.name for path in (self.cache / "models").iterdir() if len(path.name) == 64}
|
||||
self.assertEqual(published, before)
|
||||
self.assertEqual(self._output(first), 9)
|
||||
finally:
|
||||
library.write_bytes(original)
|
||||
self.assertTrue(self._build().cache_hit)
|
||||
|
||||
def test_failed_compile_or_link_does_not_publish_a_model_or_break_old_cache(self):
|
||||
first = self._build()
|
||||
before = {path.name for path in (self.cache / "models").iterdir() if len(path.name) == 64}
|
||||
for source in (
|
||||
self.program.source + "this is invalid C;\n",
|
||||
self.program.source + "int unresolved(void); int force_link_failure(void) { return unresolved(); }\n",
|
||||
):
|
||||
with self.subTest(source=source):
|
||||
with self.assertRaises(RuntimeError):
|
||||
self._build(replace(self.program, source=source))
|
||||
published = {path.name for path in (self.cache / "models").iterdir() if len(path.name) == 64}
|
||||
self.assertEqual(published, before)
|
||||
self.assertTrue(self._build().cache_hit)
|
||||
self.assertEqual(self._output(first), 9)
|
||||
|
||||
def test_concurrent_cold_builds_publish_complete_models_and_share_objects(self):
|
||||
gate = threading.Barrier(2)
|
||||
|
||||
def build_together(program):
|
||||
gate.wait(timeout=10)
|
||||
return self._build(program)
|
||||
|
||||
with ThreadPoolExecutor(max_workers=2) as pool:
|
||||
futures = [pool.submit(build_together, self.program) for _ in range(2)]
|
||||
first, second = [future.result(timeout=60) for future in futures]
|
||||
self.assertEqual(first.executable, second.executable)
|
||||
self.assertEqual(self._output(first), 9)
|
||||
self.assertEqual(self._output(second), 9)
|
||||
self.assertTrue(self._build().cache_hit)
|
||||
gate = threading.Barrier(2)
|
||||
with ThreadPoolExecutor(max_workers=2) as pool:
|
||||
futures = [pool.submit(build_together, self._parameter_program(value)) for value in (2, 3)]
|
||||
changed = [future.result(timeout=60) for future in futures]
|
||||
self.assertEqual([self._output(item) for item in changed], [10, 11])
|
||||
self.assertNotEqual(changed[0].manifest["buildKey"], changed[1].manifest["buildKey"])
|
||||
for path in self._objects():
|
||||
self.assertTrue((path / "unit.o").is_file())
|
||||
for item in (first, *changed):
|
||||
manifest = json.loads((item.executable.parent / "manifest.json").read_text())
|
||||
self.assertEqual(manifest["buildKey"], item.manifest["buildKey"])
|
||||
self.assertTrue(all((item.executable.parent / name).is_file() for name in manifest["artifacts"]))
|
||||
|
||||
|
||||
def test_generated_sources_keep_exact_lf_under_windows_default_translation(self):
|
||||
write_text = Path.write_text
|
||||
|
||||
def windows_write_text(path, data, encoding=None, errors=None, newline=None):
|
||||
# Python text output on Windows expands LF when newline is omitted.
|
||||
# Emulate that default on Linux while honoring explicit newline="\n".
|
||||
if newline is None:
|
||||
data = data.replace("\n", "\r\n")
|
||||
return write_text(path, data, encoding=encoding, errors=errors, newline="\n")
|
||||
|
||||
with patch.object(Path, "write_text", windows_write_text):
|
||||
probe = self._write("platform-newline-probe.txt", "first\nsecond\n")
|
||||
self.assertEqual(probe.read_bytes(), b"first\r\nsecond\r\n")
|
||||
first = self._build()
|
||||
self.assertEqual((first.executable.parent / "model.c").read_bytes(), self.program.source.encode())
|
||||
self.assertEqual((first.executable.parent / "model.h").read_bytes(), self.program.header.encode())
|
||||
self.assertEqual(self._output(first), 9)
|
||||
self.commands.clear()
|
||||
second = self._build()
|
||||
self.assertTrue(second.cache_hit)
|
||||
self.assertEqual(second.manifest["buildKey"], first.manifest["buildKey"])
|
||||
self.assertEqual(self._output(second), 9)
|
||||
self.assertEqual(self._compiled_count(), 0)
|
||||
self.assertEqual(self._linked_count(), 0)
|
||||
|
||||
def test_capacity_sweep_protects_build_leases_and_close_allows_eviction(self):
|
||||
older = self._build()
|
||||
newer = self._build(self._parameter_program(2))
|
||||
try:
|
||||
report = prune_cache(self.cache, model_limit_bytes=0)
|
||||
self.assertGreaterEqual(report["models"]["skippedInUse"], 2)
|
||||
self.assertTrue(older.executable.is_file())
|
||||
self.assertTrue(newer.executable.is_file())
|
||||
self.assertEqual(self._output(older), 9)
|
||||
# Release the newer lease first under the normal budget, avoiding
|
||||
# any assumption that two model keys occupy distinct lock shards.
|
||||
newer.close()
|
||||
with patch.dict(os.environ, {"SIMULATION_NATIVE_MODEL_CACHE_MB": "0"}):
|
||||
older.close()
|
||||
self.assertFalse(older.executable.exists())
|
||||
self.assertTrue(newer.executable.is_file())
|
||||
self.assertEqual(self._output(newer), 10)
|
||||
report = prune_cache(self.cache, model_limit_bytes=0)
|
||||
self.assertEqual(report["models"]["removedEntries"], 0)
|
||||
self.assertEqual(report["models"]["oversizedEntries"], 1)
|
||||
self.assertGreater(report["models"]["overLimitBytes"], 0)
|
||||
finally:
|
||||
older.close()
|
||||
newer.close()
|
||||
|
||||
def test_executing_native_worker_keeps_its_model_during_capacity_sweep(self):
|
||||
self._write("native/runtime/main.c", r"""#include "model.h"
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <time.h>
|
||||
int common_value(void);
|
||||
int main(int argc, char **argv) {
|
||||
const char *output = NULL;
|
||||
const char *release = getenv("NATIVE_CACHE_TEST_RELEASE");
|
||||
for (int i = 1; i + 1 < argc; ++i)
|
||||
if (strcmp(argv[i], "--output") == 0) output = argv[i + 1];
|
||||
if (!output || !release) return 64;
|
||||
fprintf(stderr, "{\"phase\":\"integrating\",\"time\":0,\"nfev\":0,\"acceptedSteps\":0}\n");
|
||||
fflush(stderr);
|
||||
struct timespec pause = {0, 1000000};
|
||||
int released = 0;
|
||||
for (int i = 0; i < 10000; ++i) {
|
||||
FILE *gate = fopen(release, "r");
|
||||
if (gate) { fclose(gate); released = 1; break; }
|
||||
nanosleep(&pause, NULL);
|
||||
}
|
||||
if (!released) return 70;
|
||||
FILE *result = fopen(output, "w");
|
||||
if (!result) return 74;
|
||||
int written = fprintf(result,
|
||||
"{\"simulatedUntil\":0.1,\"nfev\":1,\"acceptedSteps\":1,\"value\":%.17g}",
|
||||
model_value() + common_value() + HEADER_OFFSET);
|
||||
return fclose(result) != 0 || written < 0;
|
||||
}
|
||||
""")
|
||||
active = self._build()
|
||||
other = self._build(self._parameter_program(2))
|
||||
release = self.root / "release-worker"
|
||||
worker_running = threading.Event()
|
||||
try:
|
||||
with patch.dict(os.environ, {"NATIVE_CACHE_TEST_RELEASE": str(release)}):
|
||||
with ThreadPoolExecutor(max_workers=1) as pool:
|
||||
future = pool.submit(
|
||||
execute_native, active, SolveIVPConfig(t_stop=0.1), 0.1,
|
||||
run_dir=self.root / "worker-result", timeout=10,
|
||||
progress_callback=lambda fraction, phase: worker_running.set(),
|
||||
)
|
||||
try:
|
||||
self.assertTrue(worker_running.wait(timeout=5), "Native worker did not report readiness")
|
||||
self.assertFalse(future.done())
|
||||
report = prune_cache(self.cache, model_limit_bytes=0)
|
||||
self.assertGreaterEqual(report["models"]["skippedInUse"], 2)
|
||||
self.assertTrue(active.executable.is_file())
|
||||
self.assertTrue((active.executable.parent / "manifest.json").is_file())
|
||||
finally:
|
||||
release.write_text("finish\n")
|
||||
payload = future.result(timeout=15)
|
||||
self.assertEqual(payload["value"], 9)
|
||||
self.assertEqual(payload["buildKey"], active.manifest["buildKey"])
|
||||
other.close()
|
||||
with patch.dict(os.environ, {"SIMULATION_NATIVE_MODEL_CACHE_MB": "0"}):
|
||||
active.close()
|
||||
self.assertFalse(active.executable.exists())
|
||||
self.assertTrue(other.executable.is_file())
|
||||
finally:
|
||||
release.write_text("finish\n")
|
||||
active.close()
|
||||
other.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,376 @@
|
||||
"""Use/eviction races and byte budgets for the immutable native build cache."""
|
||||
from __future__ import annotations
|
||||
|
||||
import gc
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
from app.simulation.native_codegen import cache_storage as storage
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def key(number: int) -> str:
|
||||
# Distinct first words make shard collisions explicit in the tests.
|
||||
return f"{number:08x}" + "0" * 56
|
||||
|
||||
|
||||
class NativeCacheStorageTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.temporary = tempfile.TemporaryDirectory(prefix="native-cache-storage-")
|
||||
self.addCleanup(self.temporary.cleanup)
|
||||
self.folder = Path(self.temporary.name)
|
||||
self.cache = self.folder / "cache"
|
||||
|
||||
def entry(self, number: int, size: int, age: int, kind: str = "models") -> Path:
|
||||
path = self.cache / kind / key(number)
|
||||
path.mkdir(parents=True)
|
||||
(path / "artifact").write_bytes(b"x" * size)
|
||||
os.utime(path, ns=(1_000_000_000 + age, 1_000_000_000 + age))
|
||||
return path
|
||||
|
||||
def lease(self, number: int, **kwargs):
|
||||
lease = storage.acquire_cache_lease(self.cache, "models", key(number), **kwargs)
|
||||
if lease is not None:
|
||||
self.addCleanup(lease.close)
|
||||
return lease
|
||||
|
||||
def test_overlapping_shared_leases_and_nonblocking_exclusion(self):
|
||||
first = self.lease(0)
|
||||
second = self.lease(0, blocking=False)
|
||||
self.assertIsNotNone(second)
|
||||
self.assertIsNone(self.lease(0, exclusive=True, blocking=False))
|
||||
first.close()
|
||||
self.assertTrue(first.closed)
|
||||
self.assertIsNone(self.lease(0, exclusive=True, blocking=False))
|
||||
second.close()
|
||||
writer = self.lease(0, exclusive=True, blocking=False)
|
||||
self.assertIsNotNone(writer)
|
||||
self.assertIsNone(self.lease(0, blocking=False))
|
||||
writer.close()
|
||||
writer.close() # Explicit close and cleanup are idempotent.
|
||||
with self.assertRaises(RuntimeError):
|
||||
with writer:
|
||||
pass
|
||||
|
||||
def test_separate_process_sees_shared_and_exclusive_lock_contract(self):
|
||||
self.lease(1)
|
||||
script = r'''
|
||||
import json, sys
|
||||
from pathlib import Path
|
||||
from app.simulation.native_codegen.cache_storage import acquire_cache_lease
|
||||
cache, key = Path(sys.argv[1]), sys.argv[2]
|
||||
shared = acquire_cache_lease(cache, 'models', key, blocking=False)
|
||||
exclusive = acquire_cache_lease(cache, 'models', key, exclusive=True, blocking=False)
|
||||
print(json.dumps({'shared': shared is not None, 'exclusive': exclusive is not None}))
|
||||
if shared: shared.close()
|
||||
if exclusive: exclusive.close()
|
||||
'''
|
||||
result = subprocess.run(
|
||||
[sys.executable, "-c", script, str(self.cache), key(1)], cwd=ROOT,
|
||||
capture_output=True, text=True, timeout=15, check=True,
|
||||
)
|
||||
self.assertEqual(json.loads(result.stdout), {"shared": True, "exclusive": False})
|
||||
|
||||
def test_blocking_process_proceeds_when_last_reader_releases(self):
|
||||
lease = self.lease(2)
|
||||
script = r'''
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from app.simulation.native_codegen.cache_storage import acquire_cache_lease
|
||||
print('waiting', flush=True)
|
||||
with acquire_cache_lease(Path(sys.argv[1]), 'models', sys.argv[2], exclusive=True):
|
||||
print('acquired', flush=True)
|
||||
'''
|
||||
process = subprocess.Popen(
|
||||
[sys.executable, "-c", script, str(self.cache), key(2)], cwd=ROOT,
|
||||
stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True,
|
||||
)
|
||||
self.addCleanup(lambda: process.kill() if process.poll() is None else None)
|
||||
self.assertEqual(process.stdout.readline().strip(), "waiting")
|
||||
self.assertIsNone(process.poll())
|
||||
lease.close()
|
||||
stdout, stderr = process.communicate(timeout=15)
|
||||
self.assertEqual(process.returncode, 0, stderr)
|
||||
self.assertEqual(stdout.strip(), "acquired")
|
||||
|
||||
def test_finalizer_releases_abandoned_lease(self):
|
||||
lease = storage.acquire_cache_lease(self.cache, "models", key(3))
|
||||
self.assertIsNone(self.lease(3, exclusive=True, blocking=False))
|
||||
del lease
|
||||
gc.collect()
|
||||
self.assertIsNotNone(self.lease(3, exclusive=True, blocking=False))
|
||||
|
||||
def test_lru_removes_oldest_until_separate_budgets_are_met(self):
|
||||
oldest = self.entry(0, 60, 1)
|
||||
middle = self.entry(1, 60, 2)
|
||||
newest = self.entry(2, 60, 3)
|
||||
object_old = self.entry(3, 40, 1, "objects")
|
||||
object_new = self.entry(4, 40, 2, "objects")
|
||||
report = storage.prune_cache(self.cache, model_limit_bytes=120, object_limit_bytes=40)
|
||||
self.assertFalse(oldest.exists())
|
||||
self.assertTrue(middle.exists() and newest.exists())
|
||||
self.assertFalse(object_old.exists())
|
||||
self.assertTrue(object_new.exists())
|
||||
self.assertEqual(report["models"]["afterBytes"], 120)
|
||||
self.assertEqual(report["objects"]["afterBytes"], 40)
|
||||
self.assertEqual(report["models"]["removedEntries"], 1)
|
||||
self.assertEqual(report["models"]["errors"], [])
|
||||
|
||||
def test_held_entry_survives_and_recent_touch_changes_lru_order(self):
|
||||
first = self.entry(0, 60, 1)
|
||||
second = self.entry(1, 60, 2)
|
||||
third = self.entry(2, 60, 3)
|
||||
held = self.lease(0)
|
||||
with self.lease(1):
|
||||
storage.touch_cache_entry(self.cache, "models", key(1))
|
||||
report = storage.prune_cache(self.cache, model_limit_bytes=60)
|
||||
self.assertTrue(first.exists())
|
||||
self.assertFalse(second.exists() or third.exists())
|
||||
self.assertEqual(report["models"]["skippedInUse"], 1)
|
||||
held.close()
|
||||
|
||||
# A separate round verifies touch order without an active-entry override.
|
||||
old = self.entry(3, 60, 3)
|
||||
with self.lease(0):
|
||||
storage.touch_cache_entry(self.cache, "models", key(0))
|
||||
storage.prune_cache(self.cache, model_limit_bytes=60)
|
||||
self.assertTrue(first.exists())
|
||||
self.assertFalse(old.exists())
|
||||
|
||||
def test_in_use_overage_is_reported_and_later_sweep_recovers(self):
|
||||
first = self.entry(0, 60, 1)
|
||||
second = self.entry(1, 60, 2)
|
||||
a, b = self.lease(0), self.lease(1)
|
||||
report = storage.prune_cache(self.cache, model_limit_bytes=60)
|
||||
self.assertTrue(first.exists() and second.exists())
|
||||
self.assertEqual(report["models"]["overLimitBytes"], 60)
|
||||
self.assertEqual(report["models"]["skippedInUse"], 2)
|
||||
a.close()
|
||||
b.close()
|
||||
report = storage.prune_cache(self.cache, model_limit_bytes=60)
|
||||
self.assertFalse(first.exists())
|
||||
self.assertTrue(second.exists())
|
||||
self.assertEqual(report["models"]["overLimitBytes"], 0)
|
||||
|
||||
def test_only_final_oversized_entry_is_retained(self):
|
||||
old = self.entry(0, 200, 1)
|
||||
new = self.entry(1, 200, 2)
|
||||
report = storage.prune_cache(self.cache, model_limit_bytes=100)
|
||||
self.assertFalse(old.exists())
|
||||
self.assertTrue(new.exists())
|
||||
self.assertEqual(report["models"]["oversizedEntries"], 1)
|
||||
self.assertEqual(report["models"]["overLimitBytes"], 100)
|
||||
|
||||
def test_lock_shards_are_bounded_and_collisions_only_defer_eviction(self):
|
||||
for number in range(512):
|
||||
with storage.acquire_cache_lease(self.cache, "models", key(number)):
|
||||
pass
|
||||
self.assertEqual(len(list((self.cache / ".locks").iterdir())), storage.LOCK_SHARDS)
|
||||
self.assertTrue(all(path.stat().st_size == 0 for path in (self.cache / ".locks").iterdir()))
|
||||
colliding = self.entry(128, 60, 1)
|
||||
other = self.entry(1, 60, 2)
|
||||
self.lease(0)
|
||||
report = storage.prune_cache(self.cache, model_limit_bytes=60)
|
||||
self.assertTrue(colliding.exists())
|
||||
self.assertFalse(other.exists())
|
||||
self.assertEqual(report["models"]["skippedInUse"], 1)
|
||||
|
||||
def test_legacy_unknown_and_linked_entries_are_not_followed_or_removed(self):
|
||||
old = self.entry(0, 60, 1)
|
||||
self.entry(1, 60, 2)
|
||||
legacy = self.cache / key(50)
|
||||
legacy.mkdir()
|
||||
(legacy / "model").write_text("old cache")
|
||||
unknown = self.cache / "models" / "notes.txt"
|
||||
unknown.write_text("user notes")
|
||||
outside = self.folder / "outside"
|
||||
outside.mkdir()
|
||||
(outside / "important").write_text("keep")
|
||||
direct = self.cache / "models" / key(3)
|
||||
nested = self.cache / "models" / key(4)
|
||||
nested.mkdir()
|
||||
try:
|
||||
direct.symlink_to(outside, target_is_directory=True)
|
||||
(nested / "outside").symlink_to(outside, target_is_directory=True)
|
||||
except OSError as exc:
|
||||
self.skipTest(f"Symlinks unavailable: {exc}")
|
||||
report = storage.prune_cache(self.cache, model_limit_bytes=60)
|
||||
self.assertFalse(old.exists())
|
||||
self.assertEqual((legacy / "model").read_text(), "old cache")
|
||||
self.assertEqual(unknown.read_text(), "user notes")
|
||||
self.assertEqual((outside / "important").read_text(), "keep")
|
||||
self.assertTrue(direct.is_symlink() and (nested / "outside").is_symlink())
|
||||
self.assertEqual(report["models"]["skippedUnmanaged"], 3)
|
||||
|
||||
def test_invalid_keys_and_symlink_lock_files_are_rejected(self):
|
||||
for kind, digest in (("../models", key(0)), ("models", "../outside"),
|
||||
("models", "A" * 64), ("models", key(0) + "\n")):
|
||||
with self.subTest(kind=kind, key=digest):
|
||||
with self.assertRaises(ValueError):
|
||||
storage.acquire_cache_lease(self.cache, kind, digest)
|
||||
lock_dir = self.cache / ".locks"
|
||||
lock_dir.mkdir(parents=True)
|
||||
target = self.folder / "outside-lock"
|
||||
target.write_text("untouched")
|
||||
try:
|
||||
(lock_dir / "models-000.lock").symlink_to(target)
|
||||
except OSError as exc:
|
||||
self.skipTest(f"Symlinks unavailable: {exc}")
|
||||
with self.assertRaises(RuntimeError):
|
||||
self.lease(0)
|
||||
self.assertEqual(target.read_text(), "untouched")
|
||||
|
||||
def test_prune_serializes_sweeps_without_blocking_readers(self):
|
||||
old = self.entry(0, 200, 1)
|
||||
new = self.entry(1, 200, 2)
|
||||
import shutil
|
||||
original = shutil.rmtree
|
||||
concurrent = []
|
||||
|
||||
def removing(path, *args, **kwargs):
|
||||
concurrent.append(storage.prune_cache(self.cache, model_limit_bytes=100))
|
||||
# Reader leases do not contend with the global sweep lock.
|
||||
with self.lease(2, blocking=False):
|
||||
pass
|
||||
return original(path, *args, **kwargs)
|
||||
|
||||
with patch.object(storage.shutil, "rmtree", side_effect=removing):
|
||||
storage.prune_cache(self.cache, model_limit_bytes=100)
|
||||
self.assertFalse(old.exists())
|
||||
self.assertTrue(new.exists())
|
||||
self.assertEqual(len(concurrent), 1)
|
||||
self.assertTrue(concurrent[0]["models"]["skippedConcurrentSweep"])
|
||||
|
||||
def stage(self, number: int, size: int, namespace: str = "root") -> Path:
|
||||
parent = self.cache if namespace == "root" else self.cache / namespace
|
||||
parent.mkdir(parents=True, exist_ok=True)
|
||||
stage = Path(tempfile.mkdtemp(prefix=f"building-{key(number)}-", dir=parent))
|
||||
(stage / "unfinished.o").write_bytes(b"x" * size)
|
||||
return stage
|
||||
|
||||
def test_orphan_stages_are_removed_in_all_three_managed_locations(self):
|
||||
root = self.stage(10, 30)
|
||||
model = self.stage(11, 40, "models")
|
||||
obj = self.stage(12, 50, "objects")
|
||||
legacy = self.cache / "building-old12345"
|
||||
legacy.mkdir()
|
||||
unknown = self.cache / "models" / f"building-{key(13)}-short"
|
||||
unknown.mkdir()
|
||||
# Similar names inside an unrelated entry are outside the sweep scope.
|
||||
nested = self.cache / key(14) / f"building-{key(15)}-abcdefgh"
|
||||
nested.mkdir(parents=True)
|
||||
report = storage.prune_cache(self.cache)
|
||||
self.assertFalse(root.exists() or model.exists() or obj.exists())
|
||||
self.assertTrue(legacy.exists() and unknown.exists() and nested.exists())
|
||||
self.assertEqual(report["orphanStages"]["orphanStagesRemoved"], 3)
|
||||
self.assertEqual(report["orphanStages"]["orphanStagesBytes"], 120)
|
||||
self.assertEqual(report["orphanStages"]["skippedUnmanaged"], 2)
|
||||
self.assertEqual(report["orphanStages"]["errors"], [])
|
||||
|
||||
def test_active_stages_use_their_correct_namespace_and_survive(self):
|
||||
model_lease = self.lease(10)
|
||||
object_lease = storage.acquire_cache_lease(self.cache, "objects", key(11))
|
||||
self.addCleanup(object_lease.close)
|
||||
root_active = self.stage(10, 30)
|
||||
model_active = self.stage(10, 40, "models")
|
||||
object_active = self.stage(11, 50, "objects")
|
||||
collision = self.stage(10 + storage.LOCK_SHARDS, 20)
|
||||
# Holding models/key(10) does not protect objects/key(10), or vice versa.
|
||||
object_orphan = self.stage(10, 60, "objects")
|
||||
model_orphan = self.stage(11, 70)
|
||||
report = storage.prune_cache(self.cache)["orphanStages"]
|
||||
self.assertTrue(all(path.exists() for path in (
|
||||
root_active, model_active, object_active, collision,
|
||||
)))
|
||||
self.assertFalse(object_orphan.exists() or model_orphan.exists())
|
||||
self.assertEqual(report["skippedInUse"], 4)
|
||||
self.assertEqual(report["orphanStagesRemoved"], 2)
|
||||
self.assertEqual(report["orphanStagesBytes"], 130)
|
||||
model_lease.close()
|
||||
object_lease.close()
|
||||
report = storage.prune_cache(self.cache)["orphanStages"]
|
||||
self.assertEqual(report["orphanStagesRemoved"], 4)
|
||||
self.assertEqual(report["orphanStagesBytes"], 140)
|
||||
|
||||
def test_killed_builder_releases_stage_protection_for_next_sweep(self):
|
||||
script = r"""
|
||||
import sys, tempfile, time
|
||||
from pathlib import Path
|
||||
from app.simulation.native_codegen.cache_storage import acquire_cache_lease
|
||||
cache, key = Path(sys.argv[1]), sys.argv[2]
|
||||
lease = acquire_cache_lease(cache, 'models', key)
|
||||
stage = Path(tempfile.mkdtemp(prefix='building-' + key + '-', dir=cache))
|
||||
(stage / 'unfinished.o').write_bytes(b'x' * 37)
|
||||
print(stage.name, flush=True)
|
||||
time.sleep(60)
|
||||
"""
|
||||
process = subprocess.Popen(
|
||||
[sys.executable, "-c", script, str(self.cache), key(16)], cwd=ROOT,
|
||||
stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True,
|
||||
)
|
||||
self.addCleanup(lambda: process.kill() if process.poll() is None else None)
|
||||
stage = self.cache / process.stdout.readline().strip()
|
||||
self.assertTrue(stage.is_dir())
|
||||
report = storage.prune_cache(self.cache)["orphanStages"]
|
||||
self.assertTrue(stage.exists())
|
||||
self.assertEqual(report["skippedInUse"], 1)
|
||||
process.kill()
|
||||
process.communicate(timeout=15)
|
||||
report = storage.prune_cache(self.cache)["orphanStages"]
|
||||
self.assertFalse(stage.exists())
|
||||
self.assertEqual(report["orphanStagesRemoved"], 1)
|
||||
self.assertEqual(report["orphanStagesBytes"], 37)
|
||||
|
||||
def test_orphan_stage_symlinks_and_symlink_namespaces_are_not_followed(self):
|
||||
self.cache.mkdir()
|
||||
outside = self.folder / "outside-stage"
|
||||
outside.mkdir()
|
||||
artifact = outside / "important"
|
||||
artifact.write_text("keep")
|
||||
direct = self.cache / f"building-{key(20)}-abcdefgh"
|
||||
nested = self.stage(21, 10)
|
||||
object_namespace = self.cache / "objects"
|
||||
try:
|
||||
direct.symlink_to(outside, target_is_directory=True)
|
||||
(nested / "link").symlink_to(artifact)
|
||||
object_namespace.symlink_to(outside, target_is_directory=True)
|
||||
except OSError as exc:
|
||||
self.skipTest(f"Symlinks unavailable: {exc}")
|
||||
# A matching name behind a namespace link is never inspected or removed.
|
||||
outside_stage = outside / f"building-{key(22)}-abcdefgh"
|
||||
outside_stage.mkdir()
|
||||
report = storage.prune_cache(self.cache)["orphanStages"]
|
||||
self.assertEqual(report["orphanStagesRemoved"], 0)
|
||||
self.assertEqual(report["skippedUnmanaged"], 2)
|
||||
self.assertTrue(direct.is_symlink() and nested.exists() and outside_stage.exists())
|
||||
self.assertEqual(artifact.read_text(), "keep")
|
||||
|
||||
def test_orphan_cleanup_failure_is_reported_without_removing_other_data(self):
|
||||
stage = self.stage(24, 80)
|
||||
with patch.object(storage.shutil, "rmtree", side_effect=PermissionError("busy stage")):
|
||||
report = storage.prune_cache(self.cache)["orphanStages"]
|
||||
self.assertTrue(stage.exists())
|
||||
self.assertEqual(report["orphanStagesRemoved"], 0)
|
||||
self.assertEqual(report["orphanStagesBytes"], 0)
|
||||
self.assertEqual(len(report["errors"]), 1)
|
||||
self.assertIn("busy stage", report["errors"][0])
|
||||
|
||||
def test_filesystem_cleanup_errors_do_not_fail_simulation(self):
|
||||
self.entry(0, 60, 1)
|
||||
self.entry(1, 60, 2)
|
||||
with patch.object(storage.shutil, "rmtree", side_effect=PermissionError("busy")):
|
||||
report = storage.prune_cache(self.cache, model_limit_bytes=60)
|
||||
self.assertEqual(report["models"]["overLimitBytes"], 60)
|
||||
self.assertTrue(report["models"]["errors"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -5,6 +5,7 @@ import tempfile
|
||||
import unittest
|
||||
|
||||
from app.simulation.native_codegen.build import toolchain
|
||||
from tests.native_kernel_source import native_kernel_source
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
@@ -16,7 +17,7 @@ class NativePipeCacheTests(unittest.TestCase):
|
||||
compiler = toolchain()[0]
|
||||
except (OSError, RuntimeError, subprocess.SubprocessError) as exc:
|
||||
self.skipTest(f"Native toolchain unavailable: {exc}")
|
||||
source = (ROOT / "native/components/kernels.c").read_text()
|
||||
source = native_kernel_source()
|
||||
signature = "double d, double length, double rr, int kind) {"
|
||||
self.assertEqual(source.count(signature), 1)
|
||||
source = "static int pipe_calls;\n" + source.replace(
|
||||
|
||||
@@ -12,6 +12,7 @@ from app.simulation.core.medium import IdealGasMedium
|
||||
from app.simulation.native_codegen.build import build_native, toolchain
|
||||
from app.simulation.native_codegen.extended import compile_extended_program
|
||||
from tests.test_native_catalog import Circuit
|
||||
from tests.native_kernel_source import native_kernel_source
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
@@ -109,7 +110,7 @@ int main(void) {
|
||||
'''
|
||||
with tempfile.TemporaryDirectory(prefix='native-pipe-root-') as tmp:
|
||||
directory=Path(tmp);source=directory/'check.c';exe=directory/'check.exe'
|
||||
source.write_text((ROOT/'native/components/kernels.c').read_text()+harness)
|
||||
source.write_text(native_kernel_source()+harness)
|
||||
build=subprocess.run([self.compiler,'-std=c11','-O3','-Wall','-Wextra','-Werror',
|
||||
'-ffp-contract=off','-fno-fast-math','-static-libgcc','-I',str(ROOT/'native/include'),
|
||||
str(source),'-lm','-o',str(exe)],capture_output=True,text=True,timeout=60)
|
||||
|
||||
@@ -8,6 +8,7 @@ import shutil
|
||||
import subprocess
|
||||
import tempfile
|
||||
import unittest
|
||||
from tests.native_kernel_source import native_kernel_source
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
@@ -67,7 +68,7 @@ class NativePipeSolverTests(unittest.TestCase):
|
||||
cls.directory = tempfile.TemporaryDirectory(prefix='native-pipe-solver-')
|
||||
cls.addClassCleanup(cls.directory.cleanup)
|
||||
cls.compiler = command
|
||||
source = (ROOT / 'native/components/kernels.c').read_text()
|
||||
source = native_kernel_source()
|
||||
cls.library = cls.build_library(source, 'ordinary')
|
||||
|
||||
# Fault injection only in the temporary test translation unit. The
|
||||
|
||||
@@ -5,6 +5,7 @@ import tempfile
|
||||
import unittest
|
||||
|
||||
from app.simulation.native_codegen.build import toolchain
|
||||
from tests.native_kernel_source import native_kernel_source
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
@@ -15,7 +16,7 @@ class NativePropertyTests(unittest.TestCase):
|
||||
compiler = toolchain()[0]
|
||||
except (OSError, RuntimeError, subprocess.SubprocessError) as exc:
|
||||
self.skipTest(f"Native toolchain unavailable: {exc}")
|
||||
source = (ROOT / 'native/components/kernels.c').read_text()
|
||||
source = native_kernel_source()
|
||||
for signature, counter in (
|
||||
('static double temperature_ph(double p,double h) {', 'ph_calls'),
|
||||
('static double z_factor(double p,double T) {', 'z_calls'),
|
||||
|
||||
Reference in new issue
Block a user