diff --git a/.github/workflows/solver-regression.yml b/.github/workflows/solver-regression.yml index 52c74df..23d6796 100644 --- a/.github/workflows/solver-regression.yml +++ b/.github/workflows/solver-regression.yml @@ -86,6 +86,15 @@ jobs: "SIMULATION_NATIVE_CC=$env:CONDA_PREFIX\Library\mingw-w64\bin\gcc.exe" >> $env:GITHUB_ENV python -m pip install -r requirements-test.txt python -m pip check + - name: Verify native cache on the Windows runtime + env: + SIMULATION_NATIVE_REQUIRE_TOOLCHAIN: "1" + run: | + New-Item -ItemType Directory -Force test/ci-native-cache | Out-Null + python -W error::ResourceWarning -m unittest tests.test_native_cache_storage tests.test_native_cache_platform -v > test/ci-native-cache/regression.log 2>&1 + $cacheRegressionExit = $LASTEXITCODE + Get-Content test/ci-native-cache/regression.log + if ($cacheRegressionExit -ne 0) { exit $cacheRegressionExit } - name: Run catalog, numerical, API and schema regression run: | python -c "from app.simulation.native_codegen.build import toolchain; print(toolchain())" @@ -100,3 +109,9 @@ jobs: name: native-skill-regression path: test/ci-skill/summary.json if-no-files-found: warn + - if: always() + uses: actions/upload-artifact@v4 + with: + name: native-windows-cache-regression + path: test/ci-native-cache/regression.log + if-no-files-found: warn diff --git a/app/simulation/native_codegen/cache_storage.py b/app/simulation/native_codegen/cache_storage.py index 0ee5e12..0df4406 100644 --- a/app/simulation/native_codegen/cache_storage.py +++ b/app/simulation/native_codegen/cache_storage.py @@ -218,7 +218,13 @@ def touch_cache_entry(cache_dir: Path, kind: str, key: str) -> None: entry = cache / kind / key _require_directory(entry) # Directory mtime is separate from the checksummed artifact/manifest bytes. - os.utime(entry, None, follow_symlinks=False) + # Windows Python may expose utime without the no-follow operation. The + # checks above reject symlinks and reparse points before either call, and + # the caller's use lease protects the entry from application eviction. + if os.utime in os.supports_follow_symlinks: + os.utime(entry, None, follow_symlinks=False) + else: + os.utime(entry, None) @dataclass(frozen=True) diff --git a/docs/other/assets/2026-09-12/native-cache-windows-utime-fix.json b/docs/other/assets/2026-09-12/native-cache-windows-utime-fix.json new file mode 100644 index 0000000..01abf44 --- /dev/null +++ b/docs/other/assets/2026-09-12/native-cache-windows-utime-fix.json @@ -0,0 +1,29 @@ +{ + "host": "Linux-6.8.0-111-generic-x86_64-with-glibc2.39", + "windowsRuntimeExecuted": false, + "case": "Corrected eight-branch 0-10 s with os.utime rejecting follow_symlinks=False, real Linux GCC/CVODE execution.", + "success": true, + "simulatedUntil": 10, + "method": "BDF", + "rtol": 1e-08, + "bitwiseValuesCompared": 1790486, + "allBitsEqual": true, + "coldCacheHit": false, + "subsequentCacheHit": true, + "timestampUpdates": 14, + "solveSeconds": 2.510730743408203, + "buildSeconds": 2.51858819834888, + "sourceSha256": "ace0843c8fe84b687c7dedce9eecae1ab2a9174a0f321f047d99089cfa15b301", + "tests": { + "buildAndStorage": { + "passed": 40, + "seconds": 8.174 + }, + "realToolchainIntegration": { + "passed": 1, + "seconds": 1.399, + "windowsDllBranchExecuted": false + }, + "windowsCi": "Configured; not executed from this Linux session." + } +} diff --git a/docs/standard/跨平台交付约定.md b/docs/standard/跨平台交付约定.md new file mode 100644 index 0000000..d88e03b --- /dev/null +++ b/docs/standard/跨平台交付约定.md @@ -0,0 +1,20 @@ +# Windows 与 Linux 功能交付约定 + +2026-09-12 用户明确要求：后续功能补全同时注意 Windows 平台适配。 + +Windows x64 与 Linux x86_64 都是当前应用的使用平台。涉及文件系统、缓存、编译器、进程、动态库、环境安装或启动脚本的功能修改，应同时检查两侧的实现和测试入口。 + +- 文件系统可选能力应按运行环境检测，例如 `os.supports_follow_symlinks`，不能因为 Linux 接受某个参数就假设 Windows 也接受。缓存目录仍需验证普通目录、符号链接与 Windows reparse point；权限和真实 I/O 错误不能当成兼容问题随意吞掉。 +- 编译与运行应覆盖 Windows 的 `.exe`、导入库/DLL、工具链路径、带空格路径和换行行为。进程锁、并发清理、异常退出应有可在 Windows 执行的测试。 +- 平台专用的小型测试夹具可以明确限定平台，但不能代替跨平台集成测试。Windows CI 中必需的工具链缺失应失败，不能以整组跳过得到成功结论。 +- 交付记录分别列出兼容代码、模拟测试、实际 Windows 测试的状态。只有实际 Windows 运行通过，才能声称 Windows 验收通过。缺少 Windows 环境时继续完成可做的修复和测试准备，并明确实机验证仍待完成。 +- 本地虚拟环境、编译器、便携运行时和构建缓存保留在 Git 忽略目录，不随源码提交。 + +当前缓存检查入口： + +```powershell +$env:SIMULATION_NATIVE_REQUIRE_TOOLCHAIN = '1' +.\.venv-win\Scripts\python.exe -m unittest tests.test_native_cache_storage tests.test_native_cache_platform -v +``` + +该入口使用现有 `SIMULATION_NATIVE_CC` / `SUNDIALS_ROOT` 或构建器的默认探测；CI 配置在 `.github/workflows/solver-regression.yml` 的 `native-windows` 作业。 diff --git a/docs/update-log/更新日志-2026-09-12.md b/docs/update-log/更新日志-2026-09-12.md index c95fd08..3892a12 100644 --- a/docs/update-log/更新日志-2026-09-12.md +++ b/docs/update-log/更新日志-2026-09-12.md @@ -1,5 +1,8 @@ # 2026-09-12：按需编译与分层缓存 +后续 Windows 修复：用户实际运行报 `utime: follow_symlinks unavailable on this platform`。根因是缓存最近使用时间更新无条件传入可选参数，与数值求解无关。按 [Python 平台能力接口](https://docs.python.org/3.12/library/os.html#os.supports_follow_symlinks) 改为能力检测后选择调用，保留目录/reparse 校验和真实权限错误。新增回归在修复前复现同一异常；修复后构建/存储 40 项通过（8.174 s，Linux 环境）。新增真实工具链的跨平台缓存集成入口，本机 1 项完整序列通过（1.399 s），已加入 Windows CI。修正八路在模拟不支持该参数的条件下完成 0～10 s BDF 求解，1,790,486 个 binary64 值与已接受参考逐位一致，随后完整缓存命中成功；[修复摘要](../other/assets/2026-09-12/native-cache-windows-utime-fix.json) 保留证据，原始记录在 `test/windows-cache-utime-20260912/`。并将 [跨平台交付约定](../standard/跨平台交付约定.md) 写入仓库。本机没有 Windows 运行环境，本次不能声称 Windows 实机已验收；下文为此前优化阶段的记录。 + + 原生构建改为按实际引用选择功能模块，复用内容校验的目标文件，保留完整模型缓存。两池默认预算分别为 128 MiB 和 256 MiB，按最近使用时间清理；跨进程使用锁保护活跃构建/运行，超大单条目和在用内容允许暂时超额。旧根级缓存单独保留，新代码不继续追加旧格式。 冻结原版本同机对照，八路首次构建 3.7742→2.4875 s（-34.09%）、修改一个管长参数 3.9009→2.4838 s（-36.33%）、切四路 2.4908→1.1109 s（-55.40%）。完整模型命中检查增加约 0.13 s，已披露。八路原生 1,790,486 个 binary64 值逐位一致；网页三次中位可查看 3.7586 s、保存 3.8605 s、CSV 1.0348 s，四轮完整结果/下载/刷新一致。 diff --git a/native/README.md b/native/README.md index 0cccb7e..7a1ed4d 100644 --- a/native/README.md +++ b/native/README.md @@ -2,6 +2,8 @@ 后端 Python 校验 XML、检查连接、生成系统专用 C 并编译；独立 EXE 执行完整数值循环。原生运行不调用 Python。当前构建支持 Windows x64 和 Linux x86_64；Linux 使用静态链接的 SUNDIALS。 +平台相关修改遵循 [Windows 与 Linux 交付约定](../docs/standard/跨平台交付约定.md)。缓存时间戳更新按 `os.supports_follow_symlinks` 检测运行平台能力，Windows 不支持该可选操作时对已校验的普通缓存目录使用常规 `utime`。缓存集成回归可在两平台使用真实工具链执行：`python -m unittest tests.test_native_cache_storage tests.test_native_cache_platform -v`；设置 `SIMULATION_NATIVE_REQUIRE_TOOLCHAIN=1` 时工具链缺失将失败而不会跳过。 + ## 启用网页后端 当前默认使用 `native`。也可在启动后端的 PowerShell 中显式设置： diff --git a/tests/test_native_cache_platform.py b/tests/test_native_cache_platform.py new file mode 100644 index 0000000..d12ba0f --- /dev/null +++ b/tests/test_native_cache_platform.py @@ -0,0 +1,199 @@ +"""Real native cache integration on both Windows and Linux. + +The fixture uses the configured compiler and SUNDIALS installation directly. +CI sets SIMULATION_NATIVE_REQUIRE_TOOLCHAIN=1 so missing native dependencies +fail this suite instead of silently skipping platform acceptance. +""" +from __future__ import annotations + +from dataclasses import replace +from hashlib import sha256 +import json +import math +import os +from pathlib import Path +import subprocess +import tempfile +import threading +import unittest +from unittest.mock import patch +import xml.etree.ElementTree as ET + +from app.main import compile_system_xml_network +from app.simulation.backends import simulation_config +from app.simulation.config import SolverActivityTracker +from app.simulation.native_codegen import build as native_build +from app.simulation.native_codegen.cache_storage import prune_cache +from app.simulation.native_codegen.compiler import compile_native_program +from app.simulation.native_codegen.runner import execute_native +from app.system_xml import validate_system_xml_document + + +ROOT = Path(__file__).resolve().parents[1] +FIXTURE = ROOT / "tests/fixtures/native-skill-test.xml" + + +class NativeCachePlatformTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + try: + cls.compiler, cls.sundials, cls.compiler_version = native_build.toolchain() + except (OSError, RuntimeError, subprocess.SubprocessError) as exc: + message = f"Native cache platform test requires a working configured toolchain: {exc}" + if os.environ.get("SIMULATION_NATIVE_REQUIRE_TOOLCHAIN") == "1": + raise RuntimeError(message) from exc + raise unittest.SkipTest(message) from exc + + def setUp(self): + self.temporary = tempfile.TemporaryDirectory(prefix="native-platform-") + self.addCleanup(self.temporary.cleanup) + # Exercise argument handling with spaces without assuming shell quoting. + self.root = Path(self.temporary.name) / "cache integration" + self.root.mkdir() + self.cache = self.root / "cache" + self.real_run = subprocess.run + self._builds = [] + self.addCleanup(self._close_builds) + self.document = self._document(FIXTURE.read_bytes()) + self.program = compile_native_program(compile_system_xml_network(self.document)) + self.config = replace(simulation_config(self.document.simulation), + method="RK45", t_stop=0.1) + + def _close_builds(self): + for build in reversed(self._builds): + build.close() + + def _document(self, xml): + report = validate_system_xml_document(xml) + self.assertTrue(report.valid, report.as_dict()) + self.assertIsNotNone(report.document) + return report.document + + def _build(self, program): + commands = [] + lock = threading.Lock() + + def record(command, *args, **kwargs): + with lock: + commands.append(tuple(map(str, command))) + return self.real_run(command, *args, **kwargs) + + with patch.object(native_build.subprocess, "run", side_effect=record): + build = native_build.build_native(program, cache_dir=self.cache) + self._builds.append(build) + return build, commands + + @staticmethod + def _compile_count(commands): + return sum("-c" in command for command in commands) + + @staticmethod + def _link_count(commands): + return sum("-o" in command and "-c" not in command and "-E" not in command + for command in commands) + + def _execute(self, build, name, **kwargs): + payload = execute_native(build, self.config, 0.02, + run_dir=self.root / name, timeout=30, **kwargs) + self.assertTrue(payload["success"], payload.get("message")) + self.assertEqual(payload["simulatedUntil"], self.config.t_stop) + self.assertEqual(payload["series"]["time"][-1], self.config.t_stop) + self.assertEqual(set(payload["series"]), {"time", *(v.key for v in self.program.variables)}) + self.assertTrue(all(math.isfinite(value) + for series in payload["series"].values() for value in series)) + return payload + + def _assert_windows_package(self, build): + if os.name != "nt": + return + self.assertEqual(build.executable.name, "model.exe") + names = [f"sundials_{name}.dll" for name in native_build.LIBRARIES] + if (self.sundials / "bin/vcruntime140.dll").is_file(): + names.append("vcruntime140.dll") + for name in names: + with self.subTest(dll=name): + original = self.sundials / "bin" / name + packaged = build.executable.parent / name + self.assertTrue(packaged.is_file()) + digest = sha256(packaged.read_bytes()).hexdigest() + self.assertEqual(digest, sha256(original.read_bytes()).hexdigest()) + self.assertEqual(digest, build.manifest["artifacts"][name]) + self.assertEqual(digest, build.manifest["buildIdentity"]["dependencies"][name]) + # The EXE must load its copied DLLs without the compiler/Conda directories. + system32 = Path(os.environ.get("SystemRoot", r"C:\Windows")) / "System32" + completed = self.real_run( + [str(build.executable), "--init"], cwd=self.root, + env={**os.environ, "PATH": str(system32)}, + capture_output=True, text=True, check=True, timeout=15, + ) + self.assertEqual(len(json.loads(completed.stdout)), len(self.program.state_keys)) + + def test_real_build_execution_reuse_and_in_use_eviction(self): + self.assertFalse(self.cache.exists()) + cold, cold_commands = self._build(self.program) + self.assertFalse(cold.cache_hit) + self.assertGreater(self._compile_count(cold_commands), 1) + self.assertEqual(self._link_count(cold_commands), 1) + self.assertEqual((cold.executable.parent / "model.c").read_bytes(), self.program.source.encode()) + self.assertEqual((cold.executable.parent / "model.h").read_bytes(), self.program.header.encode()) + self._assert_windows_package(cold) + cold_result = self._execute(cold, "cold run") + + warm, warm_commands = self._build(self.program) + self.assertTrue(warm.cache_hit) + self.assertEqual(cold.manifest["buildKey"], warm.manifest["buildKey"]) + self.assertEqual(self._compile_count(warm_commands), 0) + self.assertEqual(self._link_count(warm_commands), 0) + self.assertEqual(warm.details["objectCompilations"], 0) + self.assertEqual(warm.details["linkSeconds"], 0) + warm_result = self._execute(warm, "warm run") + self.assertEqual(cold_result["series"], warm_result["series"]) + self.assertEqual(cold_result["finalState"], warm_result["finalState"]) + + xml = ET.fromstring(FIXTURE.read_bytes()) + pressure = xml.find("./Components/Component[@id='amesim_pnch023_1']/Parameter[@name='p0']") + self.assertIsNotNone(pressure) + pressure.set("value", "16000000") + modified = self._document(ET.tostring(xml, encoding="utf-8")) + changed_program = compile_native_program(compile_system_xml_network(modified)) + self.assertEqual(changed_program.header, self.program.header) + changed, changed_commands = self._build(changed_program) + self.assertFalse(changed.cache_hit) + self.assertNotEqual(changed.manifest["buildKey"], cold.manifest["buildKey"]) + self.assertEqual(self._compile_count(changed_commands), 1) + self.assertEqual(self._link_count(changed_commands), 1) + self.assertEqual(changed.details["objectCompilations"], 1) + self.assertGreater(changed.details["objectCacheHits"], 0) + self._assert_windows_package(changed) + + test = self + sweeps = [] + + class PruningTracker(SolverActivityTracker): + def start_integration(self, time): + super().start_integration(time) + # execute_native calls this after starting the real C child, + # before reading its result. Both model entries are still in use. + report = prune_cache(test.cache, model_limit_bytes=0) + sweeps.append(report) + test.assertGreaterEqual(report["models"]["skippedInUse"], 2) + test.assertTrue(cold.executable.is_file()) + test.assertTrue(changed.executable.is_file()) + + changed_result = self._execute(changed, "changed run", activity_tracker=PruningTracker()) + self.assertEqual(len(sweeps), 1) + column = "amesim_pnch023_1.p" + self.assertNotEqual(changed_result["series"][column][0], cold_result["series"][column][0]) + self.assertEqual(changed_result["buildKey"], changed.manifest["buildKey"]) + + # Once use ends, a zero budget may evict the older model while retaining + # the final oversized entry. Release every shared handle explicitly. + self._close_builds() + report = prune_cache(self.cache, model_limit_bytes=0) + self.assertEqual(report["models"]["removedEntries"], 1) + self.assertFalse(cold.executable.exists()) + self.assertTrue(changed.executable.is_file()) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_native_cache_storage.py b/tests/test_native_cache_storage.py index b77c0df..8744d42 100644 --- a/tests/test_native_cache_storage.py +++ b/tests/test_native_cache_storage.py @@ -6,6 +6,7 @@ import json import os from pathlib import Path import subprocess +from types import SimpleNamespace import sys import tempfile import unittest @@ -108,6 +109,63 @@ with acquire_cache_lease(Path(sys.argv[1]), 'models', sys.argv[2], exclusive=Tru gc.collect() self.assertIsNotNone(self.lease(3, exclusive=True, blocking=False)) + def test_touch_uses_available_utime_operation_and_preserves_lru(self): + real_utime = os.utime + for supported in (False, True): + with self.subTest(follow_symlinks_supported=supported): + self.cache = self.folder / f"cache-capability-{supported}" + touched = self.entry(0, 60, 1) + old = self.entry(1, 60, 2) + artifact_bytes = (touched / "artifact").read_bytes() + before = touched.stat().st_mtime_ns + calls = [] + + def platform_utime(path, times=None, **kwargs): + calls.append(kwargs) + if not supported and kwargs.get("follow_symlinks") is False: + raise NotImplementedError("utime: follow_symlinks unavailable on this platform") + # Both capability branches can run on either host. The + # real timestamp change is on a validated ordinary folder. + return real_utime(path, times) + + with patch.object(storage.os, "utime", platform_utime), patch.object( + storage.os, "supports_follow_symlinks", {platform_utime} if supported else set(), + ): + with self.lease(0): + storage.touch_cache_entry(self.cache, "models", key(0)) + self.assertEqual(calls, [{"follow_symlinks": False}] if supported else [{}]) + self.assertGreater(touched.stat().st_mtime_ns, before) + self.assertEqual((touched / "artifact").read_bytes(), artifact_bytes) + storage.prune_cache(self.cache, model_limit_bytes=60) + self.assertTrue(touched.exists()) + self.assertFalse(old.exists()) + + def test_touch_rejects_windows_reparse_directory_before_timestamp_update(self): + entry = self.entry(0, 60, 1) + real_lstat = Path.lstat + attributes = SimpleNamespace( + st_mode=entry.lstat().st_mode, st_file_attributes=0x400, + ) + + def reparse_lstat(path, *args, **kwargs): + return attributes if path == entry else real_lstat(path, *args, **kwargs) + + with patch.object(Path, "lstat", reparse_lstat), patch.object( + storage.os, "supports_follow_symlinks", set(), + ), patch.object(storage.os, "utime") as update: + with self.assertRaisesRegex(RuntimeError, "real directory"): + storage.touch_cache_entry(self.cache, "models", key(0)) + update.assert_not_called() + + def test_touch_does_not_hide_timestamp_permission_errors(self): + self.entry(0, 60, 1) + with patch.object(storage.os, "supports_follow_symlinks", set()), patch.object( + storage.os, "utime", side_effect=PermissionError("timestamp denied"), + ) as update: + with self.assertRaisesRegex(PermissionError, "timestamp denied"): + storage.touch_cache_entry(self.cache, "models", key(0)) + update.assert_called_once() + def test_lru_removes_oldest_until_separate_budgets_are_met(self): oldest = self.entry(0, 60, 1) middle = self.entry(1, 60, 2)