缓存功能windows平台适配

This commit is contained in:
lujingze committed 2026-09-12 05:40:18 +00:00
1 parent 151e6e4b97
commit 22579e51c9
8 files changed
+333 -1

No files matched your search

+15
View File
@@ -86,6 +86,15 @@ jobs:
"SIMULATION_NATIVE_CC=$env:CONDA_PREFIX\Library\mingw-w64\bin\gcc.exe" >> $env:GITHUB_ENV
python -m pip install -r requirements-test.txt
python -m pip check
- name: Verify native cache on the Windows runtime
env:
SIMULATION_NATIVE_REQUIRE_TOOLCHAIN: "1"
run: |
New-Item -ItemType Directory -Force test/ci-native-cache | Out-Null
python -W error::ResourceWarning -m unittest tests.test_native_cache_storage tests.test_native_cache_platform -v > test/ci-native-cache/regression.log 2>&1
$cacheRegressionExit = $LASTEXITCODE
Get-Content test/ci-native-cache/regression.log
if ($cacheRegressionExit -ne 0) { exit $cacheRegressionExit }
- name: Run catalog, numerical, API and schema regression
run: |
python -c "from app.simulation.native_codegen.build import toolchain; print(toolchain())"
@@ -100,3 +109,9 @@ jobs:
name: native-skill-regression
path: test/ci-skill/summary.json
if-no-files-found: warn
- if: always()
uses: actions/upload-artifact@v4
with:
name: native-windows-cache-regression
path: test/ci-native-cache/regression.log
if-no-files-found: warn
@@ -218,7 +218,13 @@ def touch_cache_entry(cache_dir: Path, kind: str, key: str) -> None:
entry = cache / kind / key
_require_directory(entry)
# Directory mtime is separate from the checksummed artifact/manifest bytes.
os.utime(entry, None, follow_symlinks=False)
# Windows Python may expose utime without the no-follow operation. The
# checks above reject symlinks and reparse points before either call, and
# the caller's use lease protects the entry from application eviction.
if os.utime in os.supports_follow_symlinks:
os.utime(entry, None, follow_symlinks=False)
else:
os.utime(entry, None)
@dataclass(frozen=True)
@@ -0,0 +1,29 @@
{
"host": "Linux-6.8.0-111-generic-x86_64-with-glibc2.39",
"windowsRuntimeExecuted": false,
"case": "Corrected eight-branch 0-10 s with os.utime rejecting follow_symlinks=False, real Linux GCC/CVODE execution.",
"success": true,
"simulatedUntil": 10,
"method": "BDF",
"rtol": 1e-08,
"bitwiseValuesCompared": 1790486,
"allBitsEqual": true,
"coldCacheHit": false,
"subsequentCacheHit": true,
"timestampUpdates": 14,
"solveSeconds": 2.510730743408203,
"buildSeconds": 2.51858819834888,
"sourceSha256": "ace0843c8fe84b687c7dedce9eecae1ab2a9174a0f321f047d99089cfa15b301",
"tests": {
"buildAndStorage": {
"passed": 40,
"seconds": 8.174
},
"realToolchainIntegration": {
"passed": 1,
"seconds": 1.399,
"windowsDllBranchExecuted": false
},
"windowsCi": "Configured; not executed from this Linux session."
}
}
+20
View File
@@ -0,0 +1,20 @@
# Windows 与 Linux 功能交付约定
2026-09-12 用户明确要求:后续功能补全同时注意 Windows 平台适配。
Windows x64 与 Linux x86_64 都是当前应用的使用平台。涉及文件系统、缓存、编译器、进程、动态库、环境安装或启动脚本的功能修改,应同时检查两侧的实现和测试入口。
- 文件系统可选能力应按运行环境检测,例如 `os.supports_follow_symlinks`,不能因为 Linux 接受某个参数就假设 Windows 也接受。缓存目录仍需验证普通目录、符号链接与 Windows reparse point;权限和真实 I/O 错误不能当成兼容问题随意吞掉。
- 编译与运行应覆盖 Windows 的 `.exe`、导入库/DLL、工具链路径、带空格路径和换行行为。进程锁、并发清理、异常退出应有可在 Windows 执行的测试。
- 平台专用的小型测试夹具可以明确限定平台,但不能代替跨平台集成测试。Windows CI 中必需的工具链缺失应失败,不能以整组跳过得到成功结论。
- 交付记录分别列出兼容代码、模拟测试、实际 Windows 测试的状态。只有实际 Windows 运行通过,才能声称 Windows 验收通过。缺少 Windows 环境时继续完成可做的修复和测试准备,并明确实机验证仍待完成。
- 本地虚拟环境、编译器、便携运行时和构建缓存保留在 Git 忽略目录,不随源码提交。
当前缓存检查入口:
```powershell
$env:SIMULATION_NATIVE_REQUIRE_TOOLCHAIN = '1'
.\.venv-win\Scripts\python.exe -m unittest tests.test_native_cache_storage tests.test_native_cache_platform -v
```
该入口使用现有 `SIMULATION_NATIVE_CC` / `SUNDIALS_ROOT` 或构建器的默认探测;CI 配置在 `.github/workflows/solver-regression.yml` 的 `native-windows` 作业。
@@ -1,5 +1,8 @@
# 2026-09-12:按需编译与分层缓存
后续 Windows 修复:用户实际运行报 `utime: follow_symlinks unavailable on this platform`。根因是缓存最近使用时间更新无条件传入可选参数,与数值求解无关。按 [Python 平台能力接口](https://docs.python.org/3.12/library/os.html#os.supports_follow_symlinks) 改为能力检测后选择调用,保留目录/reparse 校验和真实权限错误。新增回归在修复前复现同一异常;修复后构建/存储 40 项通过(8.174 s,Linux 环境)。新增真实工具链的跨平台缓存集成入口,本机 1 项完整序列通过(1.399 s),已加入 Windows CI。修正八路在模拟不支持该参数的条件下完成 0~10 s BDF 求解,1,790,486 个 binary64 值与已接受参考逐位一致,随后完整缓存命中成功;[修复摘要](../other/assets/2026-09-12/native-cache-windows-utime-fix.json) 保留证据,原始记录在 `test/windows-cache-utime-20260912/`。并将 [跨平台交付约定](../standard/跨平台交付约定.md) 写入仓库。本机没有 Windows 运行环境,本次不能声称 Windows 实机已验收;下文为此前优化阶段的记录。
原生构建改为按实际引用选择功能模块,复用内容校验的目标文件,保留完整模型缓存。两池默认预算分别为 128 MiB 和 256 MiB,按最近使用时间清理;跨进程使用锁保护活跃构建/运行,超大单条目和在用内容允许暂时超额。旧根级缓存单独保留,新代码不继续追加旧格式。
冻结原版本同机对照,八路首次构建 3.7742→2.4875 s(-34.09%)、修改一个管长参数 3.9009→2.4838 s(-36.33%)、切四路 2.4908→1.1109 s(-55.40%)。完整模型命中检查增加约 0.13 s,已披露。八路原生 1,790,486 个 binary64 值逐位一致;网页三次中位可查看 3.7586 s、保存 3.8605 s、CSV 1.0348 s,四轮完整结果/下载/刷新一致。
+2
View File
@@ -2,6 +2,8 @@
后端 Python 校验 XML、检查连接、生成系统专用 C 并编译;独立 EXE 执行完整数值循环。原生运行不调用 Python。当前构建支持 Windows x64 和 Linux x86_64;Linux 使用静态链接的 SUNDIALS。
平台相关修改遵循 [Windows 与 Linux 交付约定](../docs/standard/跨平台交付约定.md)。缓存时间戳更新按 `os.supports_follow_symlinks` 检测运行平台能力,Windows 不支持该可选操作时对已校验的普通缓存目录使用常规 `utime`。缓存集成回归可在两平台使用真实工具链执行:`python -m unittest tests.test_native_cache_storage tests.test_native_cache_platform -v`;设置 `SIMULATION_NATIVE_REQUIRE_TOOLCHAIN=1` 时工具链缺失将失败而不会跳过。
## 启用网页后端
当前默认使用 `native`。也可在启动后端的 PowerShell 中显式设置:
+199
View File
@@ -0,0 +1,199 @@
"""Real native cache integration on both Windows and Linux.
The fixture uses the configured compiler and SUNDIALS installation directly.
CI sets SIMULATION_NATIVE_REQUIRE_TOOLCHAIN=1 so missing native dependencies
fail this suite instead of silently skipping platform acceptance.
"""
from __future__ import annotations
from dataclasses import replace
from hashlib import sha256
import json
import math
import os
from pathlib import Path
import subprocess
import tempfile
import threading
import unittest
from unittest.mock import patch
import xml.etree.ElementTree as ET
from app.main import compile_system_xml_network
from app.simulation.backends import simulation_config
from app.simulation.config import SolverActivityTracker
from app.simulation.native_codegen import build as native_build
from app.simulation.native_codegen.cache_storage import prune_cache
from app.simulation.native_codegen.compiler import compile_native_program
from app.simulation.native_codegen.runner import execute_native
from app.system_xml import validate_system_xml_document
ROOT = Path(__file__).resolve().parents[1]
FIXTURE = ROOT / "tests/fixtures/native-skill-test.xml"
class NativeCachePlatformTests(unittest.TestCase):
@classmethod
def setUpClass(cls):
try:
cls.compiler, cls.sundials, cls.compiler_version = native_build.toolchain()
except (OSError, RuntimeError, subprocess.SubprocessError) as exc:
message = f"Native cache platform test requires a working configured toolchain: {exc}"
if os.environ.get("SIMULATION_NATIVE_REQUIRE_TOOLCHAIN") == "1":
raise RuntimeError(message) from exc
raise unittest.SkipTest(message) from exc
def setUp(self):
self.temporary = tempfile.TemporaryDirectory(prefix="native-platform-")
self.addCleanup(self.temporary.cleanup)
# Exercise argument handling with spaces without assuming shell quoting.
self.root = Path(self.temporary.name) / "cache integration"
self.root.mkdir()
self.cache = self.root / "cache"
self.real_run = subprocess.run
self._builds = []
self.addCleanup(self._close_builds)
self.document = self._document(FIXTURE.read_bytes())
self.program = compile_native_program(compile_system_xml_network(self.document))
self.config = replace(simulation_config(self.document.simulation),
method="RK45", t_stop=0.1)
def _close_builds(self):
for build in reversed(self._builds):
build.close()
def _document(self, xml):
report = validate_system_xml_document(xml)
self.assertTrue(report.valid, report.as_dict())
self.assertIsNotNone(report.document)
return report.document
def _build(self, program):
commands = []
lock = threading.Lock()
def record(command, *args, **kwargs):
with lock:
commands.append(tuple(map(str, command)))
return self.real_run(command, *args, **kwargs)
with patch.object(native_build.subprocess, "run", side_effect=record):
build = native_build.build_native(program, cache_dir=self.cache)
self._builds.append(build)
return build, commands
@staticmethod
def _compile_count(commands):
return sum("-c" in command for command in commands)
@staticmethod
def _link_count(commands):
return sum("-o" in command and "-c" not in command and "-E" not in command
for command in commands)
def _execute(self, build, name, **kwargs):
payload = execute_native(build, self.config, 0.02,
run_dir=self.root / name, timeout=30, **kwargs)
self.assertTrue(payload["success"], payload.get("message"))
self.assertEqual(payload["simulatedUntil"], self.config.t_stop)
self.assertEqual(payload["series"]["time"][-1], self.config.t_stop)
self.assertEqual(set(payload["series"]), {"time", *(v.key for v in self.program.variables)})
self.assertTrue(all(math.isfinite(value)
for series in payload["series"].values() for value in series))
return payload
def _assert_windows_package(self, build):
if os.name != "nt":
return
self.assertEqual(build.executable.name, "model.exe")
names = [f"sundials_{name}.dll" for name in native_build.LIBRARIES]
if (self.sundials / "bin/vcruntime140.dll").is_file():
names.append("vcruntime140.dll")
for name in names:
with self.subTest(dll=name):
original = self.sundials / "bin" / name
packaged = build.executable.parent / name
self.assertTrue(packaged.is_file())
digest = sha256(packaged.read_bytes()).hexdigest()
self.assertEqual(digest, sha256(original.read_bytes()).hexdigest())
self.assertEqual(digest, build.manifest["artifacts"][name])
self.assertEqual(digest, build.manifest["buildIdentity"]["dependencies"][name])
# The EXE must load its copied DLLs without the compiler/Conda directories.
system32 = Path(os.environ.get("SystemRoot", r"C:\Windows")) / "System32"
completed = self.real_run(
[str(build.executable), "--init"], cwd=self.root,
env={**os.environ, "PATH": str(system32)},
capture_output=True, text=True, check=True, timeout=15,
)
self.assertEqual(len(json.loads(completed.stdout)), len(self.program.state_keys))
def test_real_build_execution_reuse_and_in_use_eviction(self):
self.assertFalse(self.cache.exists())
cold, cold_commands = self._build(self.program)
self.assertFalse(cold.cache_hit)
self.assertGreater(self._compile_count(cold_commands), 1)
self.assertEqual(self._link_count(cold_commands), 1)
self.assertEqual((cold.executable.parent / "model.c").read_bytes(), self.program.source.encode())
self.assertEqual((cold.executable.parent / "model.h").read_bytes(), self.program.header.encode())
self._assert_windows_package(cold)
cold_result = self._execute(cold, "cold run")
warm, warm_commands = self._build(self.program)
self.assertTrue(warm.cache_hit)
self.assertEqual(cold.manifest["buildKey"], warm.manifest["buildKey"])
self.assertEqual(self._compile_count(warm_commands), 0)
self.assertEqual(self._link_count(warm_commands), 0)
self.assertEqual(warm.details["objectCompilations"], 0)
self.assertEqual(warm.details["linkSeconds"], 0)
warm_result = self._execute(warm, "warm run")
self.assertEqual(cold_result["series"], warm_result["series"])
self.assertEqual(cold_result["finalState"], warm_result["finalState"])
xml = ET.fromstring(FIXTURE.read_bytes())
pressure = xml.find("./Components/Component[@id='amesim_pnch023_1']/Parameter[@name='p0']")
self.assertIsNotNone(pressure)
pressure.set("value", "16000000")
modified = self._document(ET.tostring(xml, encoding="utf-8"))
changed_program = compile_native_program(compile_system_xml_network(modified))
self.assertEqual(changed_program.header, self.program.header)
changed, changed_commands = self._build(changed_program)
self.assertFalse(changed.cache_hit)
self.assertNotEqual(changed.manifest["buildKey"], cold.manifest["buildKey"])
self.assertEqual(self._compile_count(changed_commands), 1)
self.assertEqual(self._link_count(changed_commands), 1)
self.assertEqual(changed.details["objectCompilations"], 1)
self.assertGreater(changed.details["objectCacheHits"], 0)
self._assert_windows_package(changed)
test = self
sweeps = []
class PruningTracker(SolverActivityTracker):
def start_integration(self, time):
super().start_integration(time)
# execute_native calls this after starting the real C child,
# before reading its result. Both model entries are still in use.
report = prune_cache(test.cache, model_limit_bytes=0)
sweeps.append(report)
test.assertGreaterEqual(report["models"]["skippedInUse"], 2)
test.assertTrue(cold.executable.is_file())
test.assertTrue(changed.executable.is_file())
changed_result = self._execute(changed, "changed run", activity_tracker=PruningTracker())
self.assertEqual(len(sweeps), 1)
column = "amesim_pnch023_1.p"
self.assertNotEqual(changed_result["series"][column][0], cold_result["series"][column][0])
self.assertEqual(changed_result["buildKey"], changed.manifest["buildKey"])
# Once use ends, a zero budget may evict the older model while retaining
# the final oversized entry. Release every shared handle explicitly.
self._close_builds()
report = prune_cache(self.cache, model_limit_bytes=0)
self.assertEqual(report["models"]["removedEntries"], 1)
self.assertFalse(cold.executable.exists())
self.assertTrue(changed.executable.is_file())
if __name__ == "__main__":
unittest.main()
+58
View File
@@ -6,6 +6,7 @@ import json
import os
from pathlib import Path
import subprocess
from types import SimpleNamespace
import sys
import tempfile
import unittest
@@ -108,6 +109,63 @@ with acquire_cache_lease(Path(sys.argv[1]), 'models', sys.argv[2], exclusive=Tru
gc.collect()
self.assertIsNotNone(self.lease(3, exclusive=True, blocking=False))
def test_touch_uses_available_utime_operation_and_preserves_lru(self):
real_utime = os.utime
for supported in (False, True):
with self.subTest(follow_symlinks_supported=supported):
self.cache = self.folder / f"cache-capability-{supported}"
touched = self.entry(0, 60, 1)
old = self.entry(1, 60, 2)
artifact_bytes = (touched / "artifact").read_bytes()
before = touched.stat().st_mtime_ns
calls = []
def platform_utime(path, times=None, **kwargs):
calls.append(kwargs)
if not supported and kwargs.get("follow_symlinks") is False:
raise NotImplementedError("utime: follow_symlinks unavailable on this platform")
# Both capability branches can run on either host. The
# real timestamp change is on a validated ordinary folder.
return real_utime(path, times)
with patch.object(storage.os, "utime", platform_utime), patch.object(
storage.os, "supports_follow_symlinks", {platform_utime} if supported else set(),
):
with self.lease(0):
storage.touch_cache_entry(self.cache, "models", key(0))
self.assertEqual(calls, [{"follow_symlinks": False}] if supported else [{}])
self.assertGreater(touched.stat().st_mtime_ns, before)
self.assertEqual((touched / "artifact").read_bytes(), artifact_bytes)
storage.prune_cache(self.cache, model_limit_bytes=60)
self.assertTrue(touched.exists())
self.assertFalse(old.exists())
def test_touch_rejects_windows_reparse_directory_before_timestamp_update(self):
entry = self.entry(0, 60, 1)
real_lstat = Path.lstat
attributes = SimpleNamespace(
st_mode=entry.lstat().st_mode, st_file_attributes=0x400,
)
def reparse_lstat(path, *args, **kwargs):
return attributes if path == entry else real_lstat(path, *args, **kwargs)
with patch.object(Path, "lstat", reparse_lstat), patch.object(
storage.os, "supports_follow_symlinks", set(),
), patch.object(storage.os, "utime") as update:
with self.assertRaisesRegex(RuntimeError, "real directory"):
storage.touch_cache_entry(self.cache, "models", key(0))
update.assert_not_called()
def test_touch_does_not_hide_timestamp_permission_errors(self):
self.entry(0, 60, 1)
with patch.object(storage.os, "supports_follow_symlinks", set()), patch.object(
storage.os, "utime", side_effect=PermissionError("timestamp denied"),
) as update:
with self.assertRaisesRegex(PermissionError, "timestamp denied"):
storage.touch_cache_entry(self.cache, "models", key(0))
update.assert_called_once()
def test_lru_removes_oldest_until_separate_budgets_are_met(self):
oldest = self.entry(0, 60, 1)
middle = self.entry(1, 60, 2)