缓存功能windows平台适配
This commit is contained in:
1 parent
151e6e4b97
commit
22579e51c9
8 files changed
+333
-1
No files matched your search
@@ -86,6 +86,15 @@ jobs:
|
||||
"SIMULATION_NATIVE_CC=$env:CONDA_PREFIX\Library\mingw-w64\bin\gcc.exe" >> $env:GITHUB_ENV
|
||||
python -m pip install -r requirements-test.txt
|
||||
python -m pip check
|
||||
- name: Verify native cache on the Windows runtime
|
||||
env:
|
||||
SIMULATION_NATIVE_REQUIRE_TOOLCHAIN: "1"
|
||||
run: |
|
||||
New-Item -ItemType Directory -Force test/ci-native-cache | Out-Null
|
||||
python -W error::ResourceWarning -m unittest tests.test_native_cache_storage tests.test_native_cache_platform -v > test/ci-native-cache/regression.log 2>&1
|
||||
$cacheRegressionExit = $LASTEXITCODE
|
||||
Get-Content test/ci-native-cache/regression.log
|
||||
if ($cacheRegressionExit -ne 0) { exit $cacheRegressionExit }
|
||||
- name: Run catalog, numerical, API and schema regression
|
||||
run: |
|
||||
python -c "from app.simulation.native_codegen.build import toolchain; print(toolchain())"
|
||||
@@ -100,3 +109,9 @@ jobs:
|
||||
name: native-skill-regression
|
||||
path: test/ci-skill/summary.json
|
||||
if-no-files-found: warn
|
||||
- if: always()
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: native-windows-cache-regression
|
||||
path: test/ci-native-cache/regression.log
|
||||
if-no-files-found: warn
|
||||
@@ -218,7 +218,13 @@ def touch_cache_entry(cache_dir: Path, kind: str, key: str) -> None:
|
||||
entry = cache / kind / key
|
||||
_require_directory(entry)
|
||||
# Directory mtime is separate from the checksummed artifact/manifest bytes.
|
||||
os.utime(entry, None, follow_symlinks=False)
|
||||
# Windows Python may expose utime without the no-follow operation. The
|
||||
# checks above reject symlinks and reparse points before either call, and
|
||||
# the caller's use lease protects the entry from application eviction.
|
||||
if os.utime in os.supports_follow_symlinks:
|
||||
os.utime(entry, None, follow_symlinks=False)
|
||||
else:
|
||||
os.utime(entry, None)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
|
||||
@@ -0,0 +1,29 @@
|
||||
{
|
||||
"host": "Linux-6.8.0-111-generic-x86_64-with-glibc2.39",
|
||||
"windowsRuntimeExecuted": false,
|
||||
"case": "Corrected eight-branch 0-10 s with os.utime rejecting follow_symlinks=False, real Linux GCC/CVODE execution.",
|
||||
"success": true,
|
||||
"simulatedUntil": 10,
|
||||
"method": "BDF",
|
||||
"rtol": 1e-08,
|
||||
"bitwiseValuesCompared": 1790486,
|
||||
"allBitsEqual": true,
|
||||
"coldCacheHit": false,
|
||||
"subsequentCacheHit": true,
|
||||
"timestampUpdates": 14,
|
||||
"solveSeconds": 2.510730743408203,
|
||||
"buildSeconds": 2.51858819834888,
|
||||
"sourceSha256": "ace0843c8fe84b687c7dedce9eecae1ab2a9174a0f321f047d99089cfa15b301",
|
||||
"tests": {
|
||||
"buildAndStorage": {
|
||||
"passed": 40,
|
||||
"seconds": 8.174
|
||||
},
|
||||
"realToolchainIntegration": {
|
||||
"passed": 1,
|
||||
"seconds": 1.399,
|
||||
"windowsDllBranchExecuted": false
|
||||
},
|
||||
"windowsCi": "Configured; not executed from this Linux session."
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
# Windows 与 Linux 功能交付约定
|
||||
|
||||
2026-09-12 用户明确要求:后续功能补全同时注意 Windows 平台适配。
|
||||
|
||||
Windows x64 与 Linux x86_64 都是当前应用的使用平台。涉及文件系统、缓存、编译器、进程、动态库、环境安装或启动脚本的功能修改,应同时检查两侧的实现和测试入口。
|
||||
|
||||
- 文件系统可选能力应按运行环境检测,例如 `os.supports_follow_symlinks`,不能因为 Linux 接受某个参数就假设 Windows 也接受。缓存目录仍需验证普通目录、符号链接与 Windows reparse point;权限和真实 I/O 错误不能当成兼容问题随意吞掉。
|
||||
- 编译与运行应覆盖 Windows 的 `.exe`、导入库/DLL、工具链路径、带空格路径和换行行为。进程锁、并发清理、异常退出应有可在 Windows 执行的测试。
|
||||
- 平台专用的小型测试夹具可以明确限定平台,但不能代替跨平台集成测试。Windows CI 中必需的工具链缺失应失败,不能以整组跳过得到成功结论。
|
||||
- 交付记录分别列出兼容代码、模拟测试、实际 Windows 测试的状态。只有实际 Windows 运行通过,才能声称 Windows 验收通过。缺少 Windows 环境时继续完成可做的修复和测试准备,并明确实机验证仍待完成。
|
||||
- 本地虚拟环境、编译器、便携运行时和构建缓存保留在 Git 忽略目录,不随源码提交。
|
||||
|
||||
当前缓存检查入口:
|
||||
|
||||
```powershell
|
||||
$env:SIMULATION_NATIVE_REQUIRE_TOOLCHAIN = '1'
|
||||
.\.venv-win\Scripts\python.exe -m unittest tests.test_native_cache_storage tests.test_native_cache_platform -v
|
||||
```
|
||||
|
||||
该入口使用现有 `SIMULATION_NATIVE_CC` / `SUNDIALS_ROOT` 或构建器的默认探测;CI 配置在 `.github/workflows/solver-regression.yml` 的 `native-windows` 作业。
|
||||
@@ -1,5 +1,8 @@
|
||||
# 2026-09-12:按需编译与分层缓存
|
||||
|
||||
后续 Windows 修复:用户实际运行报 `utime: follow_symlinks unavailable on this platform`。根因是缓存最近使用时间更新无条件传入可选参数,与数值求解无关。按 [Python 平台能力接口](https://docs.python.org/3.12/library/os.html#os.supports_follow_symlinks) 改为能力检测后选择调用,保留目录/reparse 校验和真实权限错误。新增回归在修复前复现同一异常;修复后构建/存储 40 项通过(8.174 s,Linux 环境)。新增真实工具链的跨平台缓存集成入口,本机 1 项完整序列通过(1.399 s),已加入 Windows CI。修正八路在模拟不支持该参数的条件下完成 0~10 s BDF 求解,1,790,486 个 binary64 值与已接受参考逐位一致,随后完整缓存命中成功;[修复摘要](../other/assets/2026-09-12/native-cache-windows-utime-fix.json) 保留证据,原始记录在 `test/windows-cache-utime-20260912/`。并将 [跨平台交付约定](../standard/跨平台交付约定.md) 写入仓库。本机没有 Windows 运行环境,本次不能声称 Windows 实机已验收;下文为此前优化阶段的记录。
|
||||
|
||||
|
||||
原生构建改为按实际引用选择功能模块,复用内容校验的目标文件,保留完整模型缓存。两池默认预算分别为 128 MiB 和 256 MiB,按最近使用时间清理;跨进程使用锁保护活跃构建/运行,超大单条目和在用内容允许暂时超额。旧根级缓存单独保留,新代码不继续追加旧格式。
|
||||
|
||||
冻结原版本同机对照,八路首次构建 3.7742→2.4875 s(-34.09%)、修改一个管长参数 3.9009→2.4838 s(-36.33%)、切四路 2.4908→1.1109 s(-55.40%)。完整模型命中检查增加约 0.13 s,已披露。八路原生 1,790,486 个 binary64 值逐位一致;网页三次中位可查看 3.7586 s、保存 3.8605 s、CSV 1.0348 s,四轮完整结果/下载/刷新一致。
|
||||
|
||||
@@ -2,6 +2,8 @@
|
||||
|
||||
后端 Python 校验 XML、检查连接、生成系统专用 C 并编译;独立 EXE 执行完整数值循环。原生运行不调用 Python。当前构建支持 Windows x64 和 Linux x86_64;Linux 使用静态链接的 SUNDIALS。
|
||||
|
||||
平台相关修改遵循 [Windows 与 Linux 交付约定](../docs/standard/跨平台交付约定.md)。缓存时间戳更新按 `os.supports_follow_symlinks` 检测运行平台能力,Windows 不支持该可选操作时对已校验的普通缓存目录使用常规 `utime`。缓存集成回归可在两平台使用真实工具链执行:`python -m unittest tests.test_native_cache_storage tests.test_native_cache_platform -v`;设置 `SIMULATION_NATIVE_REQUIRE_TOOLCHAIN=1` 时工具链缺失将失败而不会跳过。
|
||||
|
||||
## 启用网页后端
|
||||
|
||||
当前默认使用 `native`。也可在启动后端的 PowerShell 中显式设置:
|
||||
|
||||
@@ -0,0 +1,199 @@
|
||||
"""Real native cache integration on both Windows and Linux.
|
||||
|
||||
The fixture uses the configured compiler and SUNDIALS installation directly.
|
||||
CI sets SIMULATION_NATIVE_REQUIRE_TOOLCHAIN=1 so missing native dependencies
|
||||
fail this suite instead of silently skipping platform acceptance.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import replace
|
||||
from hashlib import sha256
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import tempfile
|
||||
import threading
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
import xml.etree.ElementTree as ET
|
||||
|
||||
from app.main import compile_system_xml_network
|
||||
from app.simulation.backends import simulation_config
|
||||
from app.simulation.config import SolverActivityTracker
|
||||
from app.simulation.native_codegen import build as native_build
|
||||
from app.simulation.native_codegen.cache_storage import prune_cache
|
||||
from app.simulation.native_codegen.compiler import compile_native_program
|
||||
from app.simulation.native_codegen.runner import execute_native
|
||||
from app.system_xml import validate_system_xml_document
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
FIXTURE = ROOT / "tests/fixtures/native-skill-test.xml"
|
||||
|
||||
|
||||
class NativeCachePlatformTests(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
try:
|
||||
cls.compiler, cls.sundials, cls.compiler_version = native_build.toolchain()
|
||||
except (OSError, RuntimeError, subprocess.SubprocessError) as exc:
|
||||
message = f"Native cache platform test requires a working configured toolchain: {exc}"
|
||||
if os.environ.get("SIMULATION_NATIVE_REQUIRE_TOOLCHAIN") == "1":
|
||||
raise RuntimeError(message) from exc
|
||||
raise unittest.SkipTest(message) from exc
|
||||
|
||||
def setUp(self):
|
||||
self.temporary = tempfile.TemporaryDirectory(prefix="native-platform-")
|
||||
self.addCleanup(self.temporary.cleanup)
|
||||
# Exercise argument handling with spaces without assuming shell quoting.
|
||||
self.root = Path(self.temporary.name) / "cache integration"
|
||||
self.root.mkdir()
|
||||
self.cache = self.root / "cache"
|
||||
self.real_run = subprocess.run
|
||||
self._builds = []
|
||||
self.addCleanup(self._close_builds)
|
||||
self.document = self._document(FIXTURE.read_bytes())
|
||||
self.program = compile_native_program(compile_system_xml_network(self.document))
|
||||
self.config = replace(simulation_config(self.document.simulation),
|
||||
method="RK45", t_stop=0.1)
|
||||
|
||||
def _close_builds(self):
|
||||
for build in reversed(self._builds):
|
||||
build.close()
|
||||
|
||||
def _document(self, xml):
|
||||
report = validate_system_xml_document(xml)
|
||||
self.assertTrue(report.valid, report.as_dict())
|
||||
self.assertIsNotNone(report.document)
|
||||
return report.document
|
||||
|
||||
def _build(self, program):
|
||||
commands = []
|
||||
lock = threading.Lock()
|
||||
|
||||
def record(command, *args, **kwargs):
|
||||
with lock:
|
||||
commands.append(tuple(map(str, command)))
|
||||
return self.real_run(command, *args, **kwargs)
|
||||
|
||||
with patch.object(native_build.subprocess, "run", side_effect=record):
|
||||
build = native_build.build_native(program, cache_dir=self.cache)
|
||||
self._builds.append(build)
|
||||
return build, commands
|
||||
|
||||
@staticmethod
|
||||
def _compile_count(commands):
|
||||
return sum("-c" in command for command in commands)
|
||||
|
||||
@staticmethod
|
||||
def _link_count(commands):
|
||||
return sum("-o" in command and "-c" not in command and "-E" not in command
|
||||
for command in commands)
|
||||
|
||||
def _execute(self, build, name, **kwargs):
|
||||
payload = execute_native(build, self.config, 0.02,
|
||||
run_dir=self.root / name, timeout=30, **kwargs)
|
||||
self.assertTrue(payload["success"], payload.get("message"))
|
||||
self.assertEqual(payload["simulatedUntil"], self.config.t_stop)
|
||||
self.assertEqual(payload["series"]["time"][-1], self.config.t_stop)
|
||||
self.assertEqual(set(payload["series"]), {"time", *(v.key for v in self.program.variables)})
|
||||
self.assertTrue(all(math.isfinite(value)
|
||||
for series in payload["series"].values() for value in series))
|
||||
return payload
|
||||
|
||||
def _assert_windows_package(self, build):
|
||||
if os.name != "nt":
|
||||
return
|
||||
self.assertEqual(build.executable.name, "model.exe")
|
||||
names = [f"sundials_{name}.dll" for name in native_build.LIBRARIES]
|
||||
if (self.sundials / "bin/vcruntime140.dll").is_file():
|
||||
names.append("vcruntime140.dll")
|
||||
for name in names:
|
||||
with self.subTest(dll=name):
|
||||
original = self.sundials / "bin" / name
|
||||
packaged = build.executable.parent / name
|
||||
self.assertTrue(packaged.is_file())
|
||||
digest = sha256(packaged.read_bytes()).hexdigest()
|
||||
self.assertEqual(digest, sha256(original.read_bytes()).hexdigest())
|
||||
self.assertEqual(digest, build.manifest["artifacts"][name])
|
||||
self.assertEqual(digest, build.manifest["buildIdentity"]["dependencies"][name])
|
||||
# The EXE must load its copied DLLs without the compiler/Conda directories.
|
||||
system32 = Path(os.environ.get("SystemRoot", r"C:\Windows")) / "System32"
|
||||
completed = self.real_run(
|
||||
[str(build.executable), "--init"], cwd=self.root,
|
||||
env={**os.environ, "PATH": str(system32)},
|
||||
capture_output=True, text=True, check=True, timeout=15,
|
||||
)
|
||||
self.assertEqual(len(json.loads(completed.stdout)), len(self.program.state_keys))
|
||||
|
||||
def test_real_build_execution_reuse_and_in_use_eviction(self):
|
||||
self.assertFalse(self.cache.exists())
|
||||
cold, cold_commands = self._build(self.program)
|
||||
self.assertFalse(cold.cache_hit)
|
||||
self.assertGreater(self._compile_count(cold_commands), 1)
|
||||
self.assertEqual(self._link_count(cold_commands), 1)
|
||||
self.assertEqual((cold.executable.parent / "model.c").read_bytes(), self.program.source.encode())
|
||||
self.assertEqual((cold.executable.parent / "model.h").read_bytes(), self.program.header.encode())
|
||||
self._assert_windows_package(cold)
|
||||
cold_result = self._execute(cold, "cold run")
|
||||
|
||||
warm, warm_commands = self._build(self.program)
|
||||
self.assertTrue(warm.cache_hit)
|
||||
self.assertEqual(cold.manifest["buildKey"], warm.manifest["buildKey"])
|
||||
self.assertEqual(self._compile_count(warm_commands), 0)
|
||||
self.assertEqual(self._link_count(warm_commands), 0)
|
||||
self.assertEqual(warm.details["objectCompilations"], 0)
|
||||
self.assertEqual(warm.details["linkSeconds"], 0)
|
||||
warm_result = self._execute(warm, "warm run")
|
||||
self.assertEqual(cold_result["series"], warm_result["series"])
|
||||
self.assertEqual(cold_result["finalState"], warm_result["finalState"])
|
||||
|
||||
xml = ET.fromstring(FIXTURE.read_bytes())
|
||||
pressure = xml.find("./Components/Component[@id='amesim_pnch023_1']/Parameter[@name='p0']")
|
||||
self.assertIsNotNone(pressure)
|
||||
pressure.set("value", "16000000")
|
||||
modified = self._document(ET.tostring(xml, encoding="utf-8"))
|
||||
changed_program = compile_native_program(compile_system_xml_network(modified))
|
||||
self.assertEqual(changed_program.header, self.program.header)
|
||||
changed, changed_commands = self._build(changed_program)
|
||||
self.assertFalse(changed.cache_hit)
|
||||
self.assertNotEqual(changed.manifest["buildKey"], cold.manifest["buildKey"])
|
||||
self.assertEqual(self._compile_count(changed_commands), 1)
|
||||
self.assertEqual(self._link_count(changed_commands), 1)
|
||||
self.assertEqual(changed.details["objectCompilations"], 1)
|
||||
self.assertGreater(changed.details["objectCacheHits"], 0)
|
||||
self._assert_windows_package(changed)
|
||||
|
||||
test = self
|
||||
sweeps = []
|
||||
|
||||
class PruningTracker(SolverActivityTracker):
|
||||
def start_integration(self, time):
|
||||
super().start_integration(time)
|
||||
# execute_native calls this after starting the real C child,
|
||||
# before reading its result. Both model entries are still in use.
|
||||
report = prune_cache(test.cache, model_limit_bytes=0)
|
||||
sweeps.append(report)
|
||||
test.assertGreaterEqual(report["models"]["skippedInUse"], 2)
|
||||
test.assertTrue(cold.executable.is_file())
|
||||
test.assertTrue(changed.executable.is_file())
|
||||
|
||||
changed_result = self._execute(changed, "changed run", activity_tracker=PruningTracker())
|
||||
self.assertEqual(len(sweeps), 1)
|
||||
column = "amesim_pnch023_1.p"
|
||||
self.assertNotEqual(changed_result["series"][column][0], cold_result["series"][column][0])
|
||||
self.assertEqual(changed_result["buildKey"], changed.manifest["buildKey"])
|
||||
|
||||
# Once use ends, a zero budget may evict the older model while retaining
|
||||
# the final oversized entry. Release every shared handle explicitly.
|
||||
self._close_builds()
|
||||
report = prune_cache(self.cache, model_limit_bytes=0)
|
||||
self.assertEqual(report["models"]["removedEntries"], 1)
|
||||
self.assertFalse(cold.executable.exists())
|
||||
self.assertTrue(changed.executable.is_file())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -6,6 +6,7 @@ import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
from types import SimpleNamespace
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
@@ -108,6 +109,63 @@ with acquire_cache_lease(Path(sys.argv[1]), 'models', sys.argv[2], exclusive=Tru
|
||||
gc.collect()
|
||||
self.assertIsNotNone(self.lease(3, exclusive=True, blocking=False))
|
||||
|
||||
def test_touch_uses_available_utime_operation_and_preserves_lru(self):
|
||||
real_utime = os.utime
|
||||
for supported in (False, True):
|
||||
with self.subTest(follow_symlinks_supported=supported):
|
||||
self.cache = self.folder / f"cache-capability-{supported}"
|
||||
touched = self.entry(0, 60, 1)
|
||||
old = self.entry(1, 60, 2)
|
||||
artifact_bytes = (touched / "artifact").read_bytes()
|
||||
before = touched.stat().st_mtime_ns
|
||||
calls = []
|
||||
|
||||
def platform_utime(path, times=None, **kwargs):
|
||||
calls.append(kwargs)
|
||||
if not supported and kwargs.get("follow_symlinks") is False:
|
||||
raise NotImplementedError("utime: follow_symlinks unavailable on this platform")
|
||||
# Both capability branches can run on either host. The
|
||||
# real timestamp change is on a validated ordinary folder.
|
||||
return real_utime(path, times)
|
||||
|
||||
with patch.object(storage.os, "utime", platform_utime), patch.object(
|
||||
storage.os, "supports_follow_symlinks", {platform_utime} if supported else set(),
|
||||
):
|
||||
with self.lease(0):
|
||||
storage.touch_cache_entry(self.cache, "models", key(0))
|
||||
self.assertEqual(calls, [{"follow_symlinks": False}] if supported else [{}])
|
||||
self.assertGreater(touched.stat().st_mtime_ns, before)
|
||||
self.assertEqual((touched / "artifact").read_bytes(), artifact_bytes)
|
||||
storage.prune_cache(self.cache, model_limit_bytes=60)
|
||||
self.assertTrue(touched.exists())
|
||||
self.assertFalse(old.exists())
|
||||
|
||||
def test_touch_rejects_windows_reparse_directory_before_timestamp_update(self):
|
||||
entry = self.entry(0, 60, 1)
|
||||
real_lstat = Path.lstat
|
||||
attributes = SimpleNamespace(
|
||||
st_mode=entry.lstat().st_mode, st_file_attributes=0x400,
|
||||
)
|
||||
|
||||
def reparse_lstat(path, *args, **kwargs):
|
||||
return attributes if path == entry else real_lstat(path, *args, **kwargs)
|
||||
|
||||
with patch.object(Path, "lstat", reparse_lstat), patch.object(
|
||||
storage.os, "supports_follow_symlinks", set(),
|
||||
), patch.object(storage.os, "utime") as update:
|
||||
with self.assertRaisesRegex(RuntimeError, "real directory"):
|
||||
storage.touch_cache_entry(self.cache, "models", key(0))
|
||||
update.assert_not_called()
|
||||
|
||||
def test_touch_does_not_hide_timestamp_permission_errors(self):
|
||||
self.entry(0, 60, 1)
|
||||
with patch.object(storage.os, "supports_follow_symlinks", set()), patch.object(
|
||||
storage.os, "utime", side_effect=PermissionError("timestamp denied"),
|
||||
) as update:
|
||||
with self.assertRaisesRegex(PermissionError, "timestamp denied"):
|
||||
storage.touch_cache_entry(self.cache, "models", key(0))
|
||||
update.assert_called_once()
|
||||
|
||||
def test_lru_removes_oldest_until_separate_budgets_are_met(self):
|
||||
oldest = self.entry(0, 60, 1)
|
||||
middle = self.entry(1, 60, 2)
|
||||
|
||||
Reference in new issue
Block a user