293 lines
16 KiB
Python
293 lines
16 KiB
Python
from __future__ import annotations
|
|
|
|
import asyncio
|
|
from contextlib import contextmanager
|
|
import os
|
|
from pathlib import Path
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
from threading import Event
|
|
import time
|
|
from types import SimpleNamespace
|
|
import unittest
|
|
from unittest.mock import patch
|
|
|
|
from app.main import _app_lifespan, app
|
|
from tests.test_native_result_transport import AsgiClient
|
|
from app.simulation.native_codegen import build, self_test
|
|
from app.simulation.warmup import SimulationRuntimeCheck
|
|
|
|
|
|
@contextmanager
|
|
def runtime_client():
|
|
# Reuse the repository's ASGI client; no optional httpx dependency is needed.
|
|
with asyncio.Runner() as runner:
|
|
lifespan = _app_lifespan(app)
|
|
runner.run(lifespan.__aenter__())
|
|
try:
|
|
yield AsgiClient(app)
|
|
finally:
|
|
runner.run(lifespan.__aexit__(None, None, None))
|
|
|
|
|
|
class SimulationWarmupTests(unittest.TestCase):
|
|
def setUp(self):
|
|
self.environment = patch.dict(os.environ, {"SIMULATION_NUMERIC_ENGINE": "native", "SIMULATIONAPP_WARMUP": "on"})
|
|
self.environment.start()
|
|
self.addCleanup(self.environment.stop)
|
|
self.retry_delay = patch("app.simulation.warmup.RETRY_DELAY_SECONDS", 0.01)
|
|
self.retry_delay.start()
|
|
self.addCleanup(self.retry_delay.stop)
|
|
|
|
def finish(self, check, timeout=8):
|
|
deadline = time.monotonic() + timeout
|
|
while check.snapshot()["status"] in ("pending", "running", "retrying"):
|
|
if time.monotonic() > deadline:
|
|
self.fail("Runtime check did not finish")
|
|
time.sleep(0.01)
|
|
return check.snapshot()
|
|
|
|
def test_editor_and_status_are_available_while_check_is_blocked(self):
|
|
entered, release = Event(), Event()
|
|
def pending(progress, stop):
|
|
progress(stage="compiling", message="正在编译最小自检程序")
|
|
entered.set()
|
|
self.assertTrue(release.wait(5))
|
|
with patch("app.simulation.warmup.check_native_runtime", side_effect=pending) as probe:
|
|
try:
|
|
with runtime_client() as client:
|
|
self.assertTrue(entered.wait(2))
|
|
check = app.state.simulation_runtime_check
|
|
check.start() # Duplicate calls must not launch duplicate compilers.
|
|
response = client.get("/api/simulation/runtime-check")
|
|
self.assertEqual(response.status_code, 200)
|
|
self.assertEqual(response.headers["cache-control"], "no-store")
|
|
self.assertEqual(response.json()["status"], "running")
|
|
self.assertEqual(response.json()["stage"], "compiling")
|
|
self.assertEqual(client.get("/api/components/catalog").status_code, 200)
|
|
self.assertEqual(client.get("/").status_code, 200)
|
|
release.set()
|
|
self.assertEqual(self.finish(check)["status"], "completed")
|
|
finally:
|
|
release.set()
|
|
probe.assert_called_once()
|
|
|
|
def test_failure_reports_details_and_keeps_editor_available(self):
|
|
def failed(progress, stop):
|
|
progress(stage="linking", message="正在链接", compiler="test-gcc")
|
|
raise self_test.CheckFailure("链接失败", command=["test-gcc", "model.o", "-o", "model"],
|
|
cwd=Path.cwd(), exit_code=7, stderr="missing library",
|
|
error_type="CalledProcessError")
|
|
with patch("app.simulation.warmup.check_native_runtime", side_effect=failed) as probe, self.assertLogs("app.simulation.warmup", level="ERROR") as logs:
|
|
with runtime_client() as client:
|
|
report = self.finish(app.state.simulation_runtime_check)
|
|
self.assertEqual(report["status"], "failed")
|
|
self.assertEqual(report["stage"], "linking")
|
|
self.assertEqual(report["failure"]["exitCode"], 7)
|
|
self.assertEqual(report["failure"]["stderr"], "missing library")
|
|
self.assertEqual(report["attempt"], 6)
|
|
self.assertEqual(report["maxRetries"], 5)
|
|
self.assertEqual(len(report["attemptHistory"]), 6)
|
|
self.assertEqual([r["attempt"] for r in report["attemptHistory"]], list(range(1, 7)))
|
|
self.assertIn("首次检测及 5 轮重试均失败", report["message"])
|
|
self.assertEqual(client.get("/api/simulation/runtime-check").json(), report)
|
|
self.assertEqual(client.get("/api/components/catalog").status_code, 200)
|
|
self.assertEqual(probe.call_count, 6)
|
|
self.assertEqual(len(logs.records), 1, "Detailed errors must only be logged after retries are exhausted")
|
|
|
|
def test_success_including_last_retry_stops_immediately_and_keeps_first_evidence(self):
|
|
for success_attempt in (1, 2, 6):
|
|
with self.subTest(success_attempt=success_attempt):
|
|
check = SimulationRuntimeCheck()
|
|
self.addCleanup(check.close)
|
|
calls = []
|
|
def probe(progress, stop):
|
|
calls.append(len(calls) + 1)
|
|
progress(stage="preprocessing", message="正在预处理")
|
|
self.assertIsNone(check.snapshot()["error"])
|
|
self.assertIsNone(check.snapshot()["failure"])
|
|
self.assertNotIn("attemptHistory", check.snapshot())
|
|
if len(calls) < success_attempt:
|
|
raise self_test.CheckFailure(f"first evidence {len(calls)}", exit_code=1,
|
|
stderr="gcc.exe: error: CreateProcess: No such file or directory")
|
|
with patch("app.simulation.warmup.check_native_runtime", side_effect=probe), patch("app.simulation.warmup.LOGGER") as logger:
|
|
check.start()
|
|
report = self.finish(check)
|
|
self.assertEqual(report["status"], "completed")
|
|
self.assertIn("仿真环境可用", report["message"])
|
|
self.assertEqual(report["attempt"], success_attempt)
|
|
self.assertEqual(len(calls), success_attempt)
|
|
self.assertIsNone(report["error"])
|
|
self.assertIsNone(report["failure"])
|
|
self.assertEqual(len(report["attemptHistory"]), success_attempt)
|
|
self.assertEqual(logger.warning.call_count, success_attempt - 1)
|
|
logger.error.assert_not_called()
|
|
if success_attempt > 1:
|
|
self.assertEqual(report["attemptHistory"][0]["error"], "first evidence 1")
|
|
|
|
def test_retry_wait_is_readable_nonblocking_and_cancelled_without_another_attempt(self):
|
|
def failed(progress, stop):
|
|
progress(stage="preprocessing", message="正在预处理")
|
|
raise self_test.CheckFailure("private failure", command=["gcc"], exit_code=1, stderr="private stderr")
|
|
with patch("app.simulation.warmup.RETRY_DELAY_SECONDS", 2), patch("app.simulation.warmup.check_native_runtime", side_effect=failed) as probe, patch("app.simulation.warmup.LOGGER") as logger:
|
|
with runtime_client() as client:
|
|
check = app.state.simulation_runtime_check
|
|
deadline = time.monotonic() + 2
|
|
while check.snapshot()["status"] != "retrying" and time.monotonic() < deadline:
|
|
time.sleep(0.01)
|
|
report = client.get("/api/simulation/runtime-check").json()
|
|
self.assertEqual(report["status"], "retrying")
|
|
self.assertEqual(report["attempt"], 1)
|
|
self.assertIn("第 1/5 轮重试", report["message"])
|
|
self.assertIsNone(report["error"])
|
|
self.assertIsNone(report["failure"])
|
|
self.assertNotIn("attemptHistory", report)
|
|
self.assertEqual(client.get("/api/components/catalog").status_code, 200)
|
|
started = time.monotonic()
|
|
check.close()
|
|
self.assertLess(time.monotonic() - started, 0.8)
|
|
self.assertEqual(check.snapshot()["status"], "cancelled")
|
|
self.assertEqual(probe.call_count, 1)
|
|
self.assertEqual(logger.warning.call_count, 1)
|
|
logger.error.assert_not_called()
|
|
|
|
def test_disabled_check_does_not_invoke_compiler(self):
|
|
with patch.dict(os.environ, {"SIMULATIONAPP_WARMUP": "off"}), patch("app.simulation.warmup.check_native_runtime") as probe:
|
|
check = SimulationRuntimeCheck()
|
|
check.start()
|
|
self.addCleanup(check.close)
|
|
self.assertEqual(self.finish(check)["status"], "disabled")
|
|
probe.assert_not_called()
|
|
|
|
def test_restart_has_new_check_id_and_does_not_reuse_failed_status(self):
|
|
with patch("app.simulation.warmup.check_native_runtime", side_effect=RuntimeError("compiler missing")), self.assertLogs("app.simulation.warmup", level="ERROR"):
|
|
failed = SimulationRuntimeCheck()
|
|
failed.start()
|
|
self.addCleanup(failed.close)
|
|
old = self.finish(failed)
|
|
with patch("app.simulation.warmup.check_native_runtime"):
|
|
recovered = SimulationRuntimeCheck()
|
|
recovered.start()
|
|
self.addCleanup(recovered.close)
|
|
new = self.finish(recovered)
|
|
self.assertEqual(old["status"], "failed")
|
|
self.assertEqual(new["status"], "completed")
|
|
self.assertNotEqual(old["checkId"], new["checkId"])
|
|
|
|
def test_invalid_configuration_is_visible_without_blocking_editor(self):
|
|
with patch.dict(os.environ, {"SIMULATIONAPP_WARMUP": "invalid"}), self.assertLogs("app.simulation.warmup", level="ERROR"):
|
|
with runtime_client() as client:
|
|
report = self.finish(app.state.simulation_runtime_check)
|
|
self.assertEqual(report["status"], "failed")
|
|
self.assertIn("SIMULATIONAPP_WARMUP", report["error"])
|
|
self.assertEqual(client.get("/api/components/catalog").status_code, 200)
|
|
|
|
def test_real_compiler_preprocesses_compiles_links_and_runs(self):
|
|
try:
|
|
build.toolchain()
|
|
except (OSError, RuntimeError, subprocess.SubprocessError) as exc:
|
|
if os.environ.get("SIMULATION_NATIVE_REQUIRE_TOOLCHAIN") == "1":
|
|
raise
|
|
self.skipTest(str(exc))
|
|
check = SimulationRuntimeCheck()
|
|
check.start()
|
|
self.addCleanup(check.close)
|
|
report = self.finish(check, timeout=90)
|
|
self.assertEqual(report["status"], "completed", report)
|
|
self.assertIsNone(report["failure"])
|
|
self.assertTrue(report["compiler"])
|
|
self.assertGreater(report["durationMs"], 0)
|
|
|
|
|
|
class NativeSelfCheckTests(unittest.TestCase):
|
|
def test_each_process_failure_preserves_stage_and_command(self):
|
|
for failed_stage, selector in [
|
|
("preprocessing", lambda c: "-E" in c),
|
|
("compiling", lambda c: "-c" in c),
|
|
("linking", lambda c: "-lm" in c),
|
|
("running", lambda c: len(c) == 1),
|
|
]:
|
|
with self.subTest(stage=failed_stage), tempfile.TemporaryDirectory() as directory:
|
|
state = {}
|
|
base = Path(directory)
|
|
def command(args, cwd, stop, timeout=30):
|
|
if selector(args):
|
|
raise self_test.CheckFailure("failure", command=args, cwd=cwd,
|
|
exit_code=9, stderr="diagnostic")
|
|
return self_test.OUTPUT_MARKER
|
|
with (
|
|
patch.object(self_test, "CACHE", base / "builds"),
|
|
patch.object(self_test, "toolchain", return_value=("gcc", base, "test gcc")),
|
|
patch.object(self_test, "platform_build_inputs", return_value=([], [], [], "model")),
|
|
patch.object(self_test, "_command", side_effect=command),
|
|
):
|
|
with self.assertRaises(self_test.CheckFailure) as captured:
|
|
self_test.check_native_runtime(lambda **v: state.update(v), Event())
|
|
self.assertEqual(state["stage"], failed_stage)
|
|
self.assertEqual(captured.exception.details["exitCode"], 9)
|
|
self.assertEqual(captured.exception.details["stderr"], "diagnostic")
|
|
self.assertTrue(captured.exception.details["command"])
|
|
self.assertEqual(list((base / "native-runtime-checks").iterdir()), [])
|
|
|
|
def test_command_not_found_exit_code_and_timeout(self):
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
base = Path(directory)
|
|
cases = [
|
|
([str(base / "missing-compiler")], 3, "FileNotFoundError", None),
|
|
([sys.executable, "-c", "import sys; print('bad link',file=sys.stderr); sys.exit(7)"], 3, "CalledProcessError", 7),
|
|
([sys.executable, "-c", "import time; time.sleep(10)"], 0.1, "TimeoutExpired", None),
|
|
]
|
|
for args, timeout, error_type, exit_code in cases:
|
|
started = time.monotonic()
|
|
with self.subTest(error=error_type), self.assertRaises(self_test.CheckFailure) as captured:
|
|
self_test._command(args, base, Event(), timeout=timeout)
|
|
details = captured.exception.details
|
|
self.assertEqual(details["command"], args)
|
|
self.assertEqual(details["errorType"], error_type)
|
|
if exit_code is not None:
|
|
self.assertEqual(details["exitCode"], exit_code)
|
|
self.assertIn("bad link", details["stderr"])
|
|
if error_type == "TimeoutExpired":
|
|
self.assertLess(time.monotonic() - started, 5, "Timeout must stop child processes too")
|
|
|
|
def test_cancelled_check_never_launches_a_process(self):
|
|
stop = Event()
|
|
stop.set()
|
|
with patch.object(self_test.subprocess, "Popen") as process:
|
|
with self.assertRaises(self_test.CheckCancelled):
|
|
self_test._command(["unused"], Path.cwd(), stop)
|
|
process.assert_not_called()
|
|
|
|
def test_no_gcc_does_not_attempt_to_compile(self):
|
|
with patch.dict(os.environ, {"SIMULATION_NATIVE_CC": ""}), patch.object(build.shutil, "which", return_value=None):
|
|
with patch.object(self_test, "_command") as command:
|
|
state = {}
|
|
with self.assertRaisesRegex(RuntimeError, "C compiler not found"):
|
|
self_test.check_native_runtime(lambda **v: state.update(v), Event())
|
|
self.assertEqual(state["stage"], "toolchain")
|
|
command.assert_not_called()
|
|
|
|
def test_both_platforms_keep_existing_flags_libraries_and_link_order(self):
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
base = Path(directory)
|
|
(base / "lib").mkdir()
|
|
for name in build.LIBRARIES:
|
|
(base / "lib" / f"libsundials_{name}.a").touch()
|
|
for platform in ("linux", "win32"):
|
|
windows = platform == "win32"
|
|
with self.subTest(platform=platform), patch.object(build, "os", SimpleNamespace(name="nt" if windows else "posix")), patch.object(build, "sys", SimpleNamespace(platform=platform)):
|
|
flags, libraries, dlls, executable = build.platform_build_inputs(base)
|
|
self.assertEqual(executable, "model.exe" if windows else "model")
|
|
self.assertEqual(len(libraries), len(build.LIBRARIES))
|
|
self.assertEqual("-static-libgcc" in flags, windows)
|
|
self.assertEqual("-D_POSIX_C_SOURCE=200809L" in flags, not windows)
|
|
self.assertEqual(bool(dlls), windows)
|
|
arguments = build.link_library_arguments(libraries)
|
|
self.assertEqual("-Wl,--start-group" in arguments, not windows)
|
|
self.assertTrue(all(p.suffix == (".lib" if windows else ".a") for p in libraries))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|