from __future__ import annotations import hashlib import json from pathlib import Path import sys import tempfile import textwrap import unittest from unittest.mock import patch from app.simulation.benchmark_regression import ( DEFAULT_MANIFEST_PATH, RegressionCaseRequest, RegressionManifestError, derive_simulation_xml, evaluate_regression_golden, load_regression_manifest, load_regression_golden, main, run_bounded_child_process, run_regression_suite, runtime_snapshot, source_simulation_config, summarize_simulation_result, ) def _completed_result(request: RegressionCaseRequest, *, wall_seconds: float = 0.01): signal_times = ( [0.04, 0.8] if request.stop_time >= 1.0 else [0.04] if request.stop_time >= 0.04 else [] ) return { "outcome": "completed", "orchestrationWallSeconds": wall_seconds, "worker": { "outcome": "completed", "wallSeconds": wall_seconds, "lastSimulatedTime": request.stop_time, "summary": { "success": True, "status": "completed", "simulatedUntil": request.stop_time, "seriesHealth": { "seriesCount": 1, "scalarCount": 2, "nonfiniteCount": 0, "timeStrictlyIncreasing": True, "timeStart": 0.0, "timeEnd": request.stop_time, }, "checkpoints": [ { "requestedTime": checkpoint, "actualTime": checkpoint, "available": True, "stateValues": {"state.placeholder": 0.0}, } for checkpoint in request.checkpoint_times ], "diagnostics": { "pressureFlow": {"maxScaledResidual": 1.0e-12}, }, "eventTrace": { "signalEventTimes": signal_times, "segments": [ {"startTime": start_time} for start_time in (0.0, *signal_times) ], "mechanicalTransitionTimes": [], "mechanicalTransitionTimesAvailable": True, }, "physicalContract": { "schemaVersion": 1, "projectionCategories": ["state"], "checkpoints": [ { "requestedTime": checkpoint, "actualTime": checkpoint, "available": True, "stateValues": {"state.placeholder": 0.0}, } for checkpoint in request.checkpoint_times ], "eventTrace": { "signalEventTimes": signal_times, "segments": [ {"startTime": start_time} for start_time in (0.0, *signal_times) ], "mechanicalTransitionTimes": [], "mechanicalTransitionTimesAvailable": True, }, }, "outputContract": { "schemaVersion": 1, "sha256": "0" * 64, "variableCount": 1, "seriesKeyCount": 1, "sampleCount": 2, }, }, }, } class RegressionManifestTests(unittest.TestCase): def test_runtime_snapshot_records_repository_identity(self) -> None: repository = runtime_snapshot()["repository"] if repository["head"] is not None: self.assertEqual(len(repository["head"]), 40) self.assertIn(repository["dirty"], (True, False, None)) self.assertIsInstance(repository["status"], list) def test_manifest_locks_authoritative_source_and_two_sampling_lanes(self) -> None: manifest = load_regression_manifest(DEFAULT_MANIFEST_PATH) self.assertEqual( manifest["sequence"], ["0.01s-smoke", "0.2s", "1s", "5s", "10s"], ) self.assertTrue(manifest["source"]["xmlIsAuthoritative"]) self.assertEqual( manifest["source"]["sha256"], "170463d65d074da01f0f9e9dab730b3815c94c1cc80b5190ec2e3fe623da74d3", ) self.assertEqual(Path(manifest["_sourcePath"]).name, "test-mql-8.xml") self.assertEqual( manifest["source"]["companionProject"]["sha256"], "258c50ee4850baa72fb7c2cc24536d0a631fc6a7f1fa6cedb7b6eea7c857cbaa", ) self.assertFalse( manifest["source"]["companionProject"]["executionInput"] ) self.assertEqual(manifest["historicalReports"][0]["status"], "historicalOnly") self.assertFalse( manifest["historicalReports"][0]["compatibleWithCurrentSource"] ) self.assertEqual(manifest["lanes"]["production"]["samplingMode"], "source") self.assertEqual(manifest["lanes"]["solver-only"]["sampleStep"], 0.02) self.assertEqual(manifest["lanes"]["solver-only"]["maxStep"], 0.05) def test_in_memory_derivative_does_not_change_authoritative_xml(self) -> None: manifest = load_regression_manifest(DEFAULT_MANIFEST_PATH) source_path = Path(manifest["_sourcePath"]) before = source_path.read_bytes() derived = derive_simulation_xml( before, stop_time=1.0, sample_step=0.02, max_step=0.005, ) self.assertEqual(source_simulation_config(before)["tStop"], 0.2) self.assertEqual(source_simulation_config(before)["sampleStep"], 0.01) self.assertEqual(source_simulation_config(before)["maxStep"], 0.01) self.assertEqual(source_simulation_config(derived)["tStop"], 1.0) self.assertEqual(source_simulation_config(derived)["sampleStep"], 0.02) self.assertEqual(source_simulation_config(derived)["maxStep"], 0.005) self.assertEqual(source_path.read_bytes(), before) def test_historical_pure_xml_manifest_does_not_require_companion(self) -> None: historical_manifest_path = ( DEFAULT_MANIFEST_PATH.parent.parent / "test_mql_full_branches" / "manifest.json" ) manifest = load_regression_manifest(historical_manifest_path) self.assertNotIn("companionProject", manifest["source"]) self.assertIsNone(manifest["_companionPath"]) class RegressionGoldenProtocolTests(unittest.TestCase): def test_reviewed_golden_is_provenanced_and_compared_numerically(self) -> None: source_xml_sha256 = "1" * 64 output_contract_sha256 = "2" * 64 generated_at = "2026-08-17T00:00:00+00:00" checkpoints = [{ "requestedTime": 0.0, "actualTime": 0.0, "available": True, "stateValues": {"state.a": 1.0}, }] summary = { "physicalContract": { "schemaVersion": 1, "projectionCategories": ["state"], "checkpoints": checkpoints, "eventTrace": {}, }, "outputContract": { "schemaVersion": 1, "sha256": output_contract_sha256, }, } report = { "generatedAt": generated_at, "source": {"sha256": source_xml_sha256}, "cases": [{ "caseId": "smoke", "lane": "solver-only", "worker": {"summary": summary}, }], } with tempfile.TemporaryDirectory() as temporary_directory: repository_root = Path(temporary_directory) report_path = repository_root / "source-report.json" report_path.write_text( json.dumps(report, sort_keys=True), encoding="utf-8" ) report_sha256 = hashlib.sha256(report_path.read_bytes()).hexdigest() state_keys = ["state.a"] state_layout_sha256 = hashlib.sha256( json.dumps( state_keys, ensure_ascii=False, sort_keys=True, separators=(",", ":"), ).encode("utf-8") ).hexdigest() golden = { "schemaVersion": 1, "id": "synthetic-reviewed-golden", "caseId": "smoke", "lane": "solver-only", "sourceXmlSha256": source_xml_sha256, "approval": {"status": "approved"}, "provenance": { "sourceReport": { "path": "source-report.json", "sha256": report_sha256, "generatedAt": generated_at, "metadataCompatibility": { "status": "current", "differences": [], }, } }, "physicalLayout": { "projectionCategories": ["state"], "stateKeys": state_keys, "stateKeyLayoutSha256": state_layout_sha256, }, "tolerance": { "relative": 0.001, "absolute": 0.01, "checkpointTimeAbsoluteSeconds": 1.0e-12, }, "physicalCheckpoints": [{ "requestedTime": 0.0, "values": [1.0], }], "outputContract": {"sha256": output_contract_sha256}, } golden_path = repository_root / "golden.json" golden_path.write_text( json.dumps(golden, sort_keys=True), encoding="utf-8" ) golden_sha256 = hashlib.sha256(golden_path.read_bytes()).hexdigest() loaded = load_regression_golden( golden_path, expected_sha256=golden_sha256, repository_root=repository_root, ) passing = evaluate_regression_golden(summary, loaded) self.assertTrue(passing["passed"]) self.assertEqual(passing["comparedValueCount"], 1) self.assertEqual(passing["issues"], []) changed_physical = json.loads(json.dumps(summary)) changed_physical["physicalContract"]["checkpoints"][0][ "stateValues" ]["state.a"] = 1.02 physical_failure = evaluate_regression_golden(changed_physical, loaded) self.assertFalse(physical_failure["passed"]) self.assertIn( "stateCheckpointGoldenValueMismatch", physical_failure["issues"] ) changed_output = json.loads(json.dumps(summary)) changed_output["outputContract"]["sha256"] = "3" * 64 output_failure = evaluate_regression_golden(changed_output, loaded) self.assertFalse(output_failure["passed"]) self.assertIn("outputContractMismatch", output_failure["issues"]) class BoundedChildProcessTests(unittest.TestCase): def test_short_json_worker_completes_without_timeout(self) -> None: script = textwrap.dedent( """ import json print(json.dumps({"event": "progress", "simulatedTime": 0.1}), flush=True) print(json.dumps({ "event": "result", "outcome": "completed", "lastSimulatedTime": 0.2, "wallSeconds": 0.01, }), flush=True) """ ) result = run_bounded_child_process( [sys.executable, "-u", "-c", script], soft_timeout_seconds=1.0, hard_timeout_seconds=2.0, termination_grace_seconds=0.2, ) self.assertEqual(result["outcome"], "completed") self.assertFalse(result["softCancelSent"]) self.assertFalse(result["hardTimeoutReached"]) self.assertEqual(result["lastSimulatedTime"], 0.2) def test_parent_requests_soft_cancel_before_hard_timeout(self) -> None: script = textwrap.dedent( """ import json import sys print(json.dumps({"event": "progress", "simulatedTime": 0.125}), flush=True) for line in sys.stdin: if line.strip() == "cancel": print(json.dumps({ "event": "result", "outcome": "cancelled", "lastSimulatedTime": 0.125, "wallSeconds": 0.1, }), flush=True) break """ ) result = run_bounded_child_process( [sys.executable, "-u", "-c", script], soft_timeout_seconds=0.08, hard_timeout_seconds=1.0, termination_grace_seconds=0.2, ) self.assertEqual(result["outcome"], "soft_timeout") self.assertTrue(result["softCancelSent"]) self.assertFalse(result["hardTimeoutReached"]) self.assertEqual(result["lastSimulatedTime"], 0.125) def test_unresponsive_child_is_terminated_at_hard_timeout(self) -> None: script = textwrap.dedent( """ import json import time print(json.dumps({"event": "progress", "simulatedTime": 0.05}), flush=True) time.sleep(5) """ ) result = run_bounded_child_process( [sys.executable, "-u", "-c", script], soft_timeout_seconds=0.05, hard_timeout_seconds=0.15, termination_grace_seconds=0.1, ) self.assertEqual(result["outcome"], "hard_timeout") self.assertTrue(result["softCancelSent"]) self.assertTrue(result["hardTimeoutReached"]) self.assertEqual(result["lastSimulatedTime"], 0.05) def test_child_that_closes_stdin_does_not_break_pipe_cleanup(self) -> None: script = textwrap.dedent( """ import os import time os.close(0) time.sleep(5) """ ) result = run_bounded_child_process( [sys.executable, "-u", "-c", script], soft_timeout_seconds=0.05, hard_timeout_seconds=0.15, termination_grace_seconds=0.1, ) self.assertEqual(result["outcome"], "hard_timeout") self.assertTrue(result["softCancelSent"]) self.assertTrue(result["hardTimeoutReached"]) class ProgressiveSuiteTests(unittest.TestCase): def test_requesting_late_case_also_runs_every_predecessor(self) -> None: calls: list[str] = [] def complete(request: RegressionCaseRequest) -> dict[str, object]: calls.append(request.case_id) return _completed_result(request) report = run_regression_suite( DEFAULT_MANIFEST_PATH, lane="solver-only", case_ids=["5s"], case_executor=complete, ) self.assertEqual(calls, ["0.01s-smoke", "0.2s", "1s", "5s"]) self.assertEqual( [case["outcome"] for case in report["cases"]], ["completed", "completed", "completed", "completed"], ) def test_manifest_pins_solver_modes_in_every_request(self) -> None: seen: list[dict[str, str]] = [] def complete(request: RegressionCaseRequest) -> dict[str, object]: seen.append(dict(request.environment_overrides)) return _completed_result(request) run_regression_suite( DEFAULT_MANIFEST_PATH, lane="solver-only", case_ids=["0.01s-smoke"], case_executor=complete, ) self.assertEqual( seen, [{ "SIMULATION_CAUSAL_EXECUTOR_V2": "1", "SIMULATION_CAUSAL_COORDINATE_KERNEL": "1", "SIMULATION_CAUSAL_FAST_PATH": "1", "SIMULATION_ODE_JACOBIAN_MODE": "scipy", "SIMULATIONAPP_PROPERTY_CACHE": "on", }], ) def test_empty_explicit_case_list_is_rejected(self) -> None: with self.assertRaisesRegex( RegressionManifestError, "No regression variants were selected", ): run_regression_suite( DEFAULT_MANIFEST_PATH, lane="solver-only", case_ids=[], case_executor=_completed_result, ) def test_missing_signal_segment_fails_acceptance(self) -> None: def missing_segment(request: RegressionCaseRequest) -> dict[str, object]: result = _completed_result(request) result["worker"]["summary"]["eventTrace"]["segments"] = [ {"startTime": 0.0} ] return result report = run_regression_suite( DEFAULT_MANIFEST_PATH, lane="solver-only", case_ids=["0.2s"], case_executor=missing_segment, ) self.assertEqual(report["cases"][1]["outcome"], "correctness_failed") self.assertIn( "signalEventSegmentMissing", report["cases"][1]["acceptance"]["issues"], ) def test_deferred_suite_returns_nonzero_incomplete_exit_status(self) -> None: report = {"cases": [ {"outcome": "completed"}, {"outcome": "deferred"}, ]} with patch( "app.simulation.benchmark_regression.run_regression_suite", return_value=report, ), patch("builtins.print"): exit_code = main([]) self.assertEqual(exit_code, 2) def test_nonfinite_completed_case_fails_acceptance_and_defers_longer_runs( self, ) -> None: calls: list[str] = [] def nonfinite_first(request: RegressionCaseRequest) -> dict[str, object]: calls.append(request.case_id) result = _completed_result(request) result["worker"]["summary"]["seriesHealth"]["nonfiniteCount"] = 1 return result report = run_regression_suite( DEFAULT_MANIFEST_PATH, lane="solver-only", case_executor=nonfinite_first, ) self.assertEqual(calls, ["0.01s-smoke"]) self.assertEqual(report["cases"][0]["outcome"], "correctness_failed") self.assertEqual( report["cases"][0]["acceptance"]["issues"], ["nonfiniteSeries"], ) self.assertTrue( all(case["outcome"] == "deferred" for case in report["cases"][1:]) ) def test_failed_predecessor_defers_every_longer_horizon(self) -> None: calls: list[str] = [] def fail_first(request: RegressionCaseRequest) -> dict[str, object]: calls.append(request.case_id) return {"outcome": "error", "worker": None} report = run_regression_suite( DEFAULT_MANIFEST_PATH, lane="solver-only", case_executor=fail_first, ) self.assertEqual(calls, ["0.01s-smoke"]) self.assertEqual(report["cases"][0]["outcome"], "error") self.assertEqual( [case["outcome"] for case in report["cases"][1:]], ["deferred", "deferred", "deferred", "deferred"], ) self.assertTrue( all( case["reason"] == "predecessorDidNotComplete" for case in report["cases"][1:] ) ) def test_prediction_over_budget_defers_before_launching_next_case(self) -> None: calls: list[str] = [] def expensive_short_case(request: RegressionCaseRequest) -> dict[str, object]: calls.append(request.case_id) return _completed_result(request, wall_seconds=200.0) report = run_regression_suite( DEFAULT_MANIFEST_PATH, lane="solver-only", case_executor=expensive_short_case, ) self.assertEqual(calls, ["0.01s-smoke", "0.2s"]) self.assertIsNone(report["cases"][1]["predictedWallSeconds"]) third = report["cases"][2] self.assertEqual(third["outcome"], "deferred") self.assertEqual(third["reason"], "predictedWallExceedsSoftBudget") self.assertEqual(third["predictedWallSeconds"], 1500.0) def test_lanes_pass_distinct_sampling_to_case_executor(self) -> None: seen: list[tuple[str, float, float]] = [] def complete(request: RegressionCaseRequest) -> dict[str, object]: seen.append((request.lane, request.sample_step, request.max_step)) return _completed_result(request) run_regression_suite( DEFAULT_MANIFEST_PATH, lane="solver-only", case_ids=["0.01s-smoke"], case_executor=complete, ) run_regression_suite( DEFAULT_MANIFEST_PATH, lane="production", case_ids=["0.01s-smoke"], case_executor=complete, ) self.assertEqual( seen, [("solver-only", 0.02, 0.05), ("production", 0.01, 0.01)], ) class ResultSummaryTests(unittest.TestCase): def test_summary_preserves_diagnostics_and_available_event_trace(self) -> None: result = { "success": True, "status": "completed", "partial": False, "message": "done", "simulatedUntil": 0.2, "requestedStopTime": 0.2, "variables": [ {"key": "tank.m", "category": "state"}, {"key": "tank.p", "category": "thermodynamic"}, ], "series": { "time": [0.0, 0.2], "tank.m": [1.0, 0.9], "tank.p": [2.0, 1.5], }, "final": {"tank.m": 0.9, "tank.p": 1.5}, "diagnostics": { "stateCount": 1, "signal": {"eventTimes": [0.04]}, "integration": { "totals": { "stateTransitionCount": 2, "solverStartCount": 3, }, "segments": [ { "startTime": 0.0, "requestedStopTime": 0.2, "simulatedUntil": 0.2, "stateTransitionCount": 2, "stateTransitionTimes": [0.11, 0.17], "solverStartCount": 3, "recoverableRetryCount": 0, } ], }, }, } summary = summarize_simulation_result( result, checkpoint_times=(0.0, 0.2), sample_step=0.2, ) self.assertEqual(summary["diagnostics"], result["diagnostics"]) self.assertEqual(summary["eventTrace"]["signalEventTimes"], [0.04]) self.assertEqual(summary["eventTrace"]["stateTransitionCount"], 2) self.assertTrue( summary["eventTrace"]["mechanicalTransitionTimesAvailable"] ) self.assertEqual( summary["eventTrace"]["mechanicalTransitionTimes"], [0.11, 0.17], ) self.assertEqual( summary["checkpoints"][1]["stateValues"], {"tank.m": 0.9} ) self.assertEqual( summary["physicalContract"]["checkpoints"], summary["checkpoints"] ) self.assertEqual( summary["physicalContract"]["eventTrace"], summary["eventTrace"] ) self.assertFalse(summary["outputContract"]["containsPhysicalValues"]) self.assertEqual(summary["seriesHealth"]["nonfiniteCount"], 0) changed_values = json.loads(json.dumps(result)) changed_values["series"]["tank.m"] = [99.0, -99.0] changed_values["series"]["tank.p"] = [-2.0, 1000.0] value_changed_summary = summarize_simulation_result( changed_values, checkpoint_times=(0.0, 0.2), sample_step=0.2, ) self.assertEqual( value_changed_summary["outputContract"]["sha256"], summary["outputContract"]["sha256"], ) changed_metadata = json.loads(json.dumps(result)) changed_metadata["variables"][0]["unit"] = "kg" metadata_changed_summary = summarize_simulation_result( changed_metadata, checkpoint_times=(0.0, 0.2), sample_step=0.2, ) self.assertNotEqual( metadata_changed_summary["outputContract"]["sha256"], summary["outputContract"]["sha256"], ) def test_missing_mechanical_transition_times_are_reported_unavailable( self, ) -> None: summary = summarize_simulation_result( { "diagnostics": { "integration": { "totals": {"stateTransitionCount": 1}, "segments": [{ "startTime": 0.0, "stateTransitionCount": 1, }], }, }, }, checkpoint_times=(), sample_step=0.1, ) self.assertFalse( summary["eventTrace"]["mechanicalTransitionTimesAvailable"] ) if __name__ == "__main__": unittest.main()