713 lines
25 KiB
Python
713 lines
25 KiB
Python
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import json
|
|
from pathlib import Path
|
|
import sys
|
|
import tempfile
|
|
import textwrap
|
|
import unittest
|
|
from unittest.mock import patch
|
|
|
|
from app.simulation.benchmark_regression import (
|
|
DEFAULT_MANIFEST_PATH,
|
|
RegressionCaseRequest,
|
|
RegressionManifestError,
|
|
derive_simulation_xml,
|
|
evaluate_regression_golden,
|
|
load_regression_manifest,
|
|
load_regression_golden,
|
|
main,
|
|
run_bounded_child_process,
|
|
run_regression_suite,
|
|
runtime_snapshot,
|
|
source_simulation_config,
|
|
summarize_simulation_result,
|
|
)
|
|
|
|
|
|
def _completed_result(request: RegressionCaseRequest, *, wall_seconds: float = 0.01):
|
|
signal_times = (
|
|
[0.04, 0.8]
|
|
if request.stop_time >= 1.0
|
|
else [0.04] if request.stop_time >= 0.04 else []
|
|
)
|
|
return {
|
|
"outcome": "completed",
|
|
"orchestrationWallSeconds": wall_seconds,
|
|
"worker": {
|
|
"outcome": "completed",
|
|
"wallSeconds": wall_seconds,
|
|
"lastSimulatedTime": request.stop_time,
|
|
"summary": {
|
|
"success": True,
|
|
"status": "completed",
|
|
"simulatedUntil": request.stop_time,
|
|
"seriesHealth": {
|
|
"seriesCount": 1,
|
|
"scalarCount": 2,
|
|
"nonfiniteCount": 0,
|
|
"timeStrictlyIncreasing": True,
|
|
"timeStart": 0.0,
|
|
"timeEnd": request.stop_time,
|
|
},
|
|
"checkpoints": [
|
|
{
|
|
"requestedTime": checkpoint,
|
|
"actualTime": checkpoint,
|
|
"available": True,
|
|
"stateValues": {"state.placeholder": 0.0},
|
|
}
|
|
for checkpoint in request.checkpoint_times
|
|
],
|
|
"diagnostics": {
|
|
"pressureFlow": {"maxScaledResidual": 1.0e-12},
|
|
},
|
|
"eventTrace": {
|
|
"signalEventTimes": signal_times,
|
|
"segments": [
|
|
{"startTime": start_time}
|
|
for start_time in (0.0, *signal_times)
|
|
],
|
|
"mechanicalTransitionTimes": [],
|
|
"mechanicalTransitionTimesAvailable": True,
|
|
},
|
|
"physicalContract": {
|
|
"schemaVersion": 1,
|
|
"projectionCategories": ["state"],
|
|
"checkpoints": [
|
|
{
|
|
"requestedTime": checkpoint,
|
|
"actualTime": checkpoint,
|
|
"available": True,
|
|
"stateValues": {"state.placeholder": 0.0},
|
|
}
|
|
for checkpoint in request.checkpoint_times
|
|
],
|
|
"eventTrace": {
|
|
"signalEventTimes": signal_times,
|
|
"segments": [
|
|
{"startTime": start_time}
|
|
for start_time in (0.0, *signal_times)
|
|
],
|
|
"mechanicalTransitionTimes": [],
|
|
"mechanicalTransitionTimesAvailable": True,
|
|
},
|
|
},
|
|
"outputContract": {
|
|
"schemaVersion": 1,
|
|
"sha256": "0" * 64,
|
|
"variableCount": 1,
|
|
"seriesKeyCount": 1,
|
|
"sampleCount": 2,
|
|
},
|
|
},
|
|
},
|
|
}
|
|
|
|
|
|
class RegressionManifestTests(unittest.TestCase):
|
|
def test_runtime_snapshot_records_repository_identity(self) -> None:
|
|
repository = runtime_snapshot()["repository"]
|
|
|
|
if repository["head"] is not None:
|
|
self.assertEqual(len(repository["head"]), 40)
|
|
self.assertIn(repository["dirty"], (True, False, None))
|
|
self.assertIsInstance(repository["status"], list)
|
|
|
|
def test_manifest_locks_authoritative_source_and_two_sampling_lanes(self) -> None:
|
|
manifest = load_regression_manifest(DEFAULT_MANIFEST_PATH)
|
|
|
|
self.assertEqual(
|
|
manifest["sequence"],
|
|
["0.01s-smoke", "0.2s", "1s", "5s", "10s"],
|
|
)
|
|
self.assertTrue(manifest["source"]["xmlIsAuthoritative"])
|
|
self.assertEqual(
|
|
manifest["source"]["sha256"],
|
|
"6eb2753a15af66fd8e91c169c4587e66b0ace2164c08c2cb23a67aab5fd09164",
|
|
)
|
|
self.assertEqual(Path(manifest["_sourcePath"]).name, "test-mql-8.xml")
|
|
self.assertEqual(
|
|
manifest["source"]["companionProject"]["sha256"],
|
|
"8d7a263691a9f09f39c92aba19ae1350911dc66b4904e98bae5f20592c5a3fcd",
|
|
)
|
|
self.assertFalse(
|
|
manifest["source"]["companionProject"]["executionInput"]
|
|
)
|
|
self.assertEqual(manifest["historicalReports"][0]["status"], "historicalOnly")
|
|
self.assertFalse(
|
|
manifest["historicalReports"][0]["compatibleWithCurrentSource"]
|
|
)
|
|
self.assertEqual(manifest["lanes"]["production"]["samplingMode"], "source")
|
|
self.assertEqual(manifest["lanes"]["solver-only"]["sampleStep"], 0.02)
|
|
self.assertEqual(manifest["lanes"]["solver-only"]["maxStep"], 0.05)
|
|
self.assertEqual(manifest["historicalReports"][2]["status"], "historicalOnly")
|
|
self.assertFalse(
|
|
manifest["historicalReports"][2]["compatibleWithCurrentSource"]
|
|
)
|
|
|
|
def test_in_memory_derivative_does_not_change_authoritative_xml(self) -> None:
|
|
manifest = load_regression_manifest(DEFAULT_MANIFEST_PATH)
|
|
source_path = Path(manifest["_sourcePath"])
|
|
before = source_path.read_bytes()
|
|
|
|
derived = derive_simulation_xml(
|
|
before,
|
|
stop_time=1.0,
|
|
sample_step=0.02,
|
|
max_step=0.005,
|
|
)
|
|
|
|
self.assertEqual(source_simulation_config(before)["tStop"], 5.0)
|
|
self.assertEqual(source_simulation_config(before)["sampleStep"], 0.005)
|
|
self.assertEqual(source_simulation_config(before)["maxStep"], 0.005)
|
|
self.assertEqual(source_simulation_config(derived)["tStop"], 1.0)
|
|
self.assertEqual(source_simulation_config(derived)["sampleStep"], 0.02)
|
|
self.assertEqual(source_simulation_config(derived)["maxStep"], 0.005)
|
|
self.assertEqual(source_path.read_bytes(), before)
|
|
|
|
def test_historical_pure_xml_manifest_does_not_require_companion(self) -> None:
|
|
historical_manifest_path = (
|
|
DEFAULT_MANIFEST_PATH.parent.parent
|
|
/ "test_mql_full_branches"
|
|
/ "manifest.json"
|
|
)
|
|
|
|
manifest = load_regression_manifest(historical_manifest_path)
|
|
|
|
self.assertNotIn("companionProject", manifest["source"])
|
|
self.assertIsNone(manifest["_companionPath"])
|
|
|
|
|
|
class RegressionGoldenProtocolTests(unittest.TestCase):
|
|
def test_reviewed_golden_is_provenanced_and_compared_numerically(self) -> None:
|
|
source_xml_sha256 = "1" * 64
|
|
output_contract_sha256 = "2" * 64
|
|
generated_at = "2026-08-17T00:00:00+00:00"
|
|
checkpoints = [{
|
|
"requestedTime": 0.0,
|
|
"actualTime": 0.0,
|
|
"available": True,
|
|
"stateValues": {"state.a": 1.0},
|
|
}]
|
|
summary = {
|
|
"physicalContract": {
|
|
"schemaVersion": 1,
|
|
"projectionCategories": ["state"],
|
|
"checkpoints": checkpoints,
|
|
"eventTrace": {},
|
|
},
|
|
"outputContract": {
|
|
"schemaVersion": 1,
|
|
"sha256": output_contract_sha256,
|
|
},
|
|
}
|
|
report = {
|
|
"generatedAt": generated_at,
|
|
"source": {"sha256": source_xml_sha256},
|
|
"cases": [{
|
|
"caseId": "smoke",
|
|
"lane": "solver-only",
|
|
"worker": {"summary": summary},
|
|
}],
|
|
}
|
|
|
|
with tempfile.TemporaryDirectory() as temporary_directory:
|
|
repository_root = Path(temporary_directory)
|
|
report_path = repository_root / "source-report.json"
|
|
report_path.write_text(
|
|
json.dumps(report, sort_keys=True), encoding="utf-8"
|
|
)
|
|
report_sha256 = hashlib.sha256(report_path.read_bytes()).hexdigest()
|
|
state_keys = ["state.a"]
|
|
state_layout_sha256 = hashlib.sha256(
|
|
json.dumps(
|
|
state_keys,
|
|
ensure_ascii=False,
|
|
sort_keys=True,
|
|
separators=(",", ":"),
|
|
).encode("utf-8")
|
|
).hexdigest()
|
|
golden = {
|
|
"schemaVersion": 1,
|
|
"id": "synthetic-reviewed-golden",
|
|
"caseId": "smoke",
|
|
"lane": "solver-only",
|
|
"sourceXmlSha256": source_xml_sha256,
|
|
"approval": {"status": "approved"},
|
|
"provenance": {
|
|
"sourceReport": {
|
|
"path": "source-report.json",
|
|
"sha256": report_sha256,
|
|
"generatedAt": generated_at,
|
|
"metadataCompatibility": {
|
|
"status": "current",
|
|
"differences": [],
|
|
},
|
|
}
|
|
},
|
|
"physicalLayout": {
|
|
"projectionCategories": ["state"],
|
|
"stateKeys": state_keys,
|
|
"stateKeyLayoutSha256": state_layout_sha256,
|
|
},
|
|
"tolerance": {
|
|
"relative": 0.001,
|
|
"absolute": 0.01,
|
|
"checkpointTimeAbsoluteSeconds": 1.0e-12,
|
|
},
|
|
"physicalCheckpoints": [{
|
|
"requestedTime": 0.0,
|
|
"values": [1.0],
|
|
}],
|
|
"outputContract": {"sha256": output_contract_sha256},
|
|
}
|
|
golden_path = repository_root / "golden.json"
|
|
golden_path.write_text(
|
|
json.dumps(golden, sort_keys=True), encoding="utf-8"
|
|
)
|
|
golden_sha256 = hashlib.sha256(golden_path.read_bytes()).hexdigest()
|
|
|
|
loaded = load_regression_golden(
|
|
golden_path,
|
|
expected_sha256=golden_sha256,
|
|
repository_root=repository_root,
|
|
)
|
|
|
|
passing = evaluate_regression_golden(summary, loaded)
|
|
self.assertTrue(passing["passed"])
|
|
self.assertEqual(passing["comparedValueCount"], 1)
|
|
self.assertEqual(passing["issues"], [])
|
|
|
|
changed_physical = json.loads(json.dumps(summary))
|
|
changed_physical["physicalContract"]["checkpoints"][0][
|
|
"stateValues"
|
|
]["state.a"] = 1.02
|
|
physical_failure = evaluate_regression_golden(changed_physical, loaded)
|
|
self.assertFalse(physical_failure["passed"])
|
|
self.assertIn(
|
|
"stateCheckpointGoldenValueMismatch", physical_failure["issues"]
|
|
)
|
|
|
|
changed_output = json.loads(json.dumps(summary))
|
|
changed_output["outputContract"]["sha256"] = "3" * 64
|
|
output_failure = evaluate_regression_golden(changed_output, loaded)
|
|
self.assertFalse(output_failure["passed"])
|
|
self.assertIn("outputContractMismatch", output_failure["issues"])
|
|
|
|
|
|
class BoundedChildProcessTests(unittest.TestCase):
|
|
def test_short_json_worker_completes_without_timeout(self) -> None:
|
|
script = textwrap.dedent(
|
|
"""
|
|
import json
|
|
print(json.dumps({"event": "progress", "simulatedTime": 0.1}), flush=True)
|
|
print(json.dumps({
|
|
"event": "result",
|
|
"outcome": "completed",
|
|
"lastSimulatedTime": 0.2,
|
|
"wallSeconds": 0.01,
|
|
}), flush=True)
|
|
"""
|
|
)
|
|
|
|
result = run_bounded_child_process(
|
|
[sys.executable, "-u", "-c", script],
|
|
soft_timeout_seconds=1.0,
|
|
hard_timeout_seconds=2.0,
|
|
termination_grace_seconds=0.2,
|
|
)
|
|
|
|
self.assertEqual(result["outcome"], "completed")
|
|
self.assertFalse(result["softCancelSent"])
|
|
self.assertFalse(result["hardTimeoutReached"])
|
|
self.assertEqual(result["lastSimulatedTime"], 0.2)
|
|
|
|
def test_parent_requests_soft_cancel_before_hard_timeout(self) -> None:
|
|
script = textwrap.dedent(
|
|
"""
|
|
import json
|
|
import sys
|
|
print(json.dumps({"event": "progress", "simulatedTime": 0.125}), flush=True)
|
|
for line in sys.stdin:
|
|
if line.strip() == "cancel":
|
|
print(json.dumps({
|
|
"event": "result",
|
|
"outcome": "cancelled",
|
|
"lastSimulatedTime": 0.125,
|
|
"wallSeconds": 0.1,
|
|
}), flush=True)
|
|
break
|
|
"""
|
|
)
|
|
|
|
result = run_bounded_child_process(
|
|
[sys.executable, "-u", "-c", script],
|
|
soft_timeout_seconds=0.08,
|
|
hard_timeout_seconds=1.0,
|
|
termination_grace_seconds=0.2,
|
|
)
|
|
|
|
self.assertEqual(result["outcome"], "soft_timeout")
|
|
self.assertTrue(result["softCancelSent"])
|
|
self.assertFalse(result["hardTimeoutReached"])
|
|
self.assertEqual(result["lastSimulatedTime"], 0.125)
|
|
|
|
def test_unresponsive_child_is_terminated_at_hard_timeout(self) -> None:
|
|
script = textwrap.dedent(
|
|
"""
|
|
import json
|
|
import time
|
|
print(json.dumps({"event": "progress", "simulatedTime": 0.05}), flush=True)
|
|
time.sleep(5)
|
|
"""
|
|
)
|
|
|
|
result = run_bounded_child_process(
|
|
[sys.executable, "-u", "-c", script],
|
|
soft_timeout_seconds=0.05,
|
|
hard_timeout_seconds=0.15,
|
|
termination_grace_seconds=0.1,
|
|
)
|
|
|
|
self.assertEqual(result["outcome"], "hard_timeout")
|
|
self.assertTrue(result["softCancelSent"])
|
|
self.assertTrue(result["hardTimeoutReached"])
|
|
self.assertEqual(result["lastSimulatedTime"], 0.05)
|
|
|
|
def test_child_that_closes_stdin_does_not_break_pipe_cleanup(self) -> None:
|
|
script = textwrap.dedent(
|
|
"""
|
|
import os
|
|
import time
|
|
os.close(0)
|
|
time.sleep(5)
|
|
"""
|
|
)
|
|
|
|
result = run_bounded_child_process(
|
|
[sys.executable, "-u", "-c", script],
|
|
soft_timeout_seconds=0.05,
|
|
hard_timeout_seconds=0.15,
|
|
termination_grace_seconds=0.1,
|
|
)
|
|
|
|
self.assertEqual(result["outcome"], "hard_timeout")
|
|
self.assertTrue(result["softCancelSent"])
|
|
self.assertTrue(result["hardTimeoutReached"])
|
|
|
|
|
|
class ProgressiveSuiteTests(unittest.TestCase):
|
|
def test_requesting_late_case_also_runs_every_predecessor(self) -> None:
|
|
calls: list[str] = []
|
|
|
|
def complete(request: RegressionCaseRequest) -> dict[str, object]:
|
|
calls.append(request.case_id)
|
|
return _completed_result(request)
|
|
|
|
report = run_regression_suite(
|
|
DEFAULT_MANIFEST_PATH,
|
|
lane="solver-only",
|
|
case_ids=["5s"],
|
|
case_executor=complete,
|
|
)
|
|
|
|
self.assertEqual(calls, ["0.01s-smoke", "0.2s", "1s", "5s"])
|
|
self.assertEqual(
|
|
[case["outcome"] for case in report["cases"]],
|
|
["completed", "completed", "completed", "completed"],
|
|
)
|
|
|
|
def test_manifest_pins_solver_modes_in_every_request(self) -> None:
|
|
seen: list[dict[str, str]] = []
|
|
|
|
def complete(request: RegressionCaseRequest) -> dict[str, object]:
|
|
seen.append(dict(request.environment_overrides))
|
|
return _completed_result(request)
|
|
|
|
run_regression_suite(
|
|
DEFAULT_MANIFEST_PATH,
|
|
lane="solver-only",
|
|
case_ids=["0.01s-smoke"],
|
|
case_executor=complete,
|
|
)
|
|
|
|
self.assertEqual(
|
|
seen,
|
|
[{
|
|
"SIMULATION_CAUSAL_EXECUTOR_V2": "1",
|
|
"SIMULATION_CAUSAL_COORDINATE_KERNEL": "1",
|
|
"SIMULATION_CAUSAL_FAST_PATH": "1",
|
|
"SIMULATION_ODE_JACOBIAN_MODE": "scipy",
|
|
"SIMULATIONAPP_PROPERTY_CACHE": "on",
|
|
}],
|
|
)
|
|
|
|
def test_empty_explicit_case_list_is_rejected(self) -> None:
|
|
with self.assertRaisesRegex(
|
|
RegressionManifestError,
|
|
"No regression variants were selected",
|
|
):
|
|
run_regression_suite(
|
|
DEFAULT_MANIFEST_PATH,
|
|
lane="solver-only",
|
|
case_ids=[],
|
|
case_executor=_completed_result,
|
|
)
|
|
|
|
def test_missing_signal_segment_fails_acceptance(self) -> None:
|
|
def missing_segment(request: RegressionCaseRequest) -> dict[str, object]:
|
|
result = _completed_result(request)
|
|
result["worker"]["summary"]["eventTrace"]["segments"] = [
|
|
{"startTime": 0.0}
|
|
]
|
|
return result
|
|
|
|
report = run_regression_suite(
|
|
DEFAULT_MANIFEST_PATH,
|
|
lane="solver-only",
|
|
case_ids=["0.2s"],
|
|
case_executor=missing_segment,
|
|
)
|
|
|
|
self.assertEqual(report["cases"][1]["outcome"], "correctness_failed")
|
|
self.assertIn(
|
|
"signalEventSegmentMissing",
|
|
report["cases"][1]["acceptance"]["issues"],
|
|
)
|
|
|
|
def test_deferred_suite_returns_nonzero_incomplete_exit_status(self) -> None:
|
|
report = {"cases": [
|
|
{"outcome": "completed"},
|
|
{"outcome": "deferred"},
|
|
]}
|
|
with patch(
|
|
"app.simulation.benchmark_regression.run_regression_suite",
|
|
return_value=report,
|
|
), patch("builtins.print"):
|
|
exit_code = main([])
|
|
|
|
self.assertEqual(exit_code, 2)
|
|
|
|
def test_nonfinite_completed_case_fails_acceptance_and_defers_longer_runs(
|
|
self,
|
|
) -> None:
|
|
calls: list[str] = []
|
|
|
|
def nonfinite_first(request: RegressionCaseRequest) -> dict[str, object]:
|
|
calls.append(request.case_id)
|
|
result = _completed_result(request)
|
|
result["worker"]["summary"]["seriesHealth"]["nonfiniteCount"] = 1
|
|
return result
|
|
|
|
report = run_regression_suite(
|
|
DEFAULT_MANIFEST_PATH,
|
|
lane="solver-only",
|
|
case_executor=nonfinite_first,
|
|
)
|
|
|
|
self.assertEqual(calls, ["0.01s-smoke"])
|
|
self.assertEqual(report["cases"][0]["outcome"], "correctness_failed")
|
|
self.assertEqual(
|
|
report["cases"][0]["acceptance"]["issues"],
|
|
["nonfiniteSeries"],
|
|
)
|
|
self.assertTrue(
|
|
all(case["outcome"] == "deferred" for case in report["cases"][1:])
|
|
)
|
|
|
|
def test_failed_predecessor_defers_every_longer_horizon(self) -> None:
|
|
calls: list[str] = []
|
|
|
|
def fail_first(request: RegressionCaseRequest) -> dict[str, object]:
|
|
calls.append(request.case_id)
|
|
return {"outcome": "error", "worker": None}
|
|
|
|
report = run_regression_suite(
|
|
DEFAULT_MANIFEST_PATH,
|
|
lane="solver-only",
|
|
case_executor=fail_first,
|
|
)
|
|
|
|
self.assertEqual(calls, ["0.01s-smoke"])
|
|
self.assertEqual(report["cases"][0]["outcome"], "error")
|
|
self.assertEqual(
|
|
[case["outcome"] for case in report["cases"][1:]],
|
|
["deferred", "deferred", "deferred", "deferred"],
|
|
)
|
|
self.assertTrue(
|
|
all(
|
|
case["reason"] == "predecessorDidNotComplete"
|
|
for case in report["cases"][1:]
|
|
)
|
|
)
|
|
|
|
def test_prediction_over_budget_defers_before_launching_next_case(self) -> None:
|
|
calls: list[str] = []
|
|
|
|
def expensive_short_case(request: RegressionCaseRequest) -> dict[str, object]:
|
|
calls.append(request.case_id)
|
|
return _completed_result(request, wall_seconds=200.0)
|
|
|
|
report = run_regression_suite(
|
|
DEFAULT_MANIFEST_PATH,
|
|
lane="solver-only",
|
|
case_executor=expensive_short_case,
|
|
)
|
|
|
|
self.assertEqual(calls, ["0.01s-smoke", "0.2s"])
|
|
self.assertIsNone(report["cases"][1]["predictedWallSeconds"])
|
|
third = report["cases"][2]
|
|
self.assertEqual(third["outcome"], "deferred")
|
|
self.assertEqual(third["reason"], "predictedWallExceedsSoftBudget")
|
|
self.assertEqual(third["predictedWallSeconds"], 1500.0)
|
|
|
|
def test_lanes_pass_distinct_sampling_to_case_executor(self) -> None:
|
|
seen: list[tuple[str, float, float]] = []
|
|
|
|
def complete(request: RegressionCaseRequest) -> dict[str, object]:
|
|
seen.append((request.lane, request.sample_step, request.max_step))
|
|
return _completed_result(request)
|
|
|
|
run_regression_suite(
|
|
DEFAULT_MANIFEST_PATH,
|
|
lane="solver-only",
|
|
case_ids=["0.01s-smoke"],
|
|
case_executor=complete,
|
|
)
|
|
run_regression_suite(
|
|
DEFAULT_MANIFEST_PATH,
|
|
lane="production",
|
|
case_ids=["0.01s-smoke"],
|
|
case_executor=complete,
|
|
)
|
|
|
|
self.assertEqual(
|
|
seen,
|
|
[("solver-only", 0.02, 0.05), ("production", 0.005, 0.005)],
|
|
)
|
|
|
|
|
|
class ResultSummaryTests(unittest.TestCase):
|
|
def test_summary_preserves_diagnostics_and_available_event_trace(self) -> None:
|
|
result = {
|
|
"success": True,
|
|
"status": "completed",
|
|
"partial": False,
|
|
"message": "done",
|
|
"simulatedUntil": 0.2,
|
|
"requestedStopTime": 0.2,
|
|
"variables": [
|
|
{"key": "tank.m", "category": "state"},
|
|
{"key": "tank.p", "category": "thermodynamic"},
|
|
],
|
|
"series": {
|
|
"time": [0.0, 0.2],
|
|
"tank.m": [1.0, 0.9],
|
|
"tank.p": [2.0, 1.5],
|
|
},
|
|
"final": {"tank.m": 0.9, "tank.p": 1.5},
|
|
"diagnostics": {
|
|
"stateCount": 1,
|
|
"signal": {"eventTimes": [0.04]},
|
|
"integration": {
|
|
"totals": {
|
|
"stateTransitionCount": 2,
|
|
"solverStartCount": 3,
|
|
},
|
|
"segments": [
|
|
{
|
|
"startTime": 0.0,
|
|
"requestedStopTime": 0.2,
|
|
"simulatedUntil": 0.2,
|
|
"stateTransitionCount": 2,
|
|
"stateTransitionTimes": [0.11, 0.17],
|
|
"solverStartCount": 3,
|
|
"recoverableRetryCount": 0,
|
|
}
|
|
],
|
|
},
|
|
},
|
|
}
|
|
|
|
summary = summarize_simulation_result(
|
|
result,
|
|
checkpoint_times=(0.0, 0.2),
|
|
sample_step=0.2,
|
|
)
|
|
|
|
self.assertEqual(summary["diagnostics"], result["diagnostics"])
|
|
self.assertEqual(summary["eventTrace"]["signalEventTimes"], [0.04])
|
|
self.assertEqual(summary["eventTrace"]["stateTransitionCount"], 2)
|
|
self.assertTrue(
|
|
summary["eventTrace"]["mechanicalTransitionTimesAvailable"]
|
|
)
|
|
self.assertEqual(
|
|
summary["eventTrace"]["mechanicalTransitionTimes"],
|
|
[0.11, 0.17],
|
|
)
|
|
self.assertEqual(
|
|
summary["checkpoints"][1]["stateValues"], {"tank.m": 0.9}
|
|
)
|
|
self.assertEqual(
|
|
summary["physicalContract"]["checkpoints"], summary["checkpoints"]
|
|
)
|
|
self.assertEqual(
|
|
summary["physicalContract"]["eventTrace"], summary["eventTrace"]
|
|
)
|
|
self.assertFalse(summary["outputContract"]["containsPhysicalValues"])
|
|
self.assertEqual(summary["seriesHealth"]["nonfiniteCount"], 0)
|
|
|
|
changed_values = json.loads(json.dumps(result))
|
|
changed_values["series"]["tank.m"] = [99.0, -99.0]
|
|
changed_values["series"]["tank.p"] = [-2.0, 1000.0]
|
|
value_changed_summary = summarize_simulation_result(
|
|
changed_values,
|
|
checkpoint_times=(0.0, 0.2),
|
|
sample_step=0.2,
|
|
)
|
|
self.assertEqual(
|
|
value_changed_summary["outputContract"]["sha256"],
|
|
summary["outputContract"]["sha256"],
|
|
)
|
|
|
|
changed_metadata = json.loads(json.dumps(result))
|
|
changed_metadata["variables"][0]["unit"] = "kg"
|
|
metadata_changed_summary = summarize_simulation_result(
|
|
changed_metadata,
|
|
checkpoint_times=(0.0, 0.2),
|
|
sample_step=0.2,
|
|
)
|
|
self.assertNotEqual(
|
|
metadata_changed_summary["outputContract"]["sha256"],
|
|
summary["outputContract"]["sha256"],
|
|
)
|
|
|
|
def test_missing_mechanical_transition_times_are_reported_unavailable(
|
|
self,
|
|
) -> None:
|
|
summary = summarize_simulation_result(
|
|
{
|
|
"diagnostics": {
|
|
"integration": {
|
|
"totals": {"stateTransitionCount": 1},
|
|
"segments": [{
|
|
"startTime": 0.0,
|
|
"stateTransitionCount": 1,
|
|
}],
|
|
},
|
|
},
|
|
},
|
|
checkpoint_times=(),
|
|
sample_step=0.1,
|
|
)
|
|
|
|
self.assertFalse(
|
|
summary["eventTrace"]["mechanicalTransitionTimesAvailable"]
|
|
)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|