diff --git a/PythonModels/scripts/run_test_mql_full_state_comparison.py b/PythonModels/scripts/run_test_mql_full_state_comparison.py index 5a7930d..13ededc 100644 --- a/PythonModels/scripts/run_test_mql_full_state_comparison.py +++ b/PythonModels/scripts/run_test_mql_full_state_comparison.py @@ -8,6 +8,7 @@ from PythonModels.core.solver import SolveIVPConfig from PythonModels.reporting.amesim_results import AmesimResults, load_test_mql_amesim_results from PythonModels.reporting.test_mql_comparison import ( TestMqlComparisonResult, + interpolate_series_value, write_test_mql_comparison_csv, ) from PythonModels.reporting.test_mql_output_schema import ( @@ -36,6 +37,19 @@ DEFAULT_FULL_STATE_COMPARISON_DATA_PATHS = ( ) +@dataclass(frozen=True) +class TestMqlFullStateSignalDiagnostic: + data_path: str + initial_time_s: float + final_time_s: float + initial_python_value: float + initial_amesim_value: float + initial_abs_error: float + final_python_value: float + final_amesim_value: float + final_abs_error: float + + @dataclass(frozen=True) class TestMqlFullStateComparisonRun: system: TestMqlSystem @@ -67,6 +81,53 @@ class TestMqlFullStateComparisonRun: metrics = self.metrics_by_max_abs_error() return metrics[0] if metrics else None + def signal_diagnostic(self, data_path: str) -> TestMqlFullStateSignalDiagnostic: + times = self.output.times + python_values = self.output.series_by_data_path[data_path] + amesim_values = self.amesim_results.series(data_path) + initial_time = float(times[0]) + final_time = float(times[-1]) + initial_python = float(python_values[0]) + final_python = float(python_values[-1]) + initial_amesim = interpolate_series_value( + self.amesim_results.times, + amesim_values, + initial_time, + ) + final_amesim = interpolate_series_value( + self.amesim_results.times, + amesim_values, + final_time, + ) + return TestMqlFullStateSignalDiagnostic( + data_path=data_path, + initial_time_s=initial_time, + final_time_s=final_time, + initial_python_value=initial_python, + initial_amesim_value=initial_amesim, + initial_abs_error=abs(initial_python - initial_amesim), + final_python_value=final_python, + final_amesim_value=final_amesim, + final_abs_error=abs(final_python - final_amesim), + ) + + def diagnostics_by_final_abs_error(self): + return tuple( + sorted( + ( + self.signal_diagnostic(data_path) + for data_path in self.output.data_paths + ), + key=lambda diagnostic: diagnostic.final_abs_error, + reverse=True, + ) + ) + + @property + def largest_final_abs_error_diagnostic(self): + diagnostics = self.diagnostics_by_final_abs_error() + return diagnostics[0] if diagnostics else None + @dataclass(frozen=True) class TestMqlFullStateComparisonPathConfig: @@ -195,12 +256,28 @@ def format_test_mql_full_state_comparison_summary( "Largest absolute error: " f"{largest_metric.data_path}={largest_metric.max_abs_error}" ) + largest_diagnostic = run.largest_final_abs_error_diagnostic + if largest_diagnostic is not None: + lines.append( + "Largest final endpoint error: " + f"{largest_diagnostic.data_path}={largest_diagnostic.final_abs_error}" + ) lines.append("Metrics by max absolute error:") for metric in run.metrics_by_max_abs_error(): lines.append( f" - {metric.data_path}: max_abs_error={metric.max_abs_error}, " f"final_abs_error={metric.final_abs_error}" ) + lines.append("Endpoint diagnostics by final absolute error:") + for diagnostic in run.diagnostics_by_final_abs_error(): + lines.append( + f" - {diagnostic.data_path}: " + f"initial_python={diagnostic.initial_python_value}, " + f"initial_amesim={diagnostic.initial_amesim_value}, " + f"final_python={diagnostic.final_python_value}, " + f"final_amesim={diagnostic.final_amesim_value}, " + f"final_abs_error={diagnostic.final_abs_error}" + ) return "\n".join(lines) + "\n" diff --git a/tests/test_run_test_mql_full_state_comparison.py b/tests/test_run_test_mql_full_state_comparison.py index fcb39f9..55981f3 100644 --- a/tests/test_run_test_mql_full_state_comparison.py +++ b/tests/test_run_test_mql_full_state_comparison.py @@ -50,11 +50,24 @@ class RunTestMqlFullStateComparisonScriptTests(unittest.TestCase): run.metrics_by_max_abs_error()[0].data_path, "press@pn_c1_8", ) + diagnostic = run.signal_diagnostic("press@pn_c1_8") + self.assertEqual(diagnostic.data_path, "press@pn_c1_8") + self.assertAlmostEqual(diagnostic.initial_python_value, -1300.0) + self.assertAlmostEqual(diagnostic.final_python_value, -1300.0) + self.assertAlmostEqual(diagnostic.final_abs_error, 0.0, delta=1.0e-8) + self.assertEqual( + run.diagnostics_by_final_abs_error()[0].data_path, + "press@pn_c1_8", + ) self.assertTrue(csv_summary_exists) self.assertIn("Mode: Python 132 full-state closure comparison", summary) self.assertIn("Compared signals: 2", summary) self.assertIn("Largest absolute error: press@pn_c1_8=", summary) + self.assertIn("Largest final endpoint error: press@pn_c1_8=", summary) self.assertIn("Metrics by max absolute error:", summary) + self.assertIn("Endpoint diagnostics by final absolute error:", summary) + self.assertIn("final_python=", summary) + self.assertIn("final_amesim=", summary) self.assertIn("python.press@pn_c1_8", csv_header) self.assertIn("amesim.vol1@pn_brp2_8", csv_header) @@ -102,7 +115,9 @@ class RunTestMqlFullStateComparisonScriptTests(unittest.TestCase): self.assertIn("Output schema signals: 858", summary) self.assertIn("Compared signals: 1", summary) self.assertIn("Largest absolute error: press@pn_c1_8=", summary) + self.assertIn("Largest final endpoint error: press@pn_c1_8=", summary) self.assertIn("Metrics by max absolute error:", summary) + self.assertIn("Endpoint diagnostics by final absolute error:", summary) self.assertTrue(summary.endswith("\n"))