"""Re-run audited eight-branch inputs against the AME archive's executable. Use project Python 3.12; --output must be a new directory. Only copied time settings change unless --align-cyclic-from-ame is explicitly supplied, which permits only the audited UD00 cyclic flags to change in a new JSON copy. Numeric differences are evidence, not a claim of physical validation. """ from __future__ import annotations import argparse from contextlib import redirect_stdout from dataclasses import replace import hashlib import json import os from pathlib import Path import re import subprocess import sys import tarfile import time from unittest.mock import patch import numpy as np ROOT = Path(__file__).resolve().parents[2] sys.path.insert(0, str(ROOT)) from app.main import compile_system_xml_network from app.simulation.backends import simulation_config from app.simulation.native_codegen import build as builder, result_storage from app.simulation.native_codegen.compiler import compile_native_program from app.simulation.native_codegen.input import load_input from app.simulation.native_codegen.runner import execute_native from tests.manual import mql8_comparison as curves from tests.manual.event_phase_comparison import pair_saved_phases from tools import audit_test_mql8_model as auditor # Report boundaries, not accepted engineering tolerances. Preserve absolute # errors when the reference is small, and report the sensitivity to this choice. EPS = dict(pressure=1., temperature=1e-6, enthalpy_flow=1., mass_flow=1e-6, displacement=1e-9, velocity=1e-6, gap=1e-9, mass=1e-12, volume=1e-12, piston_volume=1e-12, volume_rate=1e-12, chamber_volume_rate=1e-12, volume_work=1., force=1e-6, signal=1e-12) PROFILES = {'default': (10., .01, 1e-8), 'full': (50., .01, 1e-8), 'startup': (.15, .0001, 1e-8), 'startup-refined': (.15, .0001, 1e-10), 'volume-startup': (2e-6, 2e-9, 1e-10), 'noncyclic': (50., .01, 1e-8)} def digest(path): return hashlib.sha256(path.read_bytes()).hexdigest() def save(path, data): path.write_text(json.dumps(data, ensure_ascii=False, indent=2, allow_nan=False) + '\n', encoding='utf-8') def audit_input(ame, project, out, allow_cyclic_mismatch=False): ame, project = ame.resolve(), project.resolve() out.mkdir(parents=True) with patch.object(auditor, 'HERE', out), patch.object(auditor, 'AME', ame), \ patch.object(auditor, 'INPUT', project), patch.object(sys, 'argv', ['audit', '--check']), \ (out / 'audit.log').open('w', encoding='utf-8') as log, redirect_stdout(log): try: auditor.main() except AssertionError: if not allow_cyclic_mismatch or not (out / 'audit.json').exists(): raise evidence = json.loads((out / 'audit.json').read_bytes()) changes = evidence['parameterChanges'] if evidence['connectionChanges'] or not changes or not all( row['parameter'] == 'iscyclic' and row['component'].startswith('amesim_ud00_') for row in changes): raise return json.loads((out / 'audit.json').read_bytes()) def prepare_ame(archive_path, target, stop, step, rtol): target.mkdir() hashes = {} with tarfile.open(archive_path) as archive: for member in archive: if not member.isfile() or member.name.endswith(('.results', '.ameperf')): continue path = (target / member.name).resolve() if not path.is_relative_to(target.resolve()): raise ValueError('Archive path outside target: ' + member.name) path.parent.mkdir(parents=True, exist_ok=True) data = archive.extractfile(member).read() path.write_bytes(data) hashes[member.name] = hashlib.sha256(data).hexdigest() sim = target / 'test_mql_.sim' lines = sim.read_text(encoding='ascii').splitlines() original = lines[0].split() fields = original[:] fields[:5] = ['0', str(stop), str(step), '1e30', str(rtol)] lines[0] = ' '.join(fields) sim.write_text('\n'.join(lines) + '\n', encoding='ascii') assert all(digest(target / name) == value for name, value in hashes.items() if name != sim.name) save(target / 'source-verification.json', dict(archiveSha256=digest(archive_path), originalSim=original, actualSim=fields, unchangedArchiveFiles=hashes)) def run_ame(target, ame_home): env = dict(os.environ, AME=str(ame_home)) env['PATH'] = str(ame_home / 'win64') + os.pathsep + str(ame_home) + os.pathsep + env['PATH'] start = time.perf_counter() with (target / 'run.log').open('wb') as log: proc = subprocess.run([str(target / 'test_mql_.exe')], cwd=target, env=env, stdout=log, stderr=subprocess.STDOUT, timeout=240, creationflags=subprocess.CREATE_NO_WINDOW if os.name == 'nt' else 0) elapsed = time.perf_counter() - start log = (target / 'run.log').read_text(encoding='utf-8', errors='replace') cpu = re.search(r'Total CPU time:\s*([\d.eE+-]+)', log) result = dict(returncode=proc.returncode, processWallSeconds=elapsed, cpuSeconds=float(cpu.group(1)) if cpu else None, normalTermination='terminated normally' in log) save(target / 'run-summary.json', result) if proc.returncode or not result['normalTermination']: raise RuntimeError(log[-3000:]) result['resultsSha256'] = digest(target / 'test_mql_.results') return result def event_times(project, stop): result = set() for node in project['nodes']: kind, p = node['data']['modelType'], node['data']['parameters'] if kind == 'amesim_step0': result.add(float(p['time'])) elif kind == 'amesim_ud00': durations = [float(p['t' + str(i)]) for i in range(1, int(p['nstages']) + 1)] period = sum(durations) cycles = range(int(stop / period) + 1) if int(p['iscyclic']) else range(1) for cycle in cycles: at = float(p['tstart']) + cycle * period result.add(at) for duration in durations: at += duration result.add(at) return sorted(t for t in result if 0 < t <= stop) def metric(actual, expected, grid, quantity, exact_events, quiet): error = actual - expected absolute = np.abs(error) nonzero = np.abs(expected) > EPS[quantity] relative = np.zeros_like(error) relative[nonzero] = 100 * error[nonzero] / expected[nonzero] bad = nonzero & (np.abs(relative) > 5) near = ~nonzero index = int(np.argmax(absolute)) def maximum(values, mask): return float(np.max(values[mask])) if np.any(mask) else None examples = [] for j in np.flatnonzero(bad)[np.argsort(absolute[bad])[-5:][::-1]]: examples.append(dict(time=float(grid[j]), platform=float(actual[j]), amesim=float(expected[j]), absoluteError=float(absolute[j]), relativePercent=float(relative[j]), atSignalEvent=bool(exact_events[j]))) return dict(maxAbsoluteError=float(absolute[index]), worstTime=float(grid[index]), platformAtWorst=float(actual[index]), amesimAtWorst=float(expected[index]), rmse=float(np.sqrt(np.mean(error**2))), finalError=float(error[-1]), referencePeak=float(np.max(np.abs(expected))), maxRelativePercent=maximum(np.abs(relative), nonzero), maxRelativeOutsideEvents=maximum(np.abs(relative), nonzero & ~exact_events), maxAbsoluteOutsideEvents=maximum(absolute, ~exact_events), quietMaxAbsolute=maximum(absolute, quiet), nonzeroCount=int(nonzero.sum()), above5PercentCount=int(bad.sum()), above5PercentOutsideEvents=int((bad & ~exact_events).sum()), nearZeroCount=int(near.sum()), nearZeroMaxAbsolute=maximum(absolute, near), nearZeroBeyondEpsilon=int((near & (absolute > EPS[quantity])).sum()), epsilon=EPS[quantity], examples=examples, relativeScreenSensitivity={str(factor): int(((np.abs(expected) > EPS[quantity] * factor) & (absolute > .05 * np.abs(expected))).sum()) for factor in (.1, 1., 10.)}) def sample_grid(times, values, grid, step): """Use the actual saved value at the same nominal output-grid position. Repeated floating-point additions move an Amesim output timestamp slightly off its nominal grid. Interpolating 1e17 -> 49000 immediately before an almost-equal endpoint invents a plateau error through cancellation. Match only within 1e-7 of one output interval; retain that record's original side of an event. Do not move event times, average duplicates, or smooth spikes. """ times = np.asarray(times) right = np.clip(np.searchsorted(times, grid), 0, len(times) - 1) left = np.maximum(right - 1, 0) closest = np.where(np.abs(times[left] - grid) < np.abs(times[right] - grid), left, right) same_output = np.abs(times[closest] - grid) <= step * 1e-7 result = np.interp(grid, times, values) result[same_output] = np.asarray(values)[closest[same_output]] return result def compare(directory, project, network, audit, settings): stop, step, rtol = settings for node in project['nodes']: if node['data']['modelType'] == 'amesim_ud00': p = node['data']['parameters'] if any(float(p['start'+str(i)]) != float(p['end'+str(i)]) for i in range(1, int(p['nstages'])+1)): raise ValueError('This baseline phase matcher requires piecewise-constant UD00 stages; ramps need explicit stage metadata.') curves.configure(audit, network) mapping = curves.curve_mapping() save(directory / 'curve-mapping.json', mapping) raw = json.loads((directory / 'native/result.json').read_bytes()) native = curves.native_curves(raw['series']) ame = curves.read_ame(directory / 'amesim') nt, at = native['time'], np.array(ame.times) assert raw['success'] and raw['simulatedUntil'] == stop assert nt[0] == at[0] == 0 and abs(at[-1] - stop) < 1e-9 and nt[-1] == stop assert np.all(np.diff(nt) > 0) and np.all(np.diff(at) >= 0) assert all(np.isfinite(v).all() for v in native.values()) assert all(np.isfinite(v).all() for v in ame.series_by_data_path.values()) grid = np.arange(round(stop / step) + 1) * step events = event_times(project, stop) exact_events = np.zeros(grid.shape, dtype=bool) quiet = grid >= .1 for event in events: exact_events |= np.abs(grid - event) <= 1e-9 quiet &= np.abs(grid - event) > .0200001 signal_mapping = [m for m in mapping if m['quantity'] == 'signal'] ni, ai, pairing = pair_saved_phases(nt, at, np.column_stack([native[m['key']] for m in signal_mapping]), np.column_stack([curves.ame_curve(ame, m) for m in signal_mapping]), grid, step) valid = (ni >= 0) & (ai >= 0) save(directory / 'phase-pairing.json', dict( policy='Reference saved forcing phase; one shared row pair for all curves; no cross-event interpolation', scope='Observed STEP/UD00 forcing phases; unregistered contact mode is not certified', signalKeys=[m['key'] for m in signal_mapping], toleranceSeconds=step*1e-7, gridCount=len(grid), pairedCount=int(valid.sum()), unpairedCount=int((~valid).sum()), adjustedCount=sum(p['status'] == 'matched-other-event-side' for p in pairing), records=pairing)) if not np.any(valid): raise ValueError('No matching saved physical phases; see phase-pairing.json') rows, raw_rows, arrays = [], [], {'time': grid, 'phaseMatched': valid} for m in mapping: raw_actual = sample_grid(nt, native[m['key']], grid, step) reference = curves.ame_curve(ame, m) raw_expected = sample_grid(at, reference, grid, step) raw_rows.append(m | metric(raw_actual, raw_expected, grid, m['quantity'], exact_events, quiet)) actual, expected = np.full(len(grid), np.nan), np.full(len(grid), np.nan) actual[valid], expected[valid] = native[m['key']][ni[valid]], reference[ai[valid]] rows.append(m | metric(actual[valid], expected[valid], grid[valid], m['quantity'], exact_events[valid], quiet[valid])) arrays['platform|' + m['key']] = actual arrays['amesim|' + m['key']] = expected np.savez_compressed(directory / 'curves.npz', **arrays) save(directory / 'raw-time-comparison.json', dict(curves=raw_rows, above5PercentCount=sum(r['above5PercentCount'] for r in raw_rows))) groups = {} for quantity in sorted({r['quantity'] for r in rows}): selected = [r for r in rows if r['quantity'] == quantity] groups[quantity] = dict(curveCount=len(selected), worstAbsolute=max(selected, key=lambda r: r['maxAbsoluteError']), worstRelative=max(selected, key=lambda r: r['maxRelativePercent'] or 0), nonzeroCount=sum(r['nonzeroCount'] for r in selected), above5PercentCount=sum(r['above5PercentCount'] for r in selected), above5PercentOutsideEvents=sum(r['above5PercentOutsideEvents'] for r in selected), nearZeroBeyondEpsilon=sum(r['nearZeroBeyondEpsilon'] for r in selected)) mass_keys = [k for k in raw['series'] if k.rsplit('.', 1)[-1] in ('m', 'm1', 'm2')] mass = sum(native[k] for k in mass_keys) pressures = [v for k, v in native.items() if k.rsplit('.', 1)[-1] in ('p', 'p1', 'p2')] temperatures = [v for k, v in native.items() if k.rsplit('.', 1)[-1] in ('T', 'T1', 'T2')] physical = dict(massKeys=mass_keys, initialMassKg=float(mass[0]), maxTotalMassDriftKg=float(np.max(np.abs(mass - mass[0]))), relativeMassDrift=float(np.max(np.abs(mass - mass[0])) / mass[0]), minimumGasMassKg=float(min(np.min(native[k]) for k in mass_keys)), minimumAbsolutePressurePa=float(min(np.min(v) for v in pressures)), minimumTemperatureK=float(min(np.min(v) for v in temperatures)), maximumTemperatureK=float(max(np.max(v) for v in temperatures))) event_samples = [] for event in events: for m in mapping: if m['quantity'] not in ('signal', 'force'): continue entry = dict(eventTime=event, key=m['key']) for label, times, values in [('platform', nt, native[m['key']]), ('amesim', at, curves.ame_curve(ame, m))]: idx = int(np.searchsorted(times, event)) entry[label] = [dict(time=float(times[i]), timeHex=float(times[i]).hex(), value=float(values[i])) for i in range(max(0, idx-2), min(len(times), idx+3))] event_samples.append(entry) save(directory / 'event-samples.json', event_samples) summary = dict(curveCount=len(rows), gridCount=len(grid), nativeSampleCount=len(nt), phaseMatchedGridCount=int(valid.sum()), phaseUnpairedGridCount=int((~valid).sum()), phaseAdjustedGridCount=sum(p['status'] == 'matched-other-event-side' for p in pairing), rawTimeAbove5PercentCount=sum(r['above5PercentCount'] for r in raw_rows), amesimSampleCount=len(at), platformOutputCount=len(raw['series']) - 1, allFinite=True, signalEvents=events, physical=physical, groups=groups, curves=rows, extraEventPointMaxContactForce=max(float(np.max(np.abs(native[m['key']]))) for m in mapping if m['quantity'] == 'force'), above5PercentCurveCount=sum(r['above5PercentCount'] > 0 for r in rows), above5PercentCount=sum(r['above5PercentCount'] for r in rows), above5PercentOutsideEvents=sum(r['above5PercentOutsideEvents'] for r in rows)) save(directory / 'comparison.json', summary) return {k: v for k, v in summary.items() if k != 'curves'} def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('--output', type=Path, required=True) parser.add_argument('--ame', type=Path, default=ROOT / 'tests/data/test_mql.ame') parser.add_argument('--project', type=Path, default=ROOT / 'tests/data/test-mql-8-corrected.json') parser.add_argument('--ame-home', type=Path, required=True) parser.add_argument('--noncyclic-ame', type=Path) parser.add_argument('--noncyclic-project', type=Path) parser.add_argument('--align-cyclic-from-ame', action='store_true') parser.add_argument('--analyze-only', action='store_true', help='Reanalyze existing raw results without solving.') parser.add_argument('--profiles', nargs='+', choices=tuple(PROFILES), default=list(PROFILES)) parser.add_argument('--native-source', type=Path, help='Frozen native source tree for a controlled before/after run.') args = parser.parse_args() if args.native_source: builder.NATIVE = args.native_source.resolve() out = args.output.resolve() if args.analyze_only: summaries = json.loads((out / 'summary.json').read_bytes()) for name, old in list(summaries.items()): directory = out / name project = json.loads((directory / 'platform.json').read_bytes()) _, document = load_input(directory / 'platform.json') audit_name = 'audit-noncyclic' if name == 'noncyclic' else 'audit-aligned' audit = json.loads((out / audit_name / 'audit.json').read_bytes()) summary = compare(directory, project, compile_system_xml_network(document), audit, PROFILES[name]) summary.update({key: old[key] for key in ('settings', 'amesimRun', 'nativeRun')}) summaries[name] = summary print(name, 'reanalyzed', summary['above5PercentCount'], flush=True) save(out / 'summary.json', summaries) manifest = json.loads((out / 'manifest.json').read_bytes()) manifest['interpolation'] = 'Same saved forcing phase within 1e-7 output intervals. Unpaired points explicitly reported; raw time-only metrics retained.' save(out / 'manifest.json', manifest) return out.mkdir(parents=True, exist_ok=False) if bool(args.noncyclic_ame) != bool(args.noncyclic_project): parser.error('Both noncyclic input paths are required together.') inputs = [args.ame, args.project] + ([args.noncyclic_ame, args.noncyclic_project] if args.noncyclic_ame else []) tracked = subprocess.check_output(['git', 'ls-files', 'app', 'native', 'frontend/src'], cwd=ROOT, text=True).splitlines() sources = {str(p.resolve()): digest(p) for p in inputs} code = {p: digest(ROOT / p) for p in tracked if (ROOT / p).is_file()} manifest = dict(gitCommit=subprocess.check_output(['git', 'rev-parse', 'HEAD'], cwd=ROOT, text=True).strip(), python=sys.version, sources=sources, productionSources=code, epsilons=EPS, criterion='5% is a diagnostic screen, not an approved engineering tolerance', freshAmesimExecution=True, amesimExecutable='Extracted from source AME; not recompiled', nativeSource=str(builder.NATIVE), interpolation='Same saved forcing phase within 1e-7 output intervals. Unpaired points explicitly reported; raw time-only metrics retained.') save(out / 'manifest.json', manifest) original_audit = audit_input(args.ame, args.project, out / 'audit-original', args.align_cyclic_from_ame) if original_audit['parameterChanges']: aligned = json.loads(args.project.read_bytes()) nodes = {node['id']: node for node in aligned['nodes']} for row in original_audit['parameterChanges']: nodes[row['component']]['data']['parameters'][row['parameter']] = row['expected'] aligned_path = out / 'aligned-source.json' save(aligned_path, aligned) save(out / 'alignment-changes.json', original_audit['parameterChanges']) args.project = aligned_path audits = {'standard': audit_input(args.ame, args.project, out / 'audit-aligned')} if args.noncyclic_ame: audits['noncyclic'] = audit_input(args.noncyclic_ame, args.noncyclic_project, out / 'audit-noncyclic') summaries = {} for name, settings in PROFILES.items(): if name not in args.profiles: continue if name == 'noncyclic' and not args.noncyclic_ame: continue stop, step, rtol = settings ame_path = args.noncyclic_ame if name == 'noncyclic' else args.ame project_path = args.noncyclic_project if name == 'noncyclic' else args.project audit = audits['noncyclic' if name == 'noncyclic' else 'standard'] directory = out / name directory.mkdir() project = json.loads(project_path.read_bytes()) project['simulation'].update(t_start=0., t_stop=stop, step=step, max_step=1e30, method='BDF') save(directory / 'platform.json', project) prepare_ame(ame_path, directory / 'amesim', stop, step, rtol) print(name, 'Amesim running', flush=True) ame_summary = run_ame(directory / 'amesim', args.ame_home) xml, document = load_input(directory / 'platform.json') (directory / 'platform.xml').write_bytes(xml) network = compile_system_xml_network(document) program = compile_native_program(network) print(name, 'current platform running', flush=True) started = time.perf_counter() # Isolate cache and result quotas from the user's live application. with patch.object(builder, 'CACHE', out / 'build-cache'), \ patch.object(result_storage, 'RESULT_ROOT', out / 'result-storage'): build = builder.build_native(program) try: result = execute_native(build, replace(simulation_config(document.simulation), rtol=rtol), step, run_dir=directory / 'native', timeout=240) native_summary = {k: v for k, v in result.items() if k not in ('series', 'final', 'finalState')} native_summary.update(buildSeconds=build.seconds, buildCacheHit=build.cache_hit, buildKey=build.manifest['buildKey'], pipelineWallSeconds=time.perf_counter()-started, stateKeys=program.state_keys) save(directory / 'native-summary.json', native_summary) finally: build.close() summary = compare(directory, project, network, audit, settings) summary.update(settings=dict(stop=stop, sampleStep=step, rtol=rtol), amesimRun=ame_summary, nativeRun=native_summary) summaries[name] = summary save(out / 'summary.json', summaries) print(name, 'completed', 'pressure=', summary['groups']['pressure']['worstAbsolute']['maxAbsoluteError'], 'temperature=', summary['groups']['temperature']['worstAbsolute']['maxAbsoluteError'], 'above5%=', summary['above5PercentCount'], flush=True) unchanged = all(digest(Path(p)) == h for p, h in sources.items()) unchanged_code = all(digest(ROOT / p) == h for p, h in code.items()) save(out / 'source-verification.json', dict(inputsUnchanged=unchanged, productionUnchanged=unchanged_code, checkedProductionFiles=len(code))) assert unchanged and unchanged_code print('All profiles completed; original inputs and production sources unchanged.', flush=True) if __name__ == '__main__': main()