"""Run isolated off/MASS/LSTP/all experiments without altering production. One fresh Amesim execution per profile supplies the common reference. Accuracy runs retain full output. Repeated solve-only runs rotate variant order and exclude compilation/output serialization from integration timing. """ from dataclasses import replace from concurrent.futures import ThreadPoolExecutor from contextlib import nullcontext import argparse import hashlib import json import os from pathlib import Path import statistics import sys import time from unittest.mock import patch ROOT=Path(__file__).resolve().parents[2] sys.path.insert(0,str(ROOT)) from app.main import compile_system_xml_network from app.simulation.backends import simulation_config from app.simulation.native_codegen import build as builder, result_storage from app.simulation.native_codegen.compiler import compile_native_program from app.simulation.native_codegen.input import load_input from app.simulation.native_codegen.runner import execute_native from tests.manual import evaluate_mql8_correctness as evaluation from tests.manual.mechanical_event_variant import event_program,prepare_runtime,MODES def tree_hashes(root): return {str(p.relative_to(root)):hashlib.sha256(p.read_bytes()).hexdigest() for folder in ('native','app/simulation/native_codegen') for p in (root/folder).rglob('*') if p.is_file() and '__pycache__' not in p.parts} def main(): parser=argparse.ArgumentParser(description=__doc__) parser.add_argument('--output',type=Path,required=True) parser.add_argument('--profiles',nargs='+',choices=('full','noncyclic'),default=['full','noncyclic']) parser.add_argument('--repeats',type=int,default=5) parser.add_argument('--resume',action='store_true',help='Resume completed accuracy/timing runs after a transient toolchain failure.') parser.add_argument('--serial-build',action='store_true',help='Serialize experiment compilation to recover Windows compiler-launch failures; solve settings are unchanged.') args=parser.parse_args();out=args.output.resolve();out.mkdir(parents=True,exist_ok=args.resume) if args.repeats<3:raise ValueError('Use at least three repeated integration timings') control=ROOT/'test/mechanical-events-20260917/source-control/native' runtime=out/'experimental-native';prepare_runtime(control,runtime) production=tree_hashes(ROOT) evaluation.save(out/'production-before.json',production) results=json.loads((out/'summary.json').read_bytes()) if args.resume and (out/'summary.json').exists() else {} for profile in args.profiles: if profile in results and all('timing' in r for r in results[profile].values()):continue base=out/profile;base.mkdir(exist_ok=args.resume) ame=ROOT/('tests/data/test_mql.ame' if profile=='full' else 'test/node-fixes-amesim-20260914/test_mql.ame') project=ROOT/'test/output-semantics-20260917/after'/profile/'platform.json' audit=(json.loads((base/'audit/audit.json').read_bytes()) if (base/'audit/audit.json').exists() else evaluation.audit_input(ame,project,base/'audit')) settings=evaluation.PROFILES[profile];stop,step,rtol=settings if not (base/'reference').exists():evaluation.prepare_ame(ame,base/'reference',stop,step,rtol) if args.resume and (base/'reference/run-summary.json').exists(): ame_run=json.loads((base/'reference/run-summary.json').read_bytes()) assert ame_run['normalTermination'] and ame_run['returncode']==0 else: print(profile,'fresh Amesim reference',flush=True) ame_run=evaluation.run_ame(base/'reference',Path('F:/AMESim2404/Amesim')) xml,document=load_input(project);network=compile_system_xml_network(document) raw_project=json.loads(project.read_bytes()) original=compile_native_program(network) builds={};records=json.loads((base/'summary.json').read_bytes()) if args.resume and (base/'summary.json').exists() else {} config=replace(simulation_config(document.simulation),rtol=rtol) try: for mode in MODES: directory=base/mode;directory.mkdir(exist_ok=args.resume);(directory/'platform.json').write_bytes(project.read_bytes()) (directory/'platform.xml').write_bytes(xml) reference=directory/'amesim';reference.mkdir(exist_ok=args.resume) for name in ('test_mql_.results','test_mql_.var'): if not (reference/name).exists():os.link(base/'reference'/name,reference/name) program,metadata=event_program(original,network,mode) evaluation.save(directory/'event-descriptors.json',metadata) assert program.state_keys==original.state_keys assert program.jacobian_structure==original.jacobian_structure assert program.evaluation_schedule==original.evaluation_schedule assert program.source.startswith(original.source) for attempt in range(3): cache=out/('build-cache' if not attempt else f'build-cache-retry-{profile}-{mode}-{time.time_ns()}') try: build_workers=(patch.object(builder,'ThreadPoolExecutor',lambda **kwargs: ThreadPoolExecutor(max_workers=1)) if args.serial_build else nullcontext()) with patch.object(builder,'NATIVE',control if mode=='off' else runtime),patch.object(builder,'CACHE',cache),build_workers: print(profile,mode,'build',attempt+1,flush=True) builds[mode]=builder.build_native(program) break except PermissionError as exc: if getattr(exc,'winerror',None)!=5 or attempt==2:raise print('Windows cache rename denied; retaining artifacts and using a fresh isolated cache.',flush=True) if mode in records and not records[mode].get('failed'): print(profile,mode,'retaining completed accuracy result',flush=True) continue print(profile,mode,'accuracy run',flush=True) with patch.object(result_storage,'RESULT_ROOT',out/'result-storage'): r=execute_native(builds[mode],config,step,run_dir=directory/'native',timeout=180) native={k:v for k,v in r.items() if k not in ('series','final','finalState')} evaluation.save(directory/'native-summary.json',native) if not r['success']: records[mode]=dict(nativeRun=native,failed=True) evaluation.save(base/'summary.json',records) raise RuntimeError(f'{profile}/{mode} failed: '+r['message']) summary=evaluation.compare(directory,raw_project,network,audit,settings) summary.update(nativeRun=native,amesimRun=ame_run,settings=dict(stop=stop,sampleStep=step,rtol=rtol)) records[mode]=summary;evaluation.save(base/'summary.json',records) print(profile,mode,'accuracy done; force=',summary['groups']['force']['worstAbsolute']['maxAbsoluteError'], 'events=',native.get('experimentalEvents',{}),flush=True) timings=json.loads((base/'timings.json').read_bytes()) if args.resume and (base/'timings.json').exists() else {mode:[] for mode in MODES} for repeat in range(args.repeats): order=MODES[repeat%4:]+MODES[:repeat%4] for mode in order: if len(timings[mode])>repeat:continue print(profile,'timing',repeat+1,mode,flush=True) r=execute_native(builds[mode],config,step,record_samples=False, run_dir=base/mode/f'timing-{repeat+1}',timeout=180) if not r['success']:raise RuntimeError(r['message']) selected={k:v for k,v in r.items() if k in ('solveSeconds','solveCpuSeconds','processWallSeconds', 'nfev','acceptedSteps','rejectedSteps','solverStarts','stateTransitions','njev','nlu','experimentalEvents')} timings[mode].append(selected) evaluation.save(base/'timings.json',timings) for mode in MODES: values=[r['solveSeconds'] for r in timings[mode]] records[mode]['timing']=dict(repeats=len(values),medianSeconds=statistics.median(values), minimumSeconds=min(values),maximumSeconds=max(values),runs=timings[mode]) results[profile]=records;evaluation.save(out/'summary.json',results) finally: for build in builds.values():build.close() after=tree_hashes(ROOT) evaluation.save(out/'production-verification.json',dict(unchanged=production==after,before=production,after=after)) assert production==after,'Production code changed during the isolated experiment' print('Completed; production source and existing numerical optimizations unchanged.',flush=True) if __name__=='__main__':main()