"""Compare exact per-block grants with batched grants on a real filesystem. Run from the repository root: py -3.12 -m tests.manual.benchmark_result_reservations --native All result directories are isolated temporary fixtures. History is synthetic; the native case also compares every binary byte and returned sample. Timings exclude fixture creation, compilation, and final archive validation in the reservation-only case. No performance threshold is used in automated tests. """ from __future__ import annotations import argparse import hashlib import json from pathlib import Path import tempfile import time from unittest.mock import patch from app.simulation.native_codegen import result_storage as storage def exact_reserve(self, amount): """The pre-optimization path, retained ONLY as a benchmark baseline.""" if type(amount) is not int or not 0 < amount <= self.limit: raise storage.ResultQuotaError('Invalid or oversized result block reservation.') with self._locked(): self._make_room(amount) self.data['reservedBytes'] += amount self._publish() def history(root, count, metadata): root.mkdir(parents=True) for i in range(count): path = root / ('run-' + f'{i + 1:064x}') path.mkdir() data = dict(format='simulation-blocks-v1', id=path.name, createdAt=i + 1, finishedAt=i + 1, status='completed', reservedBytes=65536, workerPid=None, metadata=metadata) (path / 'manifest.json').write_text(json.dumps(data), encoding='utf-8') # Only layout/stat accounting matters for historical fixtures. (path / 'states.bin').write_bytes(b'\0' * 1024) (path / 'outputs.bin').write_bytes(b'\0' * 1024) class Counters: def __init__(self): self.scans = self.publishes = self.requests = 0 self.seconds = 0. def scan(self, archive, extra): self.scans += 1 return self.original_scan(archive, extra) def publish(self, archive): self.publishes += 1 return self.original_publish(archive) def reserve(self, archive, amount): self.requests += 1 start = time.perf_counter() try: return self.reservation(archive, amount) finally: self.seconds += time.perf_counter() - start def patches(self, mode): from contextlib import ExitStack self.original_scan = storage.ResultArchive._make_room self.original_publish = storage.ResultArchive._publish self.reservation = (exact_reserve if mode == 'exact' else storage.ResultArchive.reserve) stack = ExitStack() stack.enter_context(patch.object(storage.ResultArchive, '_make_room', lambda archive, extra: self.scan(archive, extra))) stack.enter_context(patch.object(storage.ResultArchive, '_publish', lambda archive: self.publish(archive))) stack.enter_context(patch.object(storage.ResultArchive, 'reserve', lambda archive, amount: self.reserve(archive, amount))) return stack def reservations(args): metadata = {'stateKeys': ['x'], 'variables': [ {'key': f'component_{i}:pressure', 'name': f'Component {i} pressure', 'unit': 'Pa'} for i in range(args.variables)]} for count in args.histories: for repeat in range(args.repeats): for mode in ('exact', 'batched'): with tempfile.TemporaryDirectory(prefix='result reservation benchmark ') as temporary: root = Path(temporary) / 'simresults' history(root, count, metadata) archive = storage.ResultArchive(metadata, root=root) counters = Counters() try: with counters.patches(mode): start = time.perf_counter() for _ in range(args.requests): archive.reserve(args.block_kib * 1024) elapsed = time.perf_counter() - start finally: archive.close() print(json.dumps(dict(case='reservation-only', history=count, mode=mode, repeat=repeat + 1, requests=counters.requests, blockKiB=args.block_kib, variables=args.variables, scans=counters.scans, publishes=counters.publishes, seconds=round(elapsed, 6))), flush=True) def native(args): from app.simulation.native_codegen.build import build_native from app.simulation.native_codegen.compiler import compile_native_program from app.simulation.native_codegen.runner import execute_native from app.simulation.config import SolveIVPConfig from tests.test_native_catalog import Circuit circuit = Circuit() mass = circuit.add('amesim_mecmas21', 'mass', mass=1, v0=1, x0=0, useFriction=1, stoptype=4) for i in (1, 2): free = circuit.add('amesim_f000', 'free' + str(i)) circuit.connect(mass, 'port_' + str(i), free, 'port_1') build = build_native(compile_native_program(circuit.net)) expected = None try: for repeat in range(args.repeats): for mode in ('exact', 'batched'): with tempfile.TemporaryDirectory(prefix='native reservation benchmark ') as temporary: root = Path(temporary) / 'simresults' history(root, max(args.histories), build.manifest) counters = Counters() with patch.object(storage, 'RESULT_ROOT', root), counters.patches(mode): start = time.perf_counter() result = execute_native(build, SolveIVPConfig(t_stop=1, method='RK45'), .00001, run_dir=Path(temporary) / 'run') elapsed = time.perf_counter() - start assert result['success'], result.get('message') summary = result['resultStorage'] assert not summary['states']['corrupt'] and not summary['outputs']['corrupt'] directory = storage.result_archive_path(summary['id'], root=root) hashes = [] for name in ('states.bin', 'outputs.bin'): with (directory / name).open('rb') as stream: hashes.append(hashlib.file_digest(stream, 'sha256').hexdigest()) signature = (hashes, result['series']) if expected is None: expected = signature assert signature == expected, 'native result changed' print(json.dumps(dict(case='native-RK45', history=max(args.histories), mode=mode, repeat=repeat + 1, requests=counters.requests, scans=counters.scans, publishes=counters.publishes, samples=len(result['series']['time']), reservationSeconds=round(counters.seconds, 6), seconds=round(elapsed, 6), identicalBinaryAndSamples=True)), flush=True) finally: build.close() def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('--histories', type=int, nargs='+', default=[0, 100, 300]) parser.add_argument('--requests', type=int, default=256) parser.add_argument('--block-kib', type=int, default=64) parser.add_argument('--variables', type=int, default=1000) parser.add_argument('--repeats', type=int, default=2) parser.add_argument('--native', action='store_true') args = parser.parse_args() if (min(args.histories) < 0 or min(args.requests, args.block_kib, args.repeats) < 1 or args.variables < 0): parser.error('counts must be nonnegative; requests, block size and repeats must be positive') reservations(args) if args.native: native(args) if __name__ == '__main__': main()