176 lines
7.9 KiB
Python
176 lines
7.9 KiB
Python
"""Compare exact per-block grants with batched grants on a real filesystem.
|
|
|
|
Run from the repository root:
|
|
py -3.12 -m tests.manual.benchmark_result_reservations --native
|
|
|
|
All result directories are isolated temporary fixtures. History is synthetic;
|
|
the native case also compares every binary byte and returned sample. Timings
|
|
exclude fixture creation, compilation, and final archive validation in the
|
|
reservation-only case. No performance threshold is used in automated tests.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
from pathlib import Path
|
|
import tempfile
|
|
import time
|
|
from unittest.mock import patch
|
|
|
|
from app.simulation.native_codegen import result_storage as storage
|
|
|
|
|
|
def exact_reserve(self, amount):
|
|
"""The pre-optimization path, retained ONLY as a benchmark baseline."""
|
|
if type(amount) is not int or not 0 < amount <= self.limit:
|
|
raise storage.ResultQuotaError('Invalid or oversized result block reservation.')
|
|
with self._locked():
|
|
self._make_room(amount)
|
|
self.data['reservedBytes'] += amount
|
|
self._publish()
|
|
|
|
|
|
def history(root, count, metadata):
|
|
root.mkdir(parents=True)
|
|
for i in range(count):
|
|
path = root / ('run-' + f'{i + 1:064x}')
|
|
path.mkdir()
|
|
data = dict(format='simulation-blocks-v1', id=path.name,
|
|
createdAt=i + 1, finishedAt=i + 1, status='completed',
|
|
reservedBytes=65536, workerPid=None, metadata=metadata)
|
|
(path / 'manifest.json').write_text(json.dumps(data), encoding='utf-8')
|
|
# Only layout/stat accounting matters for historical fixtures.
|
|
(path / 'states.bin').write_bytes(b'\0' * 1024)
|
|
(path / 'outputs.bin').write_bytes(b'\0' * 1024)
|
|
|
|
|
|
class Counters:
|
|
def __init__(self):
|
|
self.scans = self.publishes = self.requests = 0
|
|
self.seconds = 0.
|
|
|
|
def scan(self, archive, extra):
|
|
self.scans += 1
|
|
return self.original_scan(archive, extra)
|
|
|
|
def publish(self, archive):
|
|
self.publishes += 1
|
|
return self.original_publish(archive)
|
|
|
|
def reserve(self, archive, amount):
|
|
self.requests += 1
|
|
start = time.perf_counter()
|
|
try:
|
|
return self.reservation(archive, amount)
|
|
finally:
|
|
self.seconds += time.perf_counter() - start
|
|
|
|
def patches(self, mode):
|
|
from contextlib import ExitStack
|
|
self.original_scan = storage.ResultArchive._make_room
|
|
self.original_publish = storage.ResultArchive._publish
|
|
self.reservation = (exact_reserve if mode == 'exact' else storage.ResultArchive.reserve)
|
|
stack = ExitStack()
|
|
stack.enter_context(patch.object(storage.ResultArchive, '_make_room',
|
|
lambda archive, extra: self.scan(archive, extra)))
|
|
stack.enter_context(patch.object(storage.ResultArchive, '_publish',
|
|
lambda archive: self.publish(archive)))
|
|
stack.enter_context(patch.object(storage.ResultArchive, 'reserve',
|
|
lambda archive, amount: self.reserve(archive, amount)))
|
|
return stack
|
|
|
|
|
|
def reservations(args):
|
|
metadata = {'stateKeys': ['x'], 'variables': [
|
|
{'key': f'component_{i}:pressure', 'name': f'Component {i} pressure', 'unit': 'Pa'}
|
|
for i in range(args.variables)]}
|
|
for count in args.histories:
|
|
for repeat in range(args.repeats):
|
|
for mode in ('exact', 'batched'):
|
|
with tempfile.TemporaryDirectory(prefix='result reservation benchmark ') as temporary:
|
|
root = Path(temporary) / 'simresults'
|
|
history(root, count, metadata)
|
|
archive = storage.ResultArchive(metadata, root=root)
|
|
counters = Counters()
|
|
try:
|
|
with counters.patches(mode):
|
|
start = time.perf_counter()
|
|
for _ in range(args.requests):
|
|
archive.reserve(args.block_kib * 1024)
|
|
elapsed = time.perf_counter() - start
|
|
finally:
|
|
archive.close()
|
|
print(json.dumps(dict(case='reservation-only', history=count, mode=mode,
|
|
repeat=repeat + 1, requests=counters.requests, blockKiB=args.block_kib,
|
|
variables=args.variables, scans=counters.scans, publishes=counters.publishes,
|
|
seconds=round(elapsed, 6))), flush=True)
|
|
|
|
|
|
def native(args):
|
|
from app.simulation.native_codegen.build import build_native
|
|
from app.simulation.native_codegen.compiler import compile_native_program
|
|
from app.simulation.native_codegen.runner import execute_native
|
|
from app.simulation.config import SolveIVPConfig
|
|
from tests.test_native_catalog import Circuit
|
|
|
|
circuit = Circuit()
|
|
mass = circuit.add('amesim_mecmas21', 'mass', mass=1, v0=1, x0=0, useFriction=1, stoptype=4)
|
|
for i in (1, 2):
|
|
free = circuit.add('amesim_f000', 'free' + str(i))
|
|
circuit.connect(mass, 'port_' + str(i), free, 'port_1')
|
|
build = build_native(compile_native_program(circuit.net))
|
|
expected = None
|
|
try:
|
|
for repeat in range(args.repeats):
|
|
for mode in ('exact', 'batched'):
|
|
with tempfile.TemporaryDirectory(prefix='native reservation benchmark ') as temporary:
|
|
root = Path(temporary) / 'simresults'
|
|
history(root, max(args.histories), build.manifest)
|
|
counters = Counters()
|
|
with patch.object(storage, 'RESULT_ROOT', root), counters.patches(mode):
|
|
start = time.perf_counter()
|
|
result = execute_native(build, SolveIVPConfig(t_stop=1, method='RK45'),
|
|
.00001, run_dir=Path(temporary) / 'run')
|
|
elapsed = time.perf_counter() - start
|
|
assert result['success'], result.get('message')
|
|
summary = result['resultStorage']
|
|
assert not summary['states']['corrupt'] and not summary['outputs']['corrupt']
|
|
directory = storage.result_archive_path(summary['id'], root=root)
|
|
hashes = []
|
|
for name in ('states.bin', 'outputs.bin'):
|
|
with (directory / name).open('rb') as stream:
|
|
hashes.append(hashlib.file_digest(stream, 'sha256').hexdigest())
|
|
signature = (hashes, result['series'])
|
|
if expected is None:
|
|
expected = signature
|
|
assert signature == expected, 'native result changed'
|
|
print(json.dumps(dict(case='native-RK45', history=max(args.histories), mode=mode,
|
|
repeat=repeat + 1, requests=counters.requests, scans=counters.scans,
|
|
publishes=counters.publishes, samples=len(result['series']['time']),
|
|
reservationSeconds=round(counters.seconds, 6), seconds=round(elapsed, 6),
|
|
identicalBinaryAndSamples=True)), flush=True)
|
|
finally:
|
|
build.close()
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument('--histories', type=int, nargs='+', default=[0, 100, 300])
|
|
parser.add_argument('--requests', type=int, default=256)
|
|
parser.add_argument('--block-kib', type=int, default=64)
|
|
parser.add_argument('--variables', type=int, default=1000)
|
|
parser.add_argument('--repeats', type=int, default=2)
|
|
parser.add_argument('--native', action='store_true')
|
|
args = parser.parse_args()
|
|
if (min(args.histories) < 0 or min(args.requests, args.block_kib, args.repeats) < 1 or
|
|
args.variables < 0):
|
|
parser.error('counts must be nonnegative; requests, block size and repeats must be positive')
|
|
reservations(args)
|
|
if args.native:
|
|
native(args)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|