245 lines
8.5 KiB
Python
245 lines
8.5 KiB
Python
from __future__ import annotations
|
|
|
|
import argparse
|
|
import hashlib
|
|
import importlib
|
|
import json
|
|
import os
|
|
import platform
|
|
import statistics
|
|
import sys
|
|
from datetime import UTC, datetime
|
|
from math import ceil
|
|
from pathlib import Path
|
|
from time import perf_counter_ns, process_time_ns
|
|
from typing import Any
|
|
|
|
|
|
def _named_value(value: str, *, option: str) -> tuple[str, str]:
|
|
name, separator, target = value.partition("=")
|
|
if not separator or not name.strip() or not target.strip():
|
|
raise ValueError(
|
|
f"{option} must use NAME=VALUE syntax, received {value!r}."
|
|
)
|
|
return name.strip(), target.strip()
|
|
|
|
|
|
def _percentile(values: list[float], percentile: float) -> float:
|
|
ordered = sorted(values)
|
|
index = max(0, min(len(ordered) - 1, ceil(percentile * len(ordered)) - 1))
|
|
return ordered[index]
|
|
|
|
|
|
def _duration_summary(values: list[float]) -> dict[str, object]:
|
|
return {
|
|
"samplesMs": values,
|
|
"minimumMs": min(values),
|
|
"medianMs": statistics.median(values),
|
|
"p95Ms": _percentile(values, 0.95),
|
|
"maximumMs": max(values),
|
|
}
|
|
|
|
|
|
def _load_factory_xml(specification: str) -> bytes:
|
|
module_name, separator, member_name = specification.partition(":")
|
|
if not separator or not module_name or not member_name:
|
|
raise ValueError(
|
|
"Factory specifications must use module.path:callable syntax."
|
|
)
|
|
factory = getattr(importlib.import_module(module_name), member_name)
|
|
value = factory()
|
|
if isinstance(value, bytes):
|
|
return value
|
|
if isinstance(value, str):
|
|
return value.encode("utf-8")
|
|
|
|
from app.main import build_reactflow_system_xml
|
|
|
|
return build_reactflow_system_xml(value)
|
|
|
|
|
|
def _serialize_result_event(result: dict[str, object]) -> bytes:
|
|
"""Render the final NDJSON payload shape used by the streaming endpoint."""
|
|
|
|
status = str(result.get("status", "completed"))
|
|
event = {
|
|
"event": "result",
|
|
"progress": 100 if status == "completed" else 0,
|
|
"phase": status,
|
|
"message": "仿真完成" if status == "completed" else "仿真任务结束",
|
|
"simulatedTime": result.get("simulatedUntil"),
|
|
"totalTime": result.get("requestedStopTime"),
|
|
"result": result,
|
|
}
|
|
return (
|
|
json.dumps(event, ensure_ascii=False, separators=(",", ":")) + "\n"
|
|
).encode("utf-8")
|
|
|
|
|
|
def _run_case(
|
|
name: str,
|
|
xml_bytes: bytes,
|
|
*,
|
|
warmups: int,
|
|
runs: int,
|
|
cancellable_path: bool,
|
|
allow_failures: bool,
|
|
) -> dict[str, object]:
|
|
from app.main import run_system_xml_simulation
|
|
|
|
cancel_check = (lambda: False) if cancellable_path else None
|
|
for _ in range(warmups):
|
|
result = run_system_xml_simulation(xml_bytes, cancel_check=cancel_check)
|
|
if not bool(result.get("success")) and not allow_failures:
|
|
raise RuntimeError(f"Warmup for {name!r} failed: {result.get('message')}")
|
|
|
|
wall_samples_ms: list[float] = []
|
|
cpu_samples_ms: list[float] = []
|
|
serialization_samples_ms: list[float] = []
|
|
serialized_sizes: list[int] = []
|
|
profiles: list[dict[str, object]] = []
|
|
final_result: dict[str, object] | None = None
|
|
for _ in range(runs):
|
|
wall_start = perf_counter_ns()
|
|
cpu_start = process_time_ns()
|
|
result = run_system_xml_simulation(xml_bytes, cancel_check=cancel_check)
|
|
cpu_samples_ms.append((process_time_ns() - cpu_start) / 1_000_000.0)
|
|
wall_samples_ms.append((perf_counter_ns() - wall_start) / 1_000_000.0)
|
|
if not bool(result.get("success")) and not allow_failures:
|
|
raise RuntimeError(f"Benchmark for {name!r} failed: {result.get('message')}")
|
|
diagnostics = result.get("diagnostics")
|
|
if isinstance(diagnostics, dict):
|
|
performance = diagnostics.get("performance")
|
|
if isinstance(performance, dict):
|
|
profiles.append(performance)
|
|
serialization_start = perf_counter_ns()
|
|
serialized_event = _serialize_result_event(result)
|
|
serialization_samples_ms.append(
|
|
(perf_counter_ns() - serialization_start) / 1_000_000.0
|
|
)
|
|
serialized_sizes.append(len(serialized_event))
|
|
final_result = result
|
|
|
|
assert final_result is not None
|
|
return {
|
|
"name": name,
|
|
"success": bool(final_result.get("success")),
|
|
"message": final_result.get("message"),
|
|
"inputBytes": len(xml_bytes),
|
|
"inputSha256": hashlib.sha256(xml_bytes).hexdigest(),
|
|
"status": final_result.get("status"),
|
|
"simulatedUntil": final_result.get("simulatedUntil"),
|
|
"requestedStopTime": final_result.get("requestedStopTime"),
|
|
"wall": _duration_summary(wall_samples_ms),
|
|
"cpu": _duration_summary(cpu_samples_ms),
|
|
"resultSerialization": _duration_summary(serialization_samples_ms),
|
|
"resultEventBytes": serialized_sizes,
|
|
"performanceRuns": profiles,
|
|
}
|
|
|
|
|
|
def _parse_arguments(argv: list[str] | None = None) -> argparse.Namespace:
|
|
parser = argparse.ArgumentParser(
|
|
description="Benchmark the real System XML simulation path with optional profiling."
|
|
)
|
|
parser.add_argument(
|
|
"--mode",
|
|
choices=("off", "standard", "audit"),
|
|
default="audit",
|
|
help="Instrumentation depth selected before importing the simulation modules.",
|
|
)
|
|
parser.add_argument("--warmups", type=int, default=1)
|
|
parser.add_argument("--runs", type=int, default=5)
|
|
parser.add_argument(
|
|
"--xml",
|
|
action="append",
|
|
default=[],
|
|
metavar="NAME=PATH",
|
|
help="Add an XML file benchmark case.",
|
|
)
|
|
parser.add_argument(
|
|
"--factory",
|
|
action="append",
|
|
default=[],
|
|
metavar="NAME=MODULE:CALLABLE",
|
|
help="Add a zero-argument factory returning XML or ReactFlowProjectPayload.",
|
|
)
|
|
parser.add_argument(
|
|
"--direct-path",
|
|
action="store_true",
|
|
help="Do not pass a cancel callback; use the one-shot SciPy path when eligible.",
|
|
)
|
|
parser.add_argument(
|
|
"--disable-property-cache",
|
|
action="store_true",
|
|
help="Disable the run-local exact property cache for an A/B comparison.",
|
|
)
|
|
parser.add_argument(
|
|
"--allow-failures",
|
|
action="store_true",
|
|
help="Record failed simulation runs instead of aborting the benchmark.",
|
|
)
|
|
parser.add_argument("--output", type=Path)
|
|
arguments = parser.parse_args(argv)
|
|
if arguments.warmups < 0:
|
|
parser.error("--warmups must not be negative.")
|
|
if arguments.runs <= 0:
|
|
parser.error("--runs must be positive.")
|
|
if not arguments.xml and not arguments.factory:
|
|
parser.error("At least one --xml or --factory case is required.")
|
|
return arguments
|
|
|
|
|
|
def main(argv: list[str] | None = None) -> int:
|
|
arguments = _parse_arguments(argv)
|
|
os.environ["SIMULATIONAPP_PROFILE"] = arguments.mode
|
|
os.environ["SIMULATIONAPP_PROPERTY_CACHE"] = (
|
|
"off" if arguments.disable_property_cache else "on"
|
|
)
|
|
|
|
cases: list[tuple[str, bytes]] = []
|
|
for raw_case in arguments.xml:
|
|
name, raw_path = _named_value(raw_case, option="--xml")
|
|
cases.append((name, Path(raw_path).read_bytes()))
|
|
for raw_case in arguments.factory:
|
|
name, specification = _named_value(raw_case, option="--factory")
|
|
cases.append((name, _load_factory_xml(specification)))
|
|
|
|
report: dict[str, Any] = {
|
|
"generatedAt": datetime.now(UTC).isoformat(),
|
|
"profileMode": arguments.mode,
|
|
"cancellableSolverPath": not arguments.direct_path,
|
|
"propertyCacheEnabled": not arguments.disable_property_cache,
|
|
"allowFailures": bool(arguments.allow_failures),
|
|
"warmups": arguments.warmups,
|
|
"runs": arguments.runs,
|
|
"runtime": {
|
|
"python": sys.version,
|
|
"platform": platform.platform(),
|
|
"processor": platform.processor(),
|
|
},
|
|
"cases": [
|
|
_run_case(
|
|
name,
|
|
xml_bytes,
|
|
warmups=arguments.warmups,
|
|
runs=arguments.runs,
|
|
cancellable_path=not arguments.direct_path,
|
|
allow_failures=arguments.allow_failures,
|
|
)
|
|
for name, xml_bytes in cases
|
|
],
|
|
}
|
|
text = json.dumps(report, ensure_ascii=False, indent=2)
|
|
if arguments.output is not None:
|
|
arguments.output.parent.mkdir(parents=True, exist_ok=True)
|
|
arguments.output.write_text(text + "\n", encoding="utf-8")
|
|
print(f"Performance report written to {arguments.output.resolve()}")
|
|
else:
|
|
print(text)
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|