"""Summarize one web-cost experiment without reading numerical result files. Usage: .venv/bin/python tests/manual/summarize_web_cost.py --root test/web-cost-20260911 Only small summary/trace/stages/environment JSON files are read. Warmups and control/profiled groups stay separate. This script does not compute speedups. """ from __future__ import annotations import argparse import csv import hashlib import json import math from pathlib import Path from statistics import median from typing import Any REPO = Path(__file__).resolve().parents[2] AXIS = ( ("runClick", "click"), ("fetchStart", "fetch"), ("headers", "headers"), ("lastChunk", "last_chunk"), ("resultParseStart", "parse_start"), ("resultParseEnd", "parse_end"), ("streamEof", "eof"), ("resultReadyDom", "ready"), ("indexedDbCommittedPointer", "commit"), ) # Prune the reporting tree at these declared boundaries. The indexed-read and # process-lifetime spans are atomic ONLY in this coarse HTTP partition; their # measured children remain visible in the separate inclusive span hierarchy. HTTP_BOUNDARIES = { "xml_validation", "network_compilation", "c_generation", "native_build_or_cache_validation", "native_process_lifetime_observed", "native_indexed_result_read", "profile_artifact_preservation", "response_result_json_serialization", } C_WALL_FIELDS = ( "argumentPreparationSeconds", "initializationSeconds", "integrationSeconds", "finalSampleAndStatusSeconds", "projectionSeconds", "jsonWriteSeconds", ) DEFINITIONS = { "scope": "One current experiment; control/profiled are observation modes, not before/after implementations. No speedup is computed.", "statistics": "Median/min/max are calculated from individual runs. Percentages divide each run by its own stated denominator before aggregation. Medians need not add to a median total.", "warmup": "All warmup rows are retained separately and excluded from measured statistics. Cold build is identified by cacheHit=false, not by the warmup label.", "frontendClock": "Browser performance.now() within one trace timeOrigin. Reload/restore has another time axis; no timestamps are subtracted across documents or across browser/backend clocks.", "waterfall": "Adjacent requested marks are subtracted without clipping negatives. Missing marks remain null; commit is the instrumented pointer publication, never replaced by the control polling mark.", "waterfallPercent": "Each adjacent interval / same-run click-to-commit. Negative intervals remain negative and reveal overlapping completion order; they are not exclusive CPU costs.", "backendClock": "Backend spans use request-relative perf_counter_ns. Parent links are inferred by interval containment and indicate inclusive wall intervals, not a traced call stack or exclusive CPU work.", "httpPartition": "Pruned reporting leaves use HTTP_BOUNDARIES, checked for overlap and containment before partitioning. Their union is subtracted from HTTP total to produce unnamed other time. Spawn/read/parse children must not be added again. Response assembly has a duration but no aligned timestamp and remains in other.", "backendOther": "Unclassified HTTP wall intervals include uninstrumented work, scheduling, response assembly, inter-stage gaps and sending. They are not all transport time or CPU work.", "process": "Observed lifetime runs from Popen entry to the first existing poll/wait reporting exit; includes spawn and exit-observation delay. The result processWallSeconds also includes Python result reading. Child CPU is a RUSAGE_CHILDREN delta and assumes no unrelated child is reaped in the same parent interval.", "cInitialization": "The separately measured initialization covers model_init and first sample. CVODE allocation/init/reinit and cleanup are inside integration, along with RHS, Jacobian/linear solve, events and sampling.", "cOutput": "Projection includes final/sample model_eval and allocation. JSON write includes float formatting, stdio, file close and index writing. CPU and wall are separately reported; their difference is not a measured disk-I/O stage.", "cParent": "C phase wall percentages use same-run C main wall; CPU percentages use the same-run observed child user+system CPU. Initial/final/remaining CPU is not individually measured.", "stream": "Read-wait wall overlaps backend production/transport/browser scheduling. Decode and JSON.parse are within headers-to-EOF. NDJSON scanning/join/trim, callbacks, GC and scheduler time are not independently timed.", "persistence": "Save preparation/IndexedDB run in the background with readiness and rendering. Transaction windows include asynchronous waiting; pointer commit is publication after successful writes, control observation uses polling.", "renderExport": "Two requestAnimationFrame callbacks give a paint opportunity, not GPU completion. Download completion includes automation delivery and saveAs. CSV Worker transfer/preparation overlaps Worker activity; finish-post-to-receipt is not isolated Worker CPU.", "unmeasured": "No exclusive breakdown of click preprocessing, NDJSON join/trim, React/GC, CVODE internals, C formatting versus file writes, browser network stack or GPU work is invented. Deeper/native-only experiments are intentionally not read here.", } def number(value: Any) -> bool: return type(value) in (int, float) and math.isfinite(value) def ratio(value: Any, denominator: Any) -> float | None: return 100.0 * value / denominator if number(value) and number(denominator) and denominator > 0 else None def difference(marks: dict, left: str, right: str) -> float | None: a, b = marks.get(left), marks.get(right) return b - a if number(a) and number(b) else None def statistics(values: list[Any]) -> dict: valid = [float(value) for value in values if number(value)] return {"count": len(valid), "missingCount": len(values) - len(valid), "median": median(valid) if valid else None, "min": min(valid) if valid else None, "max": max(valid) if valid else None, "values": values} def union_length(intervals: list[tuple[float, float]]) -> float: total, end = 0.0, -math.inf for start, stop in sorted(intervals): total += max(0.0, stop - max(start, end)) end = max(end, stop) return total def span_hierarchy(raw_spans: list[dict], http_ms: float) -> list[dict]: spans = [{"id": "http", "name": "http_total", "startMs": 0.0, "endMs": http_ms, "durationMs": http_ms, "source": "httpTotalSeconds"}] for index, span in enumerate(raw_spans): start, end = span.get("startMs"), span.get("endMs") if not number(start) or not number(end) or end < start: raise ValueError(f"Invalid backend span: {span}") spans.append({"id": f"span-{index}", "name": span["name"], "startMs": start, "endMs": end, "durationMs": end - start, "recordedSeconds": span.get("seconds"), "source": "spans"}) for current in spans: candidates = [other for other in spans if other["id"] != current["id"] and other["startMs"] <= current["startMs"] and other["endMs"] >= current["endMs"] and (other["durationMs"] > current["durationMs"] or other["id"] == "http")] parent = min(candidates, key=lambda s: s["durationMs"]) if current["id"] != "http" and candidates else None current["parentId"] = parent["id"] if parent else None current["parentName"] = parent["name"] if parent else None current["percentOfParent"] = ratio(current["durationMs"], parent["durationMs"]) if parent else None current["percentOfHttp"] = ratio(current["durationMs"], http_ms) current["inclusion"] = "inclusive interval; do not add its children" for current in spans: children = [s for s in spans if s["parentId"] == current["id"]] current["children"] = [s["id"] for s in children] current["uncoveredByDirectChildrenMs"] = current["durationMs"] - union_length( [(s["startMs"], s["endMs"]) for s in children]) current["partiallyOverlaps"] = [s["id"] for s in spans if s["id"] != current["id"] and max(s["startMs"], current["startMs"]) < min(s["endMs"], current["endMs"]) and not (s["startMs"] <= current["startMs"] and s["endMs"] >= current["endMs"]) and not (current["startMs"] <= s["startMs"] and current["endMs"] >= s["endMs"])] return spans class Summary: def __init__(self, root: Path): self.root = root.resolve() self.sources: dict[str, dict] = {} self.observations: list[dict] = [] self.warnings: list[str] = [] def read(self, path: Path) -> dict: path = path.resolve() path.relative_to(self.root) size = path.stat().st_size if size > 4 * 1024 * 1024: raise ValueError(f"Expected small metadata JSON, refusing {path} ({size} bytes)") raw = path.read_bytes() self.sources[str(path.relative_to(self.root))] = { "sha256": hashlib.sha256(raw).hexdigest(), "bytes": len(raw)} return json.loads(raw) def observe(self, run: dict, domain: str, metric: str, value: Any, unit: str = "ms", *, parent: str = "", denominator: Any = None, inclusion: str = "inclusive or overlapping; not additive", source: str = "") -> None: if value is not None and not number(value): raise ValueError(f"Non-numeric metric {domain}.{metric}: {value!r}") self.observations.append({"group": run["group"], "phase": run["phase"], "run": run["run"], "simulationId": run["simulationId"], "domain": domain, "metric": metric, "value": value, "unit": unit, "parent": parent, "denominatorValue": denominator if number(denominator) else None, "percentOfParent": ratio(value, denominator), "inclusion": inclusion, "source": source}) def backend(self, run: dict, data: dict, source: str) -> dict: if data.get("id") != run["simulationId"]: raise ValueError(f"Simulation ID mismatch in {source}") http_ms = data["httpTotalSeconds"] * 1000 hierarchy = span_hierarchy(data.get("spans", []), http_ms) by_name: dict[str, list[dict]] = {} for span in hierarchy: by_name.setdefault(span["name"], []).append(span) for name, occurrences in by_name.items(): parent_names = sorted({s["parentName"] or "" for s in occurrences}) self.observe(run, "backend_spans", name, sum(s["durationMs"] for s in occurrences), parent="http_total", denominator=http_ms, source=source, inclusion=f"inclusive sum of {len(occurrences)} call(s); interval parents: {', '.join(parent_names)}") chosen = [s for s in hierarchy if s["name"] in HTTP_BOUNDARIES] intervals = [(s["startMs"], s["endMs"]) for s in chosen] covered = union_length(intervals) overlaps = sum(stop - start for start, stop in intervals) - covered outside = [s["id"] for s in chosen if s["startMs"] < 0 or s["endMs"] > http_ms] partition_ok = overlaps <= 1e-6 and not outside partition = {"valid": partition_ok, "scope": "http_total", "totalMs": http_ms, "selectedSpanIds": [s["id"] for s in chosen], "measuredUnionMs": covered, "overlapMs": overlaps, "outsideHttpSpanIds": outside, "otherMs": http_ms - covered if not outside else None, "note": DEFINITIONS["httpPartition"], "segments": []} if partition_ok: partition_durations: dict[str, float] = {} for span in sorted(chosen, key=lambda s: s["startMs"]): partition["segments"].append({"metric": span["name"], "startMs": span["startMs"], "endMs": span["endMs"], "durationMs": span["durationMs"], "percentOfHttp": ratio(span["durationMs"], http_ms)}) partition_durations[span["name"]] = partition_durations.get(span["name"], 0) + span["durationMs"] for name, duration in partition_durations.items(): self.observe(run, "http_partition", name, duration, parent="http_total", denominator=http_ms, inclusion="non-overlapping at declared reporting depth", source=source) self.observe(run, "http_partition", "other_unclassified", partition["otherMs"], parent="http_total", denominator=http_ms, inclusion=DEFINITIONS["backendOther"], source=source) else: self.warnings.append(f"{run['simulationId']}: HTTP partition disabled; selected spans overlap or leave request bounds") for key, value in data.items(): if key.endswith("Ms") or key in ("responseBodyBytes", "rawSeriesBytes", "xmlBytes", "sampleCount"): self.observe(run, "backend_metrics", key, value, "ms" if key.endswith("Ms") else "bytes" if key.endswith("Bytes") else "count", source=source) elif key == "responseSendAwaitSeconds": self.observe(run, "backend_metrics", key, value * 1000, inclusion="ASGI send waits overlap HTTP and are not pure network time", source=source) phases = data.get("existingPerformance", {}).get("phases", {}) for key, phase in phases.items(): self.observe(run, "backend_existing_performance", key, phase["inclusiveNs"] / 1e6, parent="http_total", denominator=http_ms, source=source, inclusion="inclusive duration without aligned start/end; excluded from HTTP partition") c = dict(data.get("nativeStages", {})) native = data.get("native", {}) c_main = c.get("mainTotalSeconds") for key in C_WALL_FIELDS: value = c.get(key) self.observe(run, "c_wall", key, value * 1000 if number(value) else None, parent="c_main_wall", denominator=c_main * 1000 if number(c_main) else None, inclusion="sequential C phase wall duration; C main lies within observed process lifetime", source=source) self.observe(run, "c_wall", "mainTotalSeconds", c_main * 1000 if number(c_main) else None, inclusion="inclusive C main; does not include loader/exit observation", source=source) c_other = c_main - sum(c[key] for key in C_WALL_FIELDS) if number(c_main) and all(number(c.get(k)) for k in C_WALL_FIELDS) else None self.observe(run, "c_wall", "other_unclassified", c_other * 1000 if number(c_other) else None, parent="c_main_wall", denominator=c_main * 1000 if number(c_main) else None, inclusion="C main minus sequential measured wall phases, without clipping negative differences", source=source) process = dict(data.get("process", {})) child_cpu = (process["childrenUserCpuSeconds"] + process["childrenSystemCpuSeconds"] if all(number(process.get(k)) for k in ("childrenUserCpuSeconds", "childrenSystemCpuSeconds")) else None) for key, value in {"integrationCpuSeconds": native.get("solveCpuSeconds"), "projectionCpuSeconds": c.get("projectionCpuSeconds"), "jsonWriteCpuSeconds": c.get("jsonWriteCpuSeconds")}.items(): self.observe(run, "c_cpu", key, value * 1000 if number(value) else None, parent="observed_child_cpu", denominator=child_cpu * 1000 if number(child_cpu) else None, inclusion="CPU duration, separate from wall partition", source=source) for key in ("startMs", "exitObservedMs", "childrenUserCpuSeconds", "childrenSystemCpuSeconds"): value = process.get(key) self.observe(run, "process", key, value * 1000 if number(value) and key.endswith("Seconds") else value, inclusion=DEFINITIONS["process"], source=source) self.observe(run, "process", "observed_lifetime", difference(process, "startMs", "exitObservedMs"), inclusion="includes spawn; excludes subsequent Python result read", source=source) self.observe(run, "process", "observed_child_cpu", child_cpu * 1000 if number(child_cpu) else None, inclusion=DEFINITIONS["process"], source=source) return {"source": source, "httpTotalMs": http_ms, "spanHierarchy": hierarchy, "httpPartition": partition, "cStages": c, "process": process, "build": data.get("build"), "native": native, "existingPerformance": data.get("existingPerformance")} def group(self, mode: str, expected_runs: int) -> dict: folder = self.root / f"browser-{mode}" source = folder / "summary.json" summary = self.read(source) rows = summary.get("rows", []) seen: set[str] = set() runs = [] for original in rows: if original.get("mode") != mode: raise ValueError(f"Unexpected mode {original.get('mode')!r} in {source}") sid = original["simulationId"] if sid in seen: raise ValueError(f"Duplicate simulationId {sid} in {source}") seen.add(sid) run = {"group": mode, "phase": "warmup" if original.get("warmup") else "measured", "run": original["run"], "simulationId": sid, "original": original} trace_path = folder / original["artifacts"] / "trace.json" trace = self.read(trace_path) if trace.get("timeOrigin") != original.get("timeOrigin"): raise ValueError(f"Browser timeOrigin mismatch: {trace_path}") marks = trace.get("marks", {}) if mode == "profiled": request_ids = {r.get("simulationId") for r in trace.get("requests", []) if r.get("kind") == "simulation"} if request_ids != {sid}: raise ValueError(f"Browser trace request ID mismatch: {trace_path}") total = difference(marks, "runClick", "indexedDbCommittedPointer") waterfall = [] for (left, left_name), (right, right_name) in zip(AXIS, AXIS[1:]): value = difference(marks, left, right) metric = f"{left_name}_to_{right_name}" entry = {"metric": metric, "fromMark": left, "toMark": right, "startMs": marks.get(left), "endMs": marks.get(right), "durationMs": value, "percentOfClickToCommit": ratio(value, total), "negative": value is not None and value < 0} waterfall.append(entry) self.observe(run, "frontend_waterfall", metric, value, parent="click_to_commit", denominator=total, source=str(trace_path.relative_to(self.root)), inclusion=DEFINITIONS["waterfall"]) complete = all(number(part["durationMs"]) for part in waterfall) if complete and not math.isclose(sum(p["durationMs"] for p in waterfall), total, abs_tol=1e-6): raise ValueError(f"Waterfall does not telescope for {sid}") run["frontend"] = {"timeOrigin": trace["timeOrigin"], "marks": marks, "waterfall": waterfall, "clickToCommitMs": total, "waterfallComplete": complete, "negativeIntervals": [part["metric"] for part in waterfall if part["negative"]]} for key, value in original.items(): if key in ("run", "timeOrigin") or isinstance(value, bool): continue if number(value) or value is None: unit = "ms" if key.endswith("Ms") else "bytes" if key.endswith("Bytes") else "count" if key.endswith("Count") else "number" self.observe(run, "frontend_metrics", key, value, unit, source=str(source.relative_to(self.root))) native = original.get("native", {}) for key, value in native.items(): if number(value): unit = "ms" if key.endswith("Seconds") else "simulated_s" if key in ("simulatedUntil", "maxAcceptedStep") else "count" self.observe(run, "native_reported", key, value * 1000 if key.endswith("Seconds") else value, unit, source=str(source.relative_to(self.root)), inclusion="original browser native diagnostic; processWallSeconds includes result read") run["buildCost"] = {"cacheHit": native.get("cacheHit"), "buildKey": native.get("buildKey"), "reportedSeconds": native.get("buildSeconds"), "classification": "cache_hit" if native.get("cacheHit") is True else "cold_build" if native.get("cacheHit") is False else "unknown"} self.observe(run, "build", run["buildCost"]["classification"], native["buildSeconds"] * 1000 if number(native.get("buildSeconds")) else None, source=str(source.relative_to(self.root)), inclusion="warmups separate; cacheHit determines cold/cache classification") backend_path = self.root / "backend-profiled/requests" / sid / "stages.json" if mode == "profiled" and backend_path.exists(): backend_data = self.read(backend_path) run["backend"] = self.backend(run, backend_data, str(backend_path.relative_to(self.root))) for key in ("buildKey", "nfev", "acceptedSteps", "solveSeconds"): if backend_data.get("native", {}).get(key) != native.get(key): raise ValueError(f"Browser/backend native diagnostic mismatch for {sid}: {key}") else: run["backend"] = None if mode == "profiled": self.warnings.append(f"{sid}: completed browser row has no backend stages.json yet") runs.append(run) measured = sum(r["phase"] == "measured" for r in runs) if measured != expected_runs: self.warnings.append(f"{mode}: {measured} measured run(s), expected {expected_runs}") if summary.get("errors"): self.warnings.append(f"{mode}: browser summary contains errors; inspect source before interpreting results") environment = self.root / f"backend-{mode}" / "environment.json" return {"source": str(source.relative_to(self.root)), "browser": summary.get("browser"), "node": summary.get("node"), "input": summary.get("input"), "inputSha256": summary.get("inputSha256"), "buildAssetSetSha256": summary.get("buildAssetSetSha256"), "servedAssets": summary.get("servedAssets"), "sourceDefinitions": summary.get("definitions"), "initialNavigation": summary.get("initialNavigation"), "backendEnvironment": self.read(environment) if environment.exists() else None, "measuredRunCount": measured, "warmupRunCount": len(runs) - measured, "runs": runs} def aggregate(self) -> list[dict]: buckets: dict[tuple, list[dict]] = {} for observation in self.observations: key = tuple(observation[k] for k in ("group", "phase", "domain", "metric", "unit", "parent")) buckets.setdefault(key, []).append(observation) result = [] for key, rows in sorted(buckets.items()): entry = dict(zip(("group", "phase", "domain", "metric", "unit", "parent"), key)) entry.update(statistics([r["value"] for r in rows])) entry["percentOfParent"] = statistics([r["percentOfParent"] for r in rows]) entry["runs"] = [{"run": r["run"], "simulationId": r["simulationId"]} for r in rows] result.append(entry) return result def write(self, expected_runs: int) -> dict: groups = {mode: self.group(mode, expected_runs) for mode in ("control", "profiled")} if groups["control"]["inputSha256"] != groups["profiled"]["inputSha256"]: raise ValueError("Control/profiled input SHA mismatch") if groups["control"]["buildAssetSetSha256"] != groups["profiled"]["buildAssetSetSha256"]: self.warnings.append("Control/profiled frontend asset sets differ") aggregates = self.aggregate() build_costs = [{"group": mode, "run": r["run"], "phase": r["phase"], "simulationId": r["simulationId"], **r["buildCost"]} for mode, group in groups.items() for r in group["runs"]] output = {"schemaVersion": 1, "experimentRoot": str(self.root), "script": str(Path(__file__).resolve().relative_to(REPO)), "scriptSha256": hashlib.sha256(Path(__file__).read_bytes()).hexdigest(), "definitions": DEFINITIONS, "warnings": self.warnings, "sourceFiles": self.sources, "groups": groups, "observations": self.observations, "aggregates": aggregates, "complete": not self.warnings, "buildCosts": build_costs, "warmupBuildCosts": [cost for cost in build_costs if cost["phase"] == "warmup"], "coldBuildCosts": [cost for cost in build_costs if cost["classification"] == "cold_build"]} (self.root / "summary.json").write_text(json.dumps(output, ensure_ascii=False, indent=2, allow_nan=False) + "\n") fields = ["group", "phase", "run", "simulationId", "domain", "metric", "statistic", "value", "unit", "parent", "denominatorValue", "percentOfParent", "count", "missingCount", "inclusion", "source"] with (self.root / "timings.csv").open("w", encoding="utf-8", newline="") as stream: writer = csv.DictWriter(stream, fieldnames=fields) writer.writeheader() for row in self.observations: writer.writerow({**row, "statistic": "run", "count": int(number(row["value"])), "missingCount": int(row["value"] is None)}) for entry in aggregates: for stat in ("median", "min", "max"): writer.writerow({**{key: entry[key] for key in ("group", "phase", "domain", "metric", "unit", "parent")}, "statistic": stat, "value": entry[stat], "percentOfParent": entry["percentOfParent"][stat], "count": entry["count"], "missingCount": entry["missingCount"], "inclusion": "values and same-run percentages aggregated separately; do not add medians"}) return output def main() -> None: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--root", type=Path, default=REPO / "test/web-cost-20260911") parser.add_argument("--expected-runs", type=int, default=3) args = parser.parse_args() if args.expected_runs < 1: parser.error("--expected-runs must be positive") summary = Summary(args.root).write(args.expected_runs) print(json.dumps({"root": summary["experimentRoot"], "complete": summary["complete"], "runs": {mode: {"measured": value["measuredRunCount"], "warmup": value["warmupRunCount"]} for mode, value in summary["groups"].items()}, "observations": len(summary["observations"]), "warnings": summary["warnings"]}, ensure_ascii=False, indent=2)) if __name__ == "__main__": main()