"""Compare complete native numerical results as binary64, outside benchmark timing. .venv/bin/python tests/manual/compare_native_result_bits.py \ --baseline old/result.json --candidate new/run-1/result.json \ --candidate new/run-2/result.json --output test/result-bit-parity.json JSON number spelling may change. In particular, the integer token -0 must be parsed as negative floating zero before packing. All series columns (including time), final scalars and finalState entries are compared without sampling. Solver status/configuration/counters must also agree. Only solve wall/CPU timing metadata is intentionally ignored. Nonfinite payload or metadata numbers fail. """ from __future__ import annotations import argparse from hashlib import sha256 import json import math from pathlib import Path import struct import sys PAYLOAD_KEYS = ("series", "final", "finalState") FLOAT_METADATA = ("simulatedUntil", "maxAcceptedStep") COUNT_METADATA = ("nfev", "acceptedSteps", "rejectedSteps", "njev", "nlu", "stateTransitions", "solverStarts") VALUE_METADATA = ("success", "status", "message", "method", "backend", "solver", "sundialsVersion") TIMING_METADATA = ("solveSeconds", "solveCpuSeconds") NEGATIVE_ZERO = struct.pack(" None: super().__init__(f"{path}: {reason}") self.detail = {"path": path, "reason": reason, **details} def pointer(*parts: object) -> str: return "/" + "/".join(str(part).replace("~", "~0").replace("/", "~1") for part in parts) def read_result(path: Path) -> tuple[dict, str]: raw = path.read_bytes() def unique_object(items: list[tuple[str, object]]) -> dict: result = {} for key, value in items: if key in result: raise ComparisonError("/", "Duplicate JSON object key", key=key) result[key] = value return result def reject_constant(token: str) -> object: raise ComparisonError("/", "Nonfinite JSON token", token=token) value = json.loads(raw, parse_int=lambda token: -0.0 if token == "-0" else int(token), parse_constant=reject_constant, object_pairs_hook=unique_object) if not isinstance(value, dict): raise ComparisonError("/", "Expected a native result object") return value, sha256(raw).hexdigest() def bits(value: object, path: str) -> bytes: if isinstance(value, bool) or not isinstance(value, (int, float)): raise ComparisonError(path, "Expected a finite numeric value", actualType=type(value).__name__) try: number = float(value) except (OverflowError, ValueError): raise ComparisonError(path, "Number cannot be represented as finite binary64") from None if not math.isfinite(number): raise ComparisonError(path, "Nonfinite binary64 value") return struct.pack(" None: """Reject overflow-to-infinity tokens even in metadata excluded from parity.""" if isinstance(value, dict): for key, child in value.items(): ensure_finite_tree(child, path + pointer(key)) elif isinstance(value, list): for index, child in enumerate(value): ensure_finite_tree(child, path + pointer(index)) elif isinstance(value, (int, float)) and not isinstance(value, bool): bits(value, path or "/") def validate_result(result: dict) -> dict: required = set(PAYLOAD_KEYS + FLOAT_METADATA + COUNT_METADATA + VALUE_METADATA) if missing := required - result.keys(): raise ComparisonError("/", "Missing native result fields", missing=sorted(missing)) if not isinstance(result["series"], dict) or not isinstance(result["final"], dict): raise ComparisonError("/", "series and final must be objects") if not isinstance(result["finalState"], list): raise ComparisonError("/finalState", "Expected an array") for key, values in result["series"].items(): if not isinstance(values, list): raise ComparisonError(pointer("series", key), "Expected a numeric array") if result["series"] and "time" not in result["series"]: raise ComparisonError("/series", "Nonempty series has no time column") if result["series"]: samples = len(result["series"]["time"]) for key, values in result["series"].items(): if len(values) != samples: raise ComparisonError(pointer("series", key), "Column length differs from time", expectedLength=samples, actualLength=len(values)) for key in COUNT_METADATA: value = result[key] if isinstance(value, bool) or not isinstance(value, int) or value < 0: raise ComparisonError(pointer(key), "Expected a nonnegative integer solver counter") if not isinstance(result["success"], bool): raise ComparisonError("/success", "Expected a boolean") for key in VALUE_METADATA[1:]: if not isinstance(result[key], str): raise ComparisonError(pointer(key), "Expected string metadata") ensure_finite_tree(result) for key in FLOAT_METADATA: bits(result[key], pointer(key)) # Scalars in final and all payload cells must be numeric, never bool/null. count = negative_zeroes = positive_zeroes = 0 for path, value in payload_values(result): packed = bits(value, path) count += 1 negative_zeroes += packed == NEGATIVE_ZERO positive_zeroes += packed == POSITIVE_ZERO return {"payloadValues": count, "seriesColumns": len(result["series"]), "samples": len(result["series"].get("time", [])), "finalScalars": len(result["final"]), "finalStateValues": len(result["finalState"]), "negativeZeroValues": negative_zeroes, "positiveZeroValues": positive_zeroes} def payload_values(result: dict): for key, values in result["series"].items(): for index, value in enumerate(values): yield pointer("series", key, index), value for key, value in result["final"].items(): yield pointer("final", key), value for index, value in enumerate(result["finalState"]): yield pointer("finalState", index), value def match_keys(baseline: dict, candidate: dict, path: str) -> None: if baseline.keys() != candidate.keys(): raise ComparisonError(path, "Object key sets differ", missing=sorted(baseline.keys() - candidate.keys()), extra=sorted(candidate.keys() - baseline.keys())) def compare(baseline: dict, candidate: dict) -> dict: # Validate the complete structure before comparing any payload bit patterns. match_keys(baseline, candidate, "/") for key in ("series", "final"): match_keys(baseline[key], candidate[key], pointer(key)) for key, values in baseline["series"].items(): if len(values) != len(candidate["series"][key]): raise ComparisonError(pointer("series", key), "Array lengths differ", baselineLength=len(values), candidateLength=len(candidate["series"][key])) if len(baseline["finalState"]) != len(candidate["finalState"]): raise ComparisonError("/finalState", "Array lengths differ", baselineLength=len(baseline["finalState"]), candidateLength=len(candidate["finalState"])) metadata_comparisons = 0 for key in VALUE_METADATA + COUNT_METADATA: if baseline[key] != candidate[key]: raise ComparisonError(pointer(key), "Solver metadata or counter differs", baseline=baseline[key], candidate=candidate[key]) metadata_comparisons += 1 for key in FLOAT_METADATA: left, right = bits(baseline[key], pointer(key)), bits(candidate[key], pointer(key)) if left != right: raise ComparisonError(pointer(key), "Numeric metadata binary64 bits differ", baselineBitsLE=left.hex(), candidateBitsLE=right.hex()) metadata_comparisons += 1 # Any future top-level metadata field must also agree unless explicitly timed. known = set(PAYLOAD_KEYS + FLOAT_METADATA + COUNT_METADATA + VALUE_METADATA + TIMING_METADATA) for key in baseline.keys() - known: if baseline[key] != candidate[key]: raise ComparisonError(pointer(key), "Additional metadata differs") metadata_comparisons += 1 comparisons = negative_zeroes = positive_zeroes = 0 def compare_number(left_value: object, right_value: object, path: str) -> None: nonlocal comparisons, negative_zeroes, positive_zeroes left, right = bits(left_value, path), bits(right_value, path) if left != right: raise ComparisonError(path, "Payload binary64 bits differ", baselineBitsLE=left.hex(), candidateBitsLE=right.hex(), baselineValue=repr(left_value), candidateValue=repr(right_value), signedZeroMismatch=left in (POSITIVE_ZERO, NEGATIVE_ZERO) and right in (POSITIVE_ZERO, NEGATIVE_ZERO)) comparisons += 1 negative_zeroes += left == NEGATIVE_ZERO positive_zeroes += left == POSITIVE_ZERO # Look up columns by their validated key; JSON object order is immaterial. for key, values in baseline["series"].items(): for index, (left, right) in enumerate(zip(values, candidate["series"][key], strict=True)): compare_number(left, right, pointer("series", key, index)) for key, value in baseline["final"].items(): compare_number(value, candidate["final"][key], pointer("final", key)) for index, (left, right) in enumerate(zip(baseline["finalState"], candidate["finalState"], strict=True)): compare_number(left, right, pointer("finalState", index)) return {"passed": True, "comparisons": comparisons, "metadataComparisons": metadata_comparisons, "negativeZeroComparisons": negative_zeroes, "positiveZeroComparisons": positive_zeroes, "allPayloadBinary64BitsEqual": True} def main() -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--baseline", required=True, type=Path) parser.add_argument("--candidate", required=True, action="append", type=Path) parser.add_argument("--output", required=True, type=Path) args = parser.parse_args() if args.output.resolve() in {args.baseline.resolve(), *(path.resolve() for path in args.candidate)}: parser.error("--output must differ from every input file") report = {"version": 1, "baseline": str(args.baseline.resolve()), "candidates": [], "allPassed": False, "comparisonContract": "Exact finite binary64 payload bits, including signed zero; complete structure plus solver metadata/counters. JSON object ordering is ignored. solveSeconds and solveCpuSeconds are excluded. Parsing and comparison are diagnostic work outside benchmark timing."} try: baseline, baseline_hash = read_result(args.baseline) report["baselineSha256"] = baseline_hash report["baselineStatistics"] = validate_result(baseline) for path in args.candidate: item = {"path": str(path.resolve()), "passed": False} try: candidate, candidate_hash = read_result(path) item["sha256"] = candidate_hash item["statistics"] = validate_result(candidate) item.update(compare(baseline, candidate)) except ComparisonError as error: item["error"] = error.detail except (OSError, ValueError, TypeError) as error: item["error"] = {"reason": str(error), "type": type(error).__name__} report["candidates"].append(item) report["allPassed"] = all(item["passed"] for item in report["candidates"]) except ComparisonError as error: report["baselineError"] = error.detail except (OSError, ValueError, TypeError) as error: report["baselineError"] = {"reason": str(error), "type": type(error).__name__} args.output.parent.mkdir(parents=True, exist_ok=True) args.output.write_text(json.dumps(report, ensure_ascii=False, indent=2, allow_nan=False) + "\n", encoding="utf-8") print(json.dumps({"allPassed": report["allPassed"], "candidates": len(report["candidates"]), "output": str(args.output.resolve())})) return 0 if report["allPassed"] else 1 if __name__ == "__main__": sys.exit(main())