lujingze/SystemSimulationApp
1
0 Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

优化原生结果编码传输与浏览器缓存，记录八路性能基线

Browse Source 
原生结果series通过字节索引直传，C端使用Ryu精确回读编码和64 KiB批量写出；网页采用Float64缓存和CSV工作线程，减少结果处理与保存等待。

补充八路AME曲线核查、全流程分阶段计时、独立编码基准和复现工具，固定后续优化采用修正八路及rtol=1e-8。C写出1.1808→0.1638 s，点击到可查看8.0100→6.9756 s。

验证：最终10项编码专项、29项相关后端回归通过；8份原生结果逐位一致，16次网页结果/CSV/刷新恢复通过。前端构建及缓存/CSV专项在本轮结果处理工作中通过。环境、原始大结果与临时构建不纳入Git。
This commit is contained in:
lujingze committed 2026-09-11 15:09:15 +00:00
1 parent 808c484f5b
commit 3bc4be3c06
55 files changed
+20522 -141
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

tests/test_native_json_writer.py
+367
View File
@@ -0,0 +1,367 @@
"""Standalone JSON-number contracts; no model, solver or SUNDIALS is required.
The decimal text may change between correct shortest encoders. These tests use
Python's independent JSON decoder and binary64 bits, including the sign of zero,
instead of comparing against the old %.17g spelling.
"""
import ctypes
import json
import math
import os
from pathlib import Path
import random
import re
import shlex
import shutil
import struct
import subprocess
import tempfile
import unittest
ROOT = Path(__file__).resolve().parents[1]
JSON_NUMBER = re.compile(rb'-?(?:0|[1-9][0-9]*)(?:\.[0-9]+)?(?:[eE][+-]?[0-9]+)?\Z')
BUFFER_BYTES = 64 * 1024
SIGN = 1 << 63
FRACTION_MASK = (1 << 52) - 1
def double_from_bits(bits):
return struct.unpack('=d', struct.pack('=Q', bits))[0]
def double_bits(value):
return struct.unpack('=Q', struct.pack('=d', value))[0]
def finite_patterns(random_count=10000):
# Every finite exponent binade, with exact powers and significand edges.
values = {0, SIGN}
for exponent in range(0x7ff):
for fraction in (0, 1, (1 << 51) - 1, 1 << 51, FRACTION_MASK):
bits = (exponent << 52) | fraction
values.update((bits, bits | SIGN))
# Decimal carry/notation boundaries and adjacent representable values.
for exponent in range(-323, 309):
center = float(f'1e{exponent}')
for value in (math.nextafter(center, 0), center, math.nextafter(center, math.inf)):
if math.isfinite(value):
values.update((double_bits(value), double_bits(-value)))
random_source = random.Random(0x5259555F4A534F4E)
added = 0
while added < random_count:
bits = random_source.getrandbits(64)
if (bits >> 52) & 0x7ff != 0x7ff:
values.add(bits)
added += 1
return sorted(values)
class WriteStatus(ctypes.Structure):
_fields_ = [
('opened', ctypes.c_int), ('written', ctypes.c_int), ('closed', ctypes.c_int),
('start', ctypes.c_longlong), ('end', ctypes.c_longlong), ('final_position', ctypes.c_longlong),
]
HARNESS = r'''
#include <stdio.h>
#include <stdint.h>
#include <stddef.h>
#include <string.h>
#include "json_numbers.h"
/* Faults intercept only the production writer's fwrite calls in this temporary
translation unit. Real stdio still writes the accepted prefix. */
int test_write_fault_mode = 0;
int test_write_call_count = 0;
#ifdef TEST_FWRITE_FAULTS
static size_t test_fwrite(const void *data, size_t size, size_t count, FILE *stream) {
test_write_call_count++;
if (test_write_fault_mode == 1 ||
(test_write_fault_mode == 3 && test_write_call_count >= 2)) return 0;
if (test_write_fault_mode == 2)
return count ? fwrite(data, size, count - 1, stream) : 0;
return fwrite(data, size, count, stream);
}
#define fwrite test_fwrite
#include "json_numbers.c"
#undef fwrite
#endif
typedef struct {
int opened, written, closed;
long long start, end, final_position;
} TestWriteStatus;
void test_array_file(const char *path, const double *values, size_t count,
size_t stride, size_t prefix_bytes, TestWriteStatus *status) {
memset(status, 0, sizeof(*status));
FILE *stream = fopen(path, "wb");
if (!stream) return;
status->opened = 1;
for (size_t i = 0; i < prefix_bytes; i++) fputc('p', stream);
status->start = (long long)ftell(stream);
status->written = native_json_write_array(stream, values, count, stride);
status->end = (long long)ftell(stream);
if (status->written) fputs("TAIL", stream);
status->final_position = (long long)ftell(stream);
status->closed = fclose(stream) == 0;
}
void test_number_file(const char *path, double value, TestWriteStatus *status) {
memset(status, 0, sizeof(*status));
FILE *stream = fopen(path, "wb");
if (!stream) return;
status->opened = 1;
status->written = native_json_write_number(stream, value);
status->end = status->final_position = (long long)ftell(stream);
status->closed = fclose(stream) == 0;
}
/* The number writer flushes its own block to FILE, not FILE's stdio buffer.
The caller must still propagate a delayed fclose failure. */
int test_delayed_close_failure(const char *path, int *written, int *closed) {
FILE *stream = fopen(path, "wb");
if (!stream) return 0;
char buffer[4096];
if (setvbuf(stream, buffer, _IOFBF, sizeof(buffer))) { fclose(stream); return 0; }
*written = native_json_write_number(stream, 0.1);
*closed = fclose(stream) == 0;
return 1;
}
'''
class NativeJsonWriterTests(unittest.TestCase):
@classmethod
def setUpClass(cls):
command = shlex.split(os.environ.get('CC', ''))
if not command:
compiler = shutil.which('gcc') or shutil.which('clang')
if not compiler:
raise unittest.SkipTest('A native C compiler is required')
command = [compiler]
cls.compiler = command
cls.directory = tempfile.TemporaryDirectory(prefix='native-json-writer-')
cls.addClassCleanup(cls.directory.cleanup)
cls.root = Path(cls.directory.name)
cls.library = cls.build_library('ordinary', faults=False)
cls.fault_library = cls.build_library('faults', faults=True)
cls.portable_library = cls.build_library('portable-64-bit', faults=False, only_64_bit=True)
cls.fault_mode = ctypes.c_int.in_dll(cls.fault_library, 'test_write_fault_mode')
cls.fault_calls = ctypes.c_int.in_dll(cls.fault_library, 'test_write_call_count')
@classmethod
def build_library(cls, name, *, faults, only_64_bit=False):
source = cls.root / f'{name}.c'
source.write_text(HARNESS)
library_path = cls.root / (name + ('.dll' if os.name == 'nt' else '.so'))
command = cls.compiler + ['-std=c11', '-O2', '-Wall', '-Wextra', '-Werror',
'-ffp-contract=off', '-fno-fast-math', '-shared']
if os.name != 'nt':
command.append('-fPIC')
if faults:
command.append('-DTEST_FWRITE_FAULTS')
if only_64_bit:
command.append('-DRYU_ONLY_64_BIT_OPS')
command += ['-I', str(ROOT / 'native/include'), '-I', str(ROOT / 'native/runtime'),
str(source)]
if not faults:
command.append(str(ROOT / 'native/runtime/json_numbers.c'))
command += [str(ROOT / 'native/encoding/ryu/d2s.c'), '-lm', '-o', str(library_path)]
compiled = subprocess.run(command, capture_output=True, text=True, timeout=60)
if compiled.returncode:
raise AssertionError(compiled.stderr)
library = ctypes.CDLL(str(library_path))
if os.name == 'nt':
import _ctypes
cls.addClassCleanup(_ctypes.FreeLibrary, library._handle)
library.native_json_format_double.argtypes = [ctypes.c_void_p, ctypes.c_double]
library.native_json_format_double.restype = ctypes.c_int
library.native_json_write_number.argtypes = [ctypes.c_void_p, ctypes.c_double]
library.native_json_write_number.restype = ctypes.c_int
library.native_json_write_array.argtypes = [ctypes.c_void_p, ctypes.POINTER(ctypes.c_double), ctypes.c_size_t, ctypes.c_size_t]
library.native_json_write_array.restype = ctypes.c_int
library.test_array_file.argtypes = [ctypes.c_char_p, ctypes.POINTER(ctypes.c_double),
ctypes.c_size_t, ctypes.c_size_t, ctypes.c_size_t,
ctypes.POINTER(WriteStatus)]
library.test_array_file.restype = None
library.test_number_file.argtypes = [ctypes.c_char_p, ctypes.c_double, ctypes.POINTER(WriteStatus)]
library.test_number_file.restype = None
library.test_delayed_close_failure.argtypes = [ctypes.c_char_p, ctypes.POINTER(ctypes.c_int), ctypes.POINTER(ctypes.c_int)]
library.test_delayed_close_failure.restype = ctypes.c_int
return library
def encode(self, value, library=None):
# Sentinels bracket the promised 32-byte output, with no assumption that
# the returned token is NUL-terminated.
storage = (ctypes.c_ubyte * 34)(*([0xA5] * 34))
length = (library or self.library).native_json_format_double(ctypes.byref(storage, 1), value)
self.assertEqual((storage[0], storage[33]), (0xA5, 0xA5))
self.assertGreater(length, 0)
self.assertLessEqual(length, 32)
return bytes(storage[1:1 + length])
def assert_roundtrip(self, text, expected_bits):
self.assertRegex(text, JSON_NUMBER, f'Invalid JSON token for {expected_bits:016x}')
decoded = json.loads(text)
self.assertEqual(double_bits(float(decoded)), expected_bits,
f'{expected_bits:016x} became {text!r} then {decoded!r}')
def write_array(self, values, *, count=None, stride=1, prefix=0, library=None):
array = (ctypes.c_double * len(values))(*values) if values else None
status = WriteStatus()
output = self.root / 'array.json'
(library or self.library).test_array_file(os.fsencode(output), array,
len(values) if count is None else count, stride, prefix, ctypes.byref(status))
self.assertTrue(status.opened)
return output.read_bytes(), status
def test_binary64_boundaries_and_seeded_random_values_roundtrip(self):
for bits in finite_patterns():
self.assert_roundtrip(self.encode(double_from_bits(bits)), bits)
def test_64_bit_fallback_roundtrips_the_same_binary64_corpus(self):
patterns = finite_patterns()
for bits in patterns:
self.assert_roundtrip(self.encode(double_from_bits(bits), self.portable_library), bits)
values = [double_from_bits(bits) for bits in patterns]
ordinary, ordinary_status = self.write_array(values)
portable, portable_status = self.write_array(values, library=self.portable_library)
self.assertTrue(ordinary_status.written and ordinary_status.closed)
self.assertTrue(portable_status.written and portable_status.closed)
self.assertEqual(portable, ordinary)
def test_plain_decimal_is_used_only_when_it_shortens_the_token(self):
# Tie cases deliberately keep scientific notation; far exponents must
# never be expanded to hundreds of zeroes in the 32-byte destination.
cases = ((10.0, b'10'), (12.0, b'12'), (-12.0, b'-12'),
(.1, b'0.1'), (-.1, b'-0.1'), (123.45, b'123.45'),
(100.0, b'1E2'), (.01, b'1E-2'), (1e100, b'1E100'),
(double_from_bits(1), b'5E-324'))
for library in (self.library, self.portable_library):
for value, expected in cases:
with self.subTest(value=value, expected=expected):
token = self.encode(value, library)
self.assertEqual(token, expected)
self.assert_roundtrip(token, double_bits(value))
def test_zero_sign_and_single_number_file(self):
self.assertEqual(self.encode(0.0), b'0')
self.assertEqual(self.encode(-0.0), b'-0.0')
output = self.root / 'number.json'
for bits in (0, SIGN, 1, SIGN | 1, 0x0010000000000000, 0x7fefffffffffffff):
status = WriteStatus()
self.library.test_number_file(os.fsencode(output), double_from_bits(bits), ctypes.byref(status))
self.assertTrue(status.opened and status.written and status.closed)
data = output.read_bytes()
self.assertEqual(status.end, len(data))
self.assert_roundtrip(data, bits)
def test_array_crosses_block_boundaries_without_changing_offsets(self):
cases = [([], 2), ([0.0] * 32767, BUFFER_BYTES - 1),
([-0.0] + [0.0] * 32765, BUFFER_BYTES),
([0.0] * 32768, BUFFER_BYTES + 1),
([double_from_bits(bits) for bits in finite_patterns(0)[::3]], None)]
for values, expected_length in cases:
for prefix in (0, 37, BUFFER_BYTES - 1):
with self.subTest(values=len(values), expected_length=expected_length, prefix=prefix):
data, status = self.write_array(values, prefix=prefix)
self.assertTrue(status.written and status.closed)
self.assertEqual(status.start, prefix)
self.assertEqual(data[:prefix], b'p' * prefix)
self.assertEqual(data[status.end:], b'TAIL')
self.assertEqual(status.final_position, len(data))
token = data[status.start:status.end]
if expected_length is not None:
self.assertEqual(len(token), expected_length)
decoded = json.loads(token)
self.assertEqual([double_bits(float(v)) for v in decoded], [double_bits(v) for v in values])
def test_strided_array_keeps_selected_column_order(self):
selected = [double_from_bits(bits) for bits in finite_patterns(0)[::13]]
for stride in (1, 3, 17):
with self.subTest(stride=stride):
values = [math.nan] * (len(selected) * stride)
values[::stride] = selected
data, status = self.write_array(values, count=len(selected), stride=stride)
self.assertTrue(status.written and status.closed)
decoded = json.loads(data[:status.end])
self.assertEqual([double_bits(float(v)) for v in decoded], [double_bits(v) for v in selected])
def test_invalid_file_pointer_and_array_bounds_fail_before_access(self):
value = (ctypes.c_double * 1)(.1)
self.assertEqual(self.library.native_json_write_number(None, .1), 0)
self.assertEqual(self.library.native_json_write_array(None, value, 1, 1), 0)
for values, count, stride in (([], 1, 1), ([.1], 2, 0),
([.1], 2, ctypes.c_size_t(-1).value)):
with self.subTest(count=count, stride=stride):
data, status = self.write_array(values, count=count, stride=stride)
self.assertTrue(status.opened and status.closed)
self.assertFalse(status.written)
self.assertEqual(data, b'')
# A single sample never advances its pointer; an enormous stride is safe.
data, status = self.write_array([.1], count=1, stride=ctypes.c_size_t(-1).value)
self.assertTrue(status.written and status.closed)
self.assertEqual(json.loads(data[:status.end]), [.1])
def test_nonfinite_numbers_fail_instead_of_emitting_invalid_json(self):
output = self.root / 'nonfinite.json'
for bits in (0x7ff0000000000000, 0xfff0000000000000,
0x7ff8000000000000, 0xfff8000000000001, 0x7ff0000000000001):
value = double_from_bits(bits)
with self.subTest(bits=f'{bits:016x}'):
token = ctypes.create_string_buffer(32)
self.assertEqual(self.library.native_json_format_double(token, value), 0)
status = WriteStatus()
self.library.test_number_file(os.fsencode(output), value, ctypes.byref(status))
self.assertTrue(status.opened and status.closed)
self.assertFalse(status.written)
self.assertEqual(output.read_bytes(), b'')
for finite_prefix in ([], [0.0] * (BUFFER_BYTES + 1)):
data, status = self.write_array(finite_prefix + [value])
self.assertFalse(status.written)
self.assertTrue(status.closed)
self.assertNotIn(b'NaN', data)
self.assertNotIn(b'Infinity', data)
def test_zero_and_short_writes_are_reported_without_stdio_error_flag(self):
# The injected fwrite can return short without setting FILE's error bit;
# relying only on ferror/fclose would incorrectly report success.
output = self.root / 'fault-number.json'
try:
for mode in (1, 2):
with self.subTest(mode=mode):
self.fault_mode.value = mode
self.fault_calls.value = 0
status = WriteStatus()
self.fault_library.test_number_file(os.fsencode(output), .1, ctypes.byref(status))
self.assertTrue(status.opened and status.closed)
self.assertFalse(status.written)
self.assertGreater(self.fault_calls.value, 0)
self.fault_calls.value = 0
_, status = self.write_array([1.0, 2.0], library=self.fault_library)
self.assertFalse(status.written)
self.assertTrue(status.closed)
self.fault_mode.value = 3
self.fault_calls.value = 0
data, status = self.write_array([0.0] * (BUFFER_BYTES * 2), library=self.fault_library)
self.assertFalse(status.written)
self.assertTrue(status.closed)
self.assertGreaterEqual(self.fault_calls.value, 2)
self.assertTrue(data, 'The initial successful block must survive a later write failure')
finally:
self.fault_mode.value = 0
self.fault_calls.value = 0
@unittest.skipUnless(os.name == 'posix' and Path('/dev/full').exists(), '/dev/full is needed for a real delayed I/O failure')
def test_caller_must_check_delayed_fclose_failure(self):
written, closed = ctypes.c_int(), ctypes.c_int()
self.assertEqual(self.library.test_delayed_close_failure(b'/dev/full', ctypes.byref(written), ctypes.byref(closed)), 1)
self.assertEqual(written.value, 1, 'A small buffered token should not force FILE fflush')
self.assertEqual(closed.value, 0, 'FILE close must expose the delayed device error')
if __name__ == '__main__':
unittest.main()