lujingze
3bc4be3c06
原生结果series通过字节索引直传，C端使用Ryu精确回读编码和64 KiB批量写出；网页采用Float64缓存和CSV工作线程，减少结果处理与保存等待。 补充八路AME曲线核查、全流程分阶段计时、独立编码基准和复现工具，固定后续优化采用修正八路及rtol=1e-8。C写出1.1808→0.1638 s，点击到可查看8.0100→6.9756 s。 验证：最终10项编码专项、29项相关后端回归通过；8份原生结果逐位一致，16次网页结果/CSV/刷新恢复通过。前端构建及缓存/CSV专项在本轮结果处理工作中通过。环境、原始大结果与临时构建不纳入Git。
368 lines
18 KiB
Python
368 lines
18 KiB
Python
"""Standalone JSON-number contracts; no model, solver or SUNDIALS is required.
|
|
|
|
The decimal text may change between correct shortest encoders. These tests use
|
|
Python's independent JSON decoder and binary64 bits, including the sign of zero,
|
|
instead of comparing against the old %.17g spelling.
|
|
"""
|
|
import ctypes
|
|
import json
|
|
import math
|
|
import os
|
|
from pathlib import Path
|
|
import random
|
|
import re
|
|
import shlex
|
|
import shutil
|
|
import struct
|
|
import subprocess
|
|
import tempfile
|
|
import unittest
|
|
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
JSON_NUMBER = re.compile(rb'-?(?:0|[1-9][0-9]*)(?:\.[0-9]+)?(?:[eE][+-]?[0-9]+)?\Z')
|
|
BUFFER_BYTES = 64 * 1024
|
|
SIGN = 1 << 63
|
|
FRACTION_MASK = (1 << 52) - 1
|
|
|
|
|
|
def double_from_bits(bits):
|
|
return struct.unpack('=d', struct.pack('=Q', bits))[0]
|
|
|
|
|
|
def double_bits(value):
|
|
return struct.unpack('=Q', struct.pack('=d', value))[0]
|
|
|
|
|
|
def finite_patterns(random_count=10000):
|
|
# Every finite exponent binade, with exact powers and significand edges.
|
|
values = {0, SIGN}
|
|
for exponent in range(0x7ff):
|
|
for fraction in (0, 1, (1 << 51) - 1, 1 << 51, FRACTION_MASK):
|
|
bits = (exponent << 52) | fraction
|
|
values.update((bits, bits | SIGN))
|
|
# Decimal carry/notation boundaries and adjacent representable values.
|
|
for exponent in range(-323, 309):
|
|
center = float(f'1e{exponent}')
|
|
for value in (math.nextafter(center, 0), center, math.nextafter(center, math.inf)):
|
|
if math.isfinite(value):
|
|
values.update((double_bits(value), double_bits(-value)))
|
|
random_source = random.Random(0x5259555F4A534F4E)
|
|
added = 0
|
|
while added < random_count:
|
|
bits = random_source.getrandbits(64)
|
|
if (bits >> 52) & 0x7ff != 0x7ff:
|
|
values.add(bits)
|
|
added += 1
|
|
return sorted(values)
|
|
|
|
|
|
class WriteStatus(ctypes.Structure):
|
|
_fields_ = [
|
|
('opened', ctypes.c_int), ('written', ctypes.c_int), ('closed', ctypes.c_int),
|
|
('start', ctypes.c_longlong), ('end', ctypes.c_longlong), ('final_position', ctypes.c_longlong),
|
|
]
|
|
|
|
|
|
HARNESS = r'''
|
|
#include <stdio.h>
|
|
#include <stdint.h>
|
|
#include <stddef.h>
|
|
#include <string.h>
|
|
#include "json_numbers.h"
|
|
|
|
/* Faults intercept only the production writer's fwrite calls in this temporary
|
|
translation unit. Real stdio still writes the accepted prefix. */
|
|
int test_write_fault_mode = 0;
|
|
int test_write_call_count = 0;
|
|
#ifdef TEST_FWRITE_FAULTS
|
|
static size_t test_fwrite(const void *data, size_t size, size_t count, FILE *stream) {
|
|
test_write_call_count++;
|
|
if (test_write_fault_mode == 1 ||
|
|
(test_write_fault_mode == 3 && test_write_call_count >= 2)) return 0;
|
|
if (test_write_fault_mode == 2)
|
|
return count ? fwrite(data, size, count - 1, stream) : 0;
|
|
return fwrite(data, size, count, stream);
|
|
}
|
|
#define fwrite test_fwrite
|
|
#include "json_numbers.c"
|
|
#undef fwrite
|
|
#endif
|
|
|
|
typedef struct {
|
|
int opened, written, closed;
|
|
long long start, end, final_position;
|
|
} TestWriteStatus;
|
|
|
|
void test_array_file(const char *path, const double *values, size_t count,
|
|
size_t stride, size_t prefix_bytes, TestWriteStatus *status) {
|
|
memset(status, 0, sizeof(*status));
|
|
FILE *stream = fopen(path, "wb");
|
|
if (!stream) return;
|
|
status->opened = 1;
|
|
for (size_t i = 0; i < prefix_bytes; i++) fputc('p', stream);
|
|
status->start = (long long)ftell(stream);
|
|
status->written = native_json_write_array(stream, values, count, stride);
|
|
status->end = (long long)ftell(stream);
|
|
if (status->written) fputs("TAIL", stream);
|
|
status->final_position = (long long)ftell(stream);
|
|
status->closed = fclose(stream) == 0;
|
|
}
|
|
|
|
void test_number_file(const char *path, double value, TestWriteStatus *status) {
|
|
memset(status, 0, sizeof(*status));
|
|
FILE *stream = fopen(path, "wb");
|
|
if (!stream) return;
|
|
status->opened = 1;
|
|
status->written = native_json_write_number(stream, value);
|
|
status->end = status->final_position = (long long)ftell(stream);
|
|
status->closed = fclose(stream) == 0;
|
|
}
|
|
|
|
/* The number writer flushes its own block to FILE, not FILE's stdio buffer.
|
|
The caller must still propagate a delayed fclose failure. */
|
|
int test_delayed_close_failure(const char *path, int *written, int *closed) {
|
|
FILE *stream = fopen(path, "wb");
|
|
if (!stream) return 0;
|
|
char buffer[4096];
|
|
if (setvbuf(stream, buffer, _IOFBF, sizeof(buffer))) { fclose(stream); return 0; }
|
|
*written = native_json_write_number(stream, 0.1);
|
|
*closed = fclose(stream) == 0;
|
|
return 1;
|
|
}
|
|
'''
|
|
|
|
|
|
class NativeJsonWriterTests(unittest.TestCase):
|
|
@classmethod
|
|
def setUpClass(cls):
|
|
command = shlex.split(os.environ.get('CC', ''))
|
|
if not command:
|
|
compiler = shutil.which('gcc') or shutil.which('clang')
|
|
if not compiler:
|
|
raise unittest.SkipTest('A native C compiler is required')
|
|
command = [compiler]
|
|
cls.compiler = command
|
|
cls.directory = tempfile.TemporaryDirectory(prefix='native-json-writer-')
|
|
cls.addClassCleanup(cls.directory.cleanup)
|
|
cls.root = Path(cls.directory.name)
|
|
cls.library = cls.build_library('ordinary', faults=False)
|
|
cls.fault_library = cls.build_library('faults', faults=True)
|
|
cls.portable_library = cls.build_library('portable-64-bit', faults=False, only_64_bit=True)
|
|
cls.fault_mode = ctypes.c_int.in_dll(cls.fault_library, 'test_write_fault_mode')
|
|
cls.fault_calls = ctypes.c_int.in_dll(cls.fault_library, 'test_write_call_count')
|
|
|
|
@classmethod
|
|
def build_library(cls, name, *, faults, only_64_bit=False):
|
|
source = cls.root / f'{name}.c'
|
|
source.write_text(HARNESS)
|
|
library_path = cls.root / (name + ('.dll' if os.name == 'nt' else '.so'))
|
|
command = cls.compiler + ['-std=c11', '-O2', '-Wall', '-Wextra', '-Werror',
|
|
'-ffp-contract=off', '-fno-fast-math', '-shared']
|
|
if os.name != 'nt':
|
|
command.append('-fPIC')
|
|
if faults:
|
|
command.append('-DTEST_FWRITE_FAULTS')
|
|
if only_64_bit:
|
|
command.append('-DRYU_ONLY_64_BIT_OPS')
|
|
command += ['-I', str(ROOT / 'native/include'), '-I', str(ROOT / 'native/runtime'),
|
|
str(source)]
|
|
if not faults:
|
|
command.append(str(ROOT / 'native/runtime/json_numbers.c'))
|
|
command += [str(ROOT / 'native/encoding/ryu/d2s.c'), '-lm', '-o', str(library_path)]
|
|
compiled = subprocess.run(command, capture_output=True, text=True, timeout=60)
|
|
if compiled.returncode:
|
|
raise AssertionError(compiled.stderr)
|
|
library = ctypes.CDLL(str(library_path))
|
|
if os.name == 'nt':
|
|
import _ctypes
|
|
cls.addClassCleanup(_ctypes.FreeLibrary, library._handle)
|
|
library.native_json_format_double.argtypes = [ctypes.c_void_p, ctypes.c_double]
|
|
library.native_json_format_double.restype = ctypes.c_int
|
|
library.native_json_write_number.argtypes = [ctypes.c_void_p, ctypes.c_double]
|
|
library.native_json_write_number.restype = ctypes.c_int
|
|
library.native_json_write_array.argtypes = [ctypes.c_void_p, ctypes.POINTER(ctypes.c_double), ctypes.c_size_t, ctypes.c_size_t]
|
|
library.native_json_write_array.restype = ctypes.c_int
|
|
library.test_array_file.argtypes = [ctypes.c_char_p, ctypes.POINTER(ctypes.c_double),
|
|
ctypes.c_size_t, ctypes.c_size_t, ctypes.c_size_t,
|
|
ctypes.POINTER(WriteStatus)]
|
|
library.test_array_file.restype = None
|
|
library.test_number_file.argtypes = [ctypes.c_char_p, ctypes.c_double, ctypes.POINTER(WriteStatus)]
|
|
library.test_number_file.restype = None
|
|
library.test_delayed_close_failure.argtypes = [ctypes.c_char_p, ctypes.POINTER(ctypes.c_int), ctypes.POINTER(ctypes.c_int)]
|
|
library.test_delayed_close_failure.restype = ctypes.c_int
|
|
return library
|
|
|
|
def encode(self, value, library=None):
|
|
# Sentinels bracket the promised 32-byte output, with no assumption that
|
|
# the returned token is NUL-terminated.
|
|
storage = (ctypes.c_ubyte * 34)(*([0xA5] * 34))
|
|
length = (library or self.library).native_json_format_double(ctypes.byref(storage, 1), value)
|
|
self.assertEqual((storage[0], storage[33]), (0xA5, 0xA5))
|
|
self.assertGreater(length, 0)
|
|
self.assertLessEqual(length, 32)
|
|
return bytes(storage[1:1 + length])
|
|
|
|
def assert_roundtrip(self, text, expected_bits):
|
|
self.assertRegex(text, JSON_NUMBER, f'Invalid JSON token for {expected_bits:016x}')
|
|
decoded = json.loads(text)
|
|
self.assertEqual(double_bits(float(decoded)), expected_bits,
|
|
f'{expected_bits:016x} became {text!r} then {decoded!r}')
|
|
|
|
def write_array(self, values, *, count=None, stride=1, prefix=0, library=None):
|
|
array = (ctypes.c_double * len(values))(*values) if values else None
|
|
status = WriteStatus()
|
|
output = self.root / 'array.json'
|
|
(library or self.library).test_array_file(os.fsencode(output), array,
|
|
len(values) if count is None else count, stride, prefix, ctypes.byref(status))
|
|
self.assertTrue(status.opened)
|
|
return output.read_bytes(), status
|
|
|
|
def test_binary64_boundaries_and_seeded_random_values_roundtrip(self):
|
|
for bits in finite_patterns():
|
|
self.assert_roundtrip(self.encode(double_from_bits(bits)), bits)
|
|
|
|
def test_64_bit_fallback_roundtrips_the_same_binary64_corpus(self):
|
|
patterns = finite_patterns()
|
|
for bits in patterns:
|
|
self.assert_roundtrip(self.encode(double_from_bits(bits), self.portable_library), bits)
|
|
values = [double_from_bits(bits) for bits in patterns]
|
|
ordinary, ordinary_status = self.write_array(values)
|
|
portable, portable_status = self.write_array(values, library=self.portable_library)
|
|
self.assertTrue(ordinary_status.written and ordinary_status.closed)
|
|
self.assertTrue(portable_status.written and portable_status.closed)
|
|
self.assertEqual(portable, ordinary)
|
|
|
|
def test_plain_decimal_is_used_only_when_it_shortens_the_token(self):
|
|
# Tie cases deliberately keep scientific notation; far exponents must
|
|
# never be expanded to hundreds of zeroes in the 32-byte destination.
|
|
cases = ((10.0, b'10'), (12.0, b'12'), (-12.0, b'-12'),
|
|
(.1, b'0.1'), (-.1, b'-0.1'), (123.45, b'123.45'),
|
|
(100.0, b'1E2'), (.01, b'1E-2'), (1e100, b'1E100'),
|
|
(double_from_bits(1), b'5E-324'))
|
|
for library in (self.library, self.portable_library):
|
|
for value, expected in cases:
|
|
with self.subTest(value=value, expected=expected):
|
|
token = self.encode(value, library)
|
|
self.assertEqual(token, expected)
|
|
self.assert_roundtrip(token, double_bits(value))
|
|
|
|
def test_zero_sign_and_single_number_file(self):
|
|
self.assertEqual(self.encode(0.0), b'0')
|
|
self.assertEqual(self.encode(-0.0), b'-0.0')
|
|
output = self.root / 'number.json'
|
|
for bits in (0, SIGN, 1, SIGN | 1, 0x0010000000000000, 0x7fefffffffffffff):
|
|
status = WriteStatus()
|
|
self.library.test_number_file(os.fsencode(output), double_from_bits(bits), ctypes.byref(status))
|
|
self.assertTrue(status.opened and status.written and status.closed)
|
|
data = output.read_bytes()
|
|
self.assertEqual(status.end, len(data))
|
|
self.assert_roundtrip(data, bits)
|
|
|
|
def test_array_crosses_block_boundaries_without_changing_offsets(self):
|
|
cases = [([], 2), ([0.0] * 32767, BUFFER_BYTES - 1),
|
|
([-0.0] + [0.0] * 32765, BUFFER_BYTES),
|
|
([0.0] * 32768, BUFFER_BYTES + 1),
|
|
([double_from_bits(bits) for bits in finite_patterns(0)[::3]], None)]
|
|
for values, expected_length in cases:
|
|
for prefix in (0, 37, BUFFER_BYTES - 1):
|
|
with self.subTest(values=len(values), expected_length=expected_length, prefix=prefix):
|
|
data, status = self.write_array(values, prefix=prefix)
|
|
self.assertTrue(status.written and status.closed)
|
|
self.assertEqual(status.start, prefix)
|
|
self.assertEqual(data[:prefix], b'p' * prefix)
|
|
self.assertEqual(data[status.end:], b'TAIL')
|
|
self.assertEqual(status.final_position, len(data))
|
|
token = data[status.start:status.end]
|
|
if expected_length is not None:
|
|
self.assertEqual(len(token), expected_length)
|
|
decoded = json.loads(token)
|
|
self.assertEqual([double_bits(float(v)) for v in decoded], [double_bits(v) for v in values])
|
|
|
|
def test_strided_array_keeps_selected_column_order(self):
|
|
selected = [double_from_bits(bits) for bits in finite_patterns(0)[::13]]
|
|
for stride in (1, 3, 17):
|
|
with self.subTest(stride=stride):
|
|
values = [math.nan] * (len(selected) * stride)
|
|
values[::stride] = selected
|
|
data, status = self.write_array(values, count=len(selected), stride=stride)
|
|
self.assertTrue(status.written and status.closed)
|
|
decoded = json.loads(data[:status.end])
|
|
self.assertEqual([double_bits(float(v)) for v in decoded], [double_bits(v) for v in selected])
|
|
|
|
def test_invalid_file_pointer_and_array_bounds_fail_before_access(self):
|
|
value = (ctypes.c_double * 1)(.1)
|
|
self.assertEqual(self.library.native_json_write_number(None, .1), 0)
|
|
self.assertEqual(self.library.native_json_write_array(None, value, 1, 1), 0)
|
|
for values, count, stride in (([], 1, 1), ([.1], 2, 0),
|
|
([.1], 2, ctypes.c_size_t(-1).value)):
|
|
with self.subTest(count=count, stride=stride):
|
|
data, status = self.write_array(values, count=count, stride=stride)
|
|
self.assertTrue(status.opened and status.closed)
|
|
self.assertFalse(status.written)
|
|
self.assertEqual(data, b'')
|
|
# A single sample never advances its pointer; an enormous stride is safe.
|
|
data, status = self.write_array([.1], count=1, stride=ctypes.c_size_t(-1).value)
|
|
self.assertTrue(status.written and status.closed)
|
|
self.assertEqual(json.loads(data[:status.end]), [.1])
|
|
|
|
def test_nonfinite_numbers_fail_instead_of_emitting_invalid_json(self):
|
|
output = self.root / 'nonfinite.json'
|
|
for bits in (0x7ff0000000000000, 0xfff0000000000000,
|
|
0x7ff8000000000000, 0xfff8000000000001, 0x7ff0000000000001):
|
|
value = double_from_bits(bits)
|
|
with self.subTest(bits=f'{bits:016x}'):
|
|
token = ctypes.create_string_buffer(32)
|
|
self.assertEqual(self.library.native_json_format_double(token, value), 0)
|
|
status = WriteStatus()
|
|
self.library.test_number_file(os.fsencode(output), value, ctypes.byref(status))
|
|
self.assertTrue(status.opened and status.closed)
|
|
self.assertFalse(status.written)
|
|
self.assertEqual(output.read_bytes(), b'')
|
|
for finite_prefix in ([], [0.0] * (BUFFER_BYTES + 1)):
|
|
data, status = self.write_array(finite_prefix + [value])
|
|
self.assertFalse(status.written)
|
|
self.assertTrue(status.closed)
|
|
self.assertNotIn(b'NaN', data)
|
|
self.assertNotIn(b'Infinity', data)
|
|
|
|
def test_zero_and_short_writes_are_reported_without_stdio_error_flag(self):
|
|
# The injected fwrite can return short without setting FILE's error bit;
|
|
# relying only on ferror/fclose would incorrectly report success.
|
|
output = self.root / 'fault-number.json'
|
|
try:
|
|
for mode in (1, 2):
|
|
with self.subTest(mode=mode):
|
|
self.fault_mode.value = mode
|
|
self.fault_calls.value = 0
|
|
status = WriteStatus()
|
|
self.fault_library.test_number_file(os.fsencode(output), .1, ctypes.byref(status))
|
|
self.assertTrue(status.opened and status.closed)
|
|
self.assertFalse(status.written)
|
|
self.assertGreater(self.fault_calls.value, 0)
|
|
self.fault_calls.value = 0
|
|
_, status = self.write_array([1.0, 2.0], library=self.fault_library)
|
|
self.assertFalse(status.written)
|
|
self.assertTrue(status.closed)
|
|
self.fault_mode.value = 3
|
|
self.fault_calls.value = 0
|
|
data, status = self.write_array([0.0] * (BUFFER_BYTES * 2), library=self.fault_library)
|
|
self.assertFalse(status.written)
|
|
self.assertTrue(status.closed)
|
|
self.assertGreaterEqual(self.fault_calls.value, 2)
|
|
self.assertTrue(data, 'The initial successful block must survive a later write failure')
|
|
finally:
|
|
self.fault_mode.value = 0
|
|
self.fault_calls.value = 0
|
|
|
|
@unittest.skipUnless(os.name == 'posix' and Path('/dev/full').exists(), '/dev/full is needed for a real delayed I/O failure')
|
|
def test_caller_must_check_delayed_fclose_failure(self):
|
|
written, closed = ctypes.c_int(), ctypes.c_int()
|
|
self.assertEqual(self.library.test_delayed_close_failure(b'/dev/full', ctypes.byref(written), ctypes.byref(closed)), 1)
|
|
self.assertEqual(written.value, 1, 'A small buffered token should not force FILE fflush')
|
|
self.assertEqual(closed.value, 0, 'FILE close must expose the delayed device error')
|
|
|
|
|
|
if __name__ == '__main__':
|
|
unittest.main()
|