"""Standalone JSON-number contracts; no model, solver or SUNDIALS is required. The decimal text may change between correct shortest encoders. These tests use Python's independent JSON decoder and binary64 bits, including the sign of zero, instead of comparing against the old %.17g spelling. """ import ctypes import json import math import os from pathlib import Path import random import re import shlex import shutil import struct import subprocess import tempfile import unittest ROOT = Path(__file__).resolve().parents[1] JSON_NUMBER = re.compile(rb'-?(?:0|[1-9][0-9]*)(?:\.[0-9]+)?(?:[eE][+-]?[0-9]+)?\Z') BUFFER_BYTES = 64 * 1024 SIGN = 1 << 63 FRACTION_MASK = (1 << 52) - 1 def double_from_bits(bits): return struct.unpack('=d', struct.pack('=Q', bits))[0] def double_bits(value): return struct.unpack('=Q', struct.pack('=d', value))[0] def finite_patterns(random_count=10000): # Every finite exponent binade, with exact powers and significand edges. values = {0, SIGN} for exponent in range(0x7ff): for fraction in (0, 1, (1 << 51) - 1, 1 << 51, FRACTION_MASK): bits = (exponent << 52) | fraction values.update((bits, bits | SIGN)) # Decimal carry/notation boundaries and adjacent representable values. for exponent in range(-323, 309): center = float(f'1e{exponent}') for value in (math.nextafter(center, 0), center, math.nextafter(center, math.inf)): if math.isfinite(value): values.update((double_bits(value), double_bits(-value))) random_source = random.Random(0x5259555F4A534F4E) added = 0 while added < random_count: bits = random_source.getrandbits(64) if (bits >> 52) & 0x7ff != 0x7ff: values.add(bits) added += 1 return sorted(values) class WriteStatus(ctypes.Structure): _fields_ = [ ('opened', ctypes.c_int), ('written', ctypes.c_int), ('closed', ctypes.c_int), ('start', ctypes.c_longlong), ('end', ctypes.c_longlong), ('final_position', ctypes.c_longlong), ] HARNESS = r''' #include #include #include #include #include "json_numbers.h" /* Faults intercept only the production writer's fwrite calls in this temporary translation unit. Real stdio still writes the accepted prefix. */ int test_write_fault_mode = 0; int test_write_call_count = 0; #ifdef TEST_FWRITE_FAULTS static size_t test_fwrite(const void *data, size_t size, size_t count, FILE *stream) { test_write_call_count++; if (test_write_fault_mode == 1 || (test_write_fault_mode == 3 && test_write_call_count >= 2)) return 0; if (test_write_fault_mode == 2) return count ? fwrite(data, size, count - 1, stream) : 0; return fwrite(data, size, count, stream); } #define fwrite test_fwrite #include "json_numbers.c" #undef fwrite #endif typedef struct { int opened, written, closed; long long start, end, final_position; } TestWriteStatus; void test_array_file(const char *path, const double *values, size_t count, size_t stride, size_t prefix_bytes, TestWriteStatus *status) { memset(status, 0, sizeof(*status)); FILE *stream = fopen(path, "wb"); if (!stream) return; status->opened = 1; for (size_t i = 0; i < prefix_bytes; i++) fputc('p', stream); status->start = (long long)ftell(stream); status->written = native_json_write_array(stream, values, count, stride); status->end = (long long)ftell(stream); if (status->written) fputs("TAIL", stream); status->final_position = (long long)ftell(stream); status->closed = fclose(stream) == 0; } void test_number_file(const char *path, double value, TestWriteStatus *status) { memset(status, 0, sizeof(*status)); FILE *stream = fopen(path, "wb"); if (!stream) return; status->opened = 1; status->written = native_json_write_number(stream, value); status->end = status->final_position = (long long)ftell(stream); status->closed = fclose(stream) == 0; } /* The number writer flushes its own block to FILE, not FILE's stdio buffer. The caller must still propagate a delayed fclose failure. */ int test_delayed_close_failure(const char *path, int *written, int *closed) { FILE *stream = fopen(path, "wb"); if (!stream) return 0; char buffer[4096]; if (setvbuf(stream, buffer, _IOFBF, sizeof(buffer))) { fclose(stream); return 0; } *written = native_json_write_number(stream, 0.1); *closed = fclose(stream) == 0; return 1; } ''' class NativeJsonWriterTests(unittest.TestCase): @classmethod def setUpClass(cls): command = shlex.split(os.environ.get('CC', '')) if not command: compiler = shutil.which('gcc') or shutil.which('clang') if not compiler: raise unittest.SkipTest('A native C compiler is required') command = [compiler] cls.compiler = command cls.directory = tempfile.TemporaryDirectory(prefix='native-json-writer-') cls.addClassCleanup(cls.directory.cleanup) cls.root = Path(cls.directory.name) cls.library = cls.build_library('ordinary', faults=False) cls.fault_library = cls.build_library('faults', faults=True) cls.portable_library = cls.build_library('portable-64-bit', faults=False, only_64_bit=True) cls.fault_mode = ctypes.c_int.in_dll(cls.fault_library, 'test_write_fault_mode') cls.fault_calls = ctypes.c_int.in_dll(cls.fault_library, 'test_write_call_count') @classmethod def build_library(cls, name, *, faults, only_64_bit=False): source = cls.root / f'{name}.c' source.write_text(HARNESS) library_path = cls.root / (name + ('.dll' if os.name == 'nt' else '.so')) command = cls.compiler + ['-std=c11', '-O2', '-Wall', '-Wextra', '-Werror', '-ffp-contract=off', '-fno-fast-math', '-shared'] if os.name != 'nt': command.append('-fPIC') if faults: command.append('-DTEST_FWRITE_FAULTS') if only_64_bit: command.append('-DRYU_ONLY_64_BIT_OPS') command += ['-I', str(ROOT / 'native/include'), '-I', str(ROOT / 'native/runtime'), str(source)] if not faults: command.append(str(ROOT / 'native/runtime/json_numbers.c')) command += [str(ROOT / 'native/encoding/ryu/d2s.c'), '-lm', '-o', str(library_path)] compiled = subprocess.run(command, capture_output=True, text=True, timeout=60) if compiled.returncode: raise AssertionError(compiled.stderr) library = ctypes.CDLL(str(library_path)) if os.name == 'nt': import _ctypes cls.addClassCleanup(_ctypes.FreeLibrary, library._handle) library.native_json_format_double.argtypes = [ctypes.c_void_p, ctypes.c_double] library.native_json_format_double.restype = ctypes.c_int library.native_json_write_number.argtypes = [ctypes.c_void_p, ctypes.c_double] library.native_json_write_number.restype = ctypes.c_int library.native_json_write_array.argtypes = [ctypes.c_void_p, ctypes.POINTER(ctypes.c_double), ctypes.c_size_t, ctypes.c_size_t] library.native_json_write_array.restype = ctypes.c_int library.test_array_file.argtypes = [ctypes.c_char_p, ctypes.POINTER(ctypes.c_double), ctypes.c_size_t, ctypes.c_size_t, ctypes.c_size_t, ctypes.POINTER(WriteStatus)] library.test_array_file.restype = None library.test_number_file.argtypes = [ctypes.c_char_p, ctypes.c_double, ctypes.POINTER(WriteStatus)] library.test_number_file.restype = None library.test_delayed_close_failure.argtypes = [ctypes.c_char_p, ctypes.POINTER(ctypes.c_int), ctypes.POINTER(ctypes.c_int)] library.test_delayed_close_failure.restype = ctypes.c_int return library def encode(self, value, library=None): # Sentinels bracket the promised 32-byte output, with no assumption that # the returned token is NUL-terminated. storage = (ctypes.c_ubyte * 34)(*([0xA5] * 34)) length = (library or self.library).native_json_format_double(ctypes.byref(storage, 1), value) self.assertEqual((storage[0], storage[33]), (0xA5, 0xA5)) self.assertGreater(length, 0) self.assertLessEqual(length, 32) return bytes(storage[1:1 + length]) def assert_roundtrip(self, text, expected_bits): self.assertRegex(text, JSON_NUMBER, f'Invalid JSON token for {expected_bits:016x}') decoded = json.loads(text) self.assertEqual(double_bits(float(decoded)), expected_bits, f'{expected_bits:016x} became {text!r} then {decoded!r}') def write_array(self, values, *, count=None, stride=1, prefix=0, library=None): array = (ctypes.c_double * len(values))(*values) if values else None status = WriteStatus() output = self.root / 'array.json' (library or self.library).test_array_file(os.fsencode(output), array, len(values) if count is None else count, stride, prefix, ctypes.byref(status)) self.assertTrue(status.opened) return output.read_bytes(), status def test_binary64_boundaries_and_seeded_random_values_roundtrip(self): for bits in finite_patterns(): self.assert_roundtrip(self.encode(double_from_bits(bits)), bits) def test_64_bit_fallback_roundtrips_the_same_binary64_corpus(self): patterns = finite_patterns() for bits in patterns: self.assert_roundtrip(self.encode(double_from_bits(bits), self.portable_library), bits) values = [double_from_bits(bits) for bits in patterns] ordinary, ordinary_status = self.write_array(values) portable, portable_status = self.write_array(values, library=self.portable_library) self.assertTrue(ordinary_status.written and ordinary_status.closed) self.assertTrue(portable_status.written and portable_status.closed) self.assertEqual(portable, ordinary) def test_plain_decimal_is_used_only_when_it_shortens_the_token(self): # Tie cases deliberately keep scientific notation; far exponents must # never be expanded to hundreds of zeroes in the 32-byte destination. cases = ((10.0, b'10'), (12.0, b'12'), (-12.0, b'-12'), (.1, b'0.1'), (-.1, b'-0.1'), (123.45, b'123.45'), (100.0, b'1E2'), (.01, b'1E-2'), (1e100, b'1E100'), (double_from_bits(1), b'5E-324')) for library in (self.library, self.portable_library): for value, expected in cases: with self.subTest(value=value, expected=expected): token = self.encode(value, library) self.assertEqual(token, expected) self.assert_roundtrip(token, double_bits(value)) def test_zero_sign_and_single_number_file(self): self.assertEqual(self.encode(0.0), b'0') self.assertEqual(self.encode(-0.0), b'-0.0') output = self.root / 'number.json' for bits in (0, SIGN, 1, SIGN | 1, 0x0010000000000000, 0x7fefffffffffffff): status = WriteStatus() self.library.test_number_file(os.fsencode(output), double_from_bits(bits), ctypes.byref(status)) self.assertTrue(status.opened and status.written and status.closed) data = output.read_bytes() self.assertEqual(status.end, len(data)) self.assert_roundtrip(data, bits) def test_array_crosses_block_boundaries_without_changing_offsets(self): cases = [([], 2), ([0.0] * 32767, BUFFER_BYTES - 1), ([-0.0] + [0.0] * 32765, BUFFER_BYTES), ([0.0] * 32768, BUFFER_BYTES + 1), ([double_from_bits(bits) for bits in finite_patterns(0)[::3]], None)] for values, expected_length in cases: for prefix in (0, 37, BUFFER_BYTES - 1): with self.subTest(values=len(values), expected_length=expected_length, prefix=prefix): data, status = self.write_array(values, prefix=prefix) self.assertTrue(status.written and status.closed) self.assertEqual(status.start, prefix) self.assertEqual(data[:prefix], b'p' * prefix) self.assertEqual(data[status.end:], b'TAIL') self.assertEqual(status.final_position, len(data)) token = data[status.start:status.end] if expected_length is not None: self.assertEqual(len(token), expected_length) decoded = json.loads(token) self.assertEqual([double_bits(float(v)) for v in decoded], [double_bits(v) for v in values]) def test_strided_array_keeps_selected_column_order(self): selected = [double_from_bits(bits) for bits in finite_patterns(0)[::13]] for stride in (1, 3, 17): with self.subTest(stride=stride): values = [math.nan] * (len(selected) * stride) values[::stride] = selected data, status = self.write_array(values, count=len(selected), stride=stride) self.assertTrue(status.written and status.closed) decoded = json.loads(data[:status.end]) self.assertEqual([double_bits(float(v)) for v in decoded], [double_bits(v) for v in selected]) def test_invalid_file_pointer_and_array_bounds_fail_before_access(self): value = (ctypes.c_double * 1)(.1) self.assertEqual(self.library.native_json_write_number(None, .1), 0) self.assertEqual(self.library.native_json_write_array(None, value, 1, 1), 0) for values, count, stride in (([], 1, 1), ([.1], 2, 0), ([.1], 2, ctypes.c_size_t(-1).value)): with self.subTest(count=count, stride=stride): data, status = self.write_array(values, count=count, stride=stride) self.assertTrue(status.opened and status.closed) self.assertFalse(status.written) self.assertEqual(data, b'') # A single sample never advances its pointer; an enormous stride is safe. data, status = self.write_array([.1], count=1, stride=ctypes.c_size_t(-1).value) self.assertTrue(status.written and status.closed) self.assertEqual(json.loads(data[:status.end]), [.1]) def test_nonfinite_numbers_fail_instead_of_emitting_invalid_json(self): output = self.root / 'nonfinite.json' for bits in (0x7ff0000000000000, 0xfff0000000000000, 0x7ff8000000000000, 0xfff8000000000001, 0x7ff0000000000001): value = double_from_bits(bits) with self.subTest(bits=f'{bits:016x}'): token = ctypes.create_string_buffer(32) self.assertEqual(self.library.native_json_format_double(token, value), 0) status = WriteStatus() self.library.test_number_file(os.fsencode(output), value, ctypes.byref(status)) self.assertTrue(status.opened and status.closed) self.assertFalse(status.written) self.assertEqual(output.read_bytes(), b'') for finite_prefix in ([], [0.0] * (BUFFER_BYTES + 1)): data, status = self.write_array(finite_prefix + [value]) self.assertFalse(status.written) self.assertTrue(status.closed) self.assertNotIn(b'NaN', data) self.assertNotIn(b'Infinity', data) def test_zero_and_short_writes_are_reported_without_stdio_error_flag(self): # The injected fwrite can return short without setting FILE's error bit; # relying only on ferror/fclose would incorrectly report success. output = self.root / 'fault-number.json' try: for mode in (1, 2): with self.subTest(mode=mode): self.fault_mode.value = mode self.fault_calls.value = 0 status = WriteStatus() self.fault_library.test_number_file(os.fsencode(output), .1, ctypes.byref(status)) self.assertTrue(status.opened and status.closed) self.assertFalse(status.written) self.assertGreater(self.fault_calls.value, 0) self.fault_calls.value = 0 _, status = self.write_array([1.0, 2.0], library=self.fault_library) self.assertFalse(status.written) self.assertTrue(status.closed) self.fault_mode.value = 3 self.fault_calls.value = 0 data, status = self.write_array([0.0] * (BUFFER_BYTES * 2), library=self.fault_library) self.assertFalse(status.written) self.assertTrue(status.closed) self.assertGreaterEqual(self.fault_calls.value, 2) self.assertTrue(data, 'The initial successful block must survive a later write failure') finally: self.fault_mode.value = 0 self.fault_calls.value = 0 @unittest.skipUnless(os.name == 'posix' and Path('/dev/full').exists(), '/dev/full is needed for a real delayed I/O failure') def test_caller_must_check_delayed_fclose_failure(self): written, closed = ctypes.c_int(), ctypes.c_int() self.assertEqual(self.library.test_delayed_close_failure(b'/dev/full', ctypes.byref(written), ctypes.byref(closed)), 1) self.assertEqual(written.value, 1, 'A small buffered token should not force FILE fflush') self.assertEqual(closed.value, 0, 'FILE close must expose the delayed device error') if __name__ == '__main__': unittest.main()