--- a/app/main.py +++ b/app/main.py @@ -22,6 +22,7 @@ from fastapi.responses import FileResponse, HTMLResponse, StreamingResponse from pydantic import BaseModel, ConfigDict, Field, ValidationError +from app.result_compression import SimulationResultCompressionMiddleware from app.simulation.performance import performance_span, profile_phase, profile_run from app.simulation.native_codegen.transport import NativeSeriesJson, serialize_result_parts from app.simulation.config import SolverActivityTracker @@ -58,6 +59,7 @@ title="System Simulation ReactFlow App", lifespan=_app_lifespan, ) +app.add_middleware(SimulationResultCompressionMiddleware) FRONTEND_DIST_DIR = Path(__file__).resolve().parent.parent / "frontend" / "dist" PROJECT_STORAGE_DIR = Path(__file__).parent / "data" / "reactflow-projects" SYSTEM_XML_SCHEMA_VERSION = "3" @@ -73,7 +75,7 @@ "postprocessing": "正在整理采样结果", "cancelled": "正在整理已终止仿真的部分结果", "failed": "正在整理异常终止前的部分结果", - "complete": "正在汇总仿真结果", + "complete": "计算已完成,正在接收仿真结果", } SIMULATION_STREAM_HEARTBEAT_SECONDS = 5.0 SIMULATION_TASK_RETENTION_SECONDS = 600.0 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -11341,6 +11341,8 @@ throw new SimulationStreamError("浏览器未收到仿真进度数据流"); } + const contentEncoding = response.headers.get("Content-Encoding")?.trim().toLowerCase(); + const requiresCompleteBody = Boolean(contentEncoding && contentEncoding !== "identity"); const reader = response.body.getReader(); let result: SimulationResult | null = null; let activityWatchdog = createSimulationActivityWatchdog(); @@ -11411,6 +11413,10 @@ while (true) { const { done, value } = await readSimulationStreamChunk(reader); if (value) lineDecoder.write(value); + // Identity responses can finish at the terminal record. Encoded bodies + // must reach EOF so the browser validates gzip's trailer (or equivalent + // encoding integrity); decoded JSON can arrive before that validation. + if (result && !requiresCompleteBody) break; if (done) { lineDecoder.finish(); break; @@ -11419,7 +11425,9 @@ } finally { abortController.abort(); try { - await reader.cancel(); + // cancel() closes the reader locally before its underlying-source promise + // settles; remote cleanup must not delay publishing a completed result. + void reader.cancel().catch(() => {}); } catch { // The stream may already be closed or aborted. } --- a/tests/manual/browser_stage_profile.mjs +++ b/tests/manual/browser_stage_profile.mjs @@ -1,4 +1,5 @@ -// Real production-page profiling. No route mocks, response cloning, or duplicate body parsing. +// Real-page profiling; production by default, development only with explicit opt-in. +// No route mocks, response cloning, or duplicate body parsing. // All stage timestamps use the active document's performance.now(). A reload starts a new axis. import { chromium } from '../../frontend/node_modules/playwright/index.mjs'; import fs from 'node:fs/promises'; @@ -6,11 +7,12 @@ import assert from 'node:assert/strict'; import { createHash } from 'node:crypto'; -const usage = `node tests/manual/browser_stage_profile.mjs --output DIR [--input tests/data/test-mql-8-corrected.json] [--url http://127.0.0.1:8011] [--runs 3 (0 for one smoke run)] [--mode both|profiled|control] [--deep] [--cpu-interval-us 1000] [--source-map-dir DIR] [--check] +const usage = `node tests/manual/browser_stage_profile.mjs --output DIR [--input tests/data/test-mql-8-corrected.json] [--url http://127.0.0.1:8011] [--runs 3 (0 for one smoke run)] [--mode both|profiled|control] [--deep] [--cpu-interval-us 1000] [--source-map-dir DIR] [--allow-development] [--check] Offline only: node tests/manual/browser_stage_profile.mjs --summarize-cpu-only EXISTING_DIRECTORY [--source-map-dir DIR] [--check] Each mode runs one warmup followed by RUNS measured runs, sequentially. --check validates inputs without launching a browser. Optional --deep (alias --cpu-profile) records renderer-main-thread .cpuprofile diagnostics separately from ordinary endpoint timing. --source-map-dir accepts an offline hidden-source-map build only when its generated JS bytes exactly match served assets. +--allow-development permits Vite development assets (for example forwarded port 5173); reports are labelled development and must not be mixed with production timings. Run with the repository Node 24 and Chromium runtime library environment. No application code is modified.`; const args = process.argv.slice(2); if (args.includes('--help')) { console.log(usage); process.exit(0); } @@ -18,6 +20,7 @@ for (let i = 0; i < args.length; i++) { if (['--deep', '--cpu-profile'].includes(args[i])) { options.deep = true; continue; } if (args[i] === '--check') { options.check = true; continue; } + if (args[i] === '--allow-development') { options.allowDevelopment = true; continue; } const cliKey = args[i].replace(/^--/, ''); const key = ({ 'cpu-interval-us': 'cpuIntervalUs', 'source-map-dir': 'sourceMapDir', 'summarize-cpu-only': 'summarizeCpuOnly' })[cliKey] ?? cliKey; if (!['input', 'output', 'url', 'runs', 'mode', 'cpuIntervalUs', 'sourceMapDir', 'summarizeCpuOnly'].includes(key) || !args[i + 1]) throw new Error(usage); @@ -63,6 +66,8 @@ Object.assign(data, { marks: {}, requests: [], reads: [], parses: [], decodes: [], transactions: [], workers: [], runFailure: undefined, downloads: [], longTasks: [], streamActive: false, activeRun: true }); performance.clearMarks(); + // Resource entries for this run must not be displaced by Vite startup imports. + performance.clearResourceTimings(); }; data.armClick = (name, selector) => { const listener = event => { @@ -83,7 +88,10 @@ const failure = document.querySelector('.simulation-console-dock-progress.error, .simulation-console-progress.error'); if (failure) { data.runFailure = failure.textContent; mark('runFailure'); observer.disconnect(); return; } const button = document.querySelector('button[aria-label="运行仿真"]'); - observedBusy ||= Boolean(button?.disabled || document.querySelector('.simulation-console-dock-progress.running, .simulation-console-progress.running')); + const runningProgress = document.querySelector('.simulation-console-dock-progress.running, .simulation-console-progress.running'); + observedBusy ||= Boolean(button?.disabled || runningProgress); + const progressText = runningProgress?.textContent ?? ''; + if (progressText.includes('正在汇总仿真结果') || progressText.includes('计算已完成,正在接收仿真结果')) mark('completeProgressDom'); const success = document.querySelector('.simulation-console-dock-progress.success, .simulation-console-progress.success'); if (observedBusy && button && !button.disabled && success?.textContent.includes('仿真完成')) { mark('resultReadyDom'); @@ -151,7 +159,8 @@ .filter(e => /simulate-stream|simulation-results\/csv/.test(e.name)) .map(e => ({ name: e.name, startTime: e.startTime, requestStart: e.requestStart, responseStart: e.responseStart, responseEnd: e.responseEnd, duration: e.duration, transferSize: e.transferSize, - encodedBodySize: e.encodedBodySize, decodedBodySize: e.decodedBodySize })) }); + encodedBodySize: e.encodedBodySize, decodedBodySize: e.decodedBodySize, + nextHopProtocol: e.nextHopProtocol, deliveryType: e.deliveryType ?? null })) }); if (location.hash === '#/results' && sessionStorage.getItem(storageKey)) { data.watchDom('restoredResults', '.results-shell .results-system-panel'); } @@ -169,6 +178,8 @@ if (kind === 'simulation') { mark('fetchStart', row.fetchStart); data.streamActive = true; } return Reflect.apply(originalFetch, this, args).then(response => { row.headers = performance.now(); row.status = response.status; + row.contentEncoding = response.headers.get('content-encoding'); + row.contentLength = response.headers.get('content-length'); if (kind === 'simulation') mark('headers', row.headers); if (response.body) bodies.set(response.body, row); return response; @@ -200,9 +211,14 @@ const parsed = Reflect.apply(nativeParse, this, args); const end = performance.now(); if (parsed && ['progress', 'result', 'error'].includes(parsed.event)) { - data.parses.push({ event: parsed.event, phase: parsed.phase, start, end, characters: typeof args[0] === 'string' ? args[0].length : null }); + data.parses.push({ event: parsed.event, phase: parsed.phase, heartbeat: Boolean(parsed.heartbeat), start, end, characters: typeof args[0] === 'string' ? args[0].length : null }); + if (parsed.event === 'progress' && parsed.phase === 'complete' && !parsed.heartbeat) { + mark('completeProgressParsed', end); + } if (parsed.event === 'result') { mark('resultParseStart', start); mark('resultParseEnd', end); + // A later suffix/newline/read may otherwise overwrite lastChunk. + if (data.marks.lastChunk !== undefined) mark('resultBoundaryReadAtParseStart', data.marks.lastChunk); // This microtask is only a checkpoint after the current consumer continuation, // not a claim that all EOF/finally/publish/React work has completed. queueMicrotask(() => mark('resultConsumerMicrotaskCheckpoint')); @@ -676,16 +692,17 @@ if (options.check) { checkCpuSummaryContract(); + checkFinalizationMeasurementContract(); if (options.summarizeCpuOnly) { const manifest = JSON.parse(await fs.readFile(path.join(options.summarizeCpuOnly, 'summary.json'))); assert.ok(manifest.cpuDiagnostics?.profiles?.length); console.log(JSON.stringify({ directory: options.summarizeCpuOnly, profiles: manifest.cpuDiagnostics.profiles.length, - cpuClassificationContractPassed: true, browserLaunched: false })); + cpuClassificationContractPassed: true, finalizationMeasurementContractPassed: true, browserLaunched: false })); } else { console.log(JSON.stringify({ input: options.input, inputSha256: sha(inputText), nodes: project.nodes.length, edges: project.edges.length, curveNodeId, mode: options.mode, deep: Boolean(options.deep), cpuIntervalUs: Number(options.cpuIntervalUs), - sourceMapDir: options.sourceMapDir ?? null, warmupsPerMode: 1, measuredRunsPerMode: Number(options.runs), - cpuClassificationContractPassed: true, browserLaunched: false })); + sourceMapDir: options.sourceMapDir ?? null, allowDevelopment: Boolean(options.allowDevelopment), warmupsPerMode: 1, measuredRunsPerMode: Number(options.runs), + cpuClassificationContractPassed: true, finalizationMeasurementContractPassed: true, browserLaunched: false })); } process.exit(0); } @@ -701,6 +718,9 @@ headersAndReads: 'fetch resolution and consumer read delivery. Outstanding-read intervals include backend production, transport and browser scheduling; not pure network time.', unobservedCpu: 'NDJSON fragment scanning/join/trim and React handler CPU are not isolated. Their residual intervals can also include scheduling and cannot be attributed wholesale to parsing, transport or drawing.', jsonParse: 'Only the original synchronous JSON.parse call, called once per application parse. No duplicate body read, decode, scan or parse.', + finalization: 'completeProgressDom is the first target text in the active running-progress DOM, observed in control and profiled modes; not a paint timestamp. completeProgressParsed is the first non-heartbeat complete progress JSON parse end, profiled only. The two origins are independent and may be absent or differently ordered. Endpoints include result parse/EOF/ready; null and negative deltas are retained. If the application finishes on the result event and cancels before EOF, EOF stays absent rather than being invented.', + responseEncoding: 'Content-Encoding/Content-Length are captured from real response headers without reading the body. Resource Timing encodedBodySize is compressed HTTP body size; decodedBodySize and reader byte counts are after content decoding. transferSize also includes browser-reported response-header overhead; none is a precise TLS/port-forward wire-byte measurement. Zero sizes may mean caching or unavailable/incomplete timing and are not replaced by decoded bytes. An early stream cancel may leave Resource Timing incomplete or absent.', + frontendEnvironment: 'Production assets are required by default. --allow-development permits observed Vite assets and labels the group development; development costs must not be combined with production groups. Asset hashes cover document tags and observed workers, not the entire Vite module graph.', resultReady: 'First DOM observation of successful completion plus an enabled run button after busy state. React state/handler boundaries are not directly instrumented.', indexedDb: 'Profiled: session pointer publication immediately after all save transactions commit. Control: pointer polling, up to 16 ms plus scheduling delay. Transaction windows also include asynchronous waiting and may include old-cache cleanup.', render: 'First visible DOM, then two requestAnimationFrame callbacks (paint opportunity, not GPU completion); stable means scoped DOM quiet for 120 ms followed by two frames.', @@ -716,6 +736,7 @@ const errors = []; const browser = await chromium.launch({ headless: true }); const evidence = { input: path.resolve(options.input), inputSha256: sha(inputText), baseURL: options.url, + allowDevelopment: Boolean(options.allowDevelopment), frontendEnvironment: null, browser: browser.version(), node: process.version, deep: Boolean(options.deep), cpuIntervalUs: options.deep ? Number(options.cpuIntervalUs) : null, initialNavigation: {}, scriptSha256: sha(await fs.readFile(new URL(import.meta.url))), definitions, rows, errors, servedAssets: [] }; const writeSummary = () => fs.writeFile(path.join(options.output, 'summary.json'), JSON.stringify(evidence, null, 2)); const waitMark = async (page, name) => { @@ -728,16 +749,40 @@ const watch = (page, name, selector) => page.evaluate(({ name, selector }) => window.__stageProfile.watchDom(name, selector), { name, selector }); const clickTab = async (page, name) => { await page.getByRole('tab', { name }).click(); }; const resultDigest = result => sha(JSON.stringify(result)); -const delta = (m, a, b) => m[a] === undefined || m[b] === undefined ? null : m[b] - m[a]; +function delta(m, a, b) { return m[a] === undefined || m[b] === undefined ? null : m[b] - m[a]; } function metrics(trace) { const m = trace.marks; const csvAnchor = trace.downloads.find(d => d.name.endsWith('.csv'))?.anchorClick; const resultAnchor = trace.downloads.find(d => d.name.endsWith('.simresult'))?.anchorClick; const csvRequest = trace.requests.find(r => r.kind === 'csv'); + const simulationRequest = trace.requests.find(r => r.kind === 'simulation'); + const simulationResponse = trace.simulationResponse ?? simulationRequest; + const resources = (trace.resources ?? []).filter(r => r.name.includes('/api/system-xml/simulate-stream')); + const resource = resources.length === 1 ? resources[0] : null; + const contentLength = simulationResponse?.contentLength; + const finalization = {}; + for (const [label, start] of [['completeDom', 'completeProgressDom'], ['completeParsed', 'completeProgressParsed']]) { + for (const [endpoint, end] of [['ReadyDom', 'resultReadyDom'], ['ReadyPaintOpportunity', 'resultReadyPaintOpportunity'], + ['ParseStart', 'resultParseStart'], ['ParseEnd', 'resultParseEnd'], ['Eof', 'streamEof'], + ['ResultBoundaryRead', 'resultBoundaryReadAtParseStart']]) { + finalization[`${label}To${endpoint}Ms`] = delta(m, start, end); + } + } const worker = trace.workers?.[0]; const workerStart = worker?.posts.find(p => p.type === 'start'); const workerFinish = worker?.posts.find(p => p.type === 'finish'); return { + ...finalization, + completeParsedToDomMs: delta(m, 'completeProgressParsed', 'completeProgressDom'), + resultBoundaryReadToParseStartMs: delta(m, 'resultBoundaryReadAtParseStart', 'resultParseStart'), + responseContentEncoding: simulationResponse?.contentEncoding ?? null, + responseContentLengthBytes: typeof contentLength === 'string' && /^\d+$/.test(contentLength) ? Number(contentLength) : null, + responseTransferEncoding: simulationResponse?.transferEncoding ?? null, + resourceTimingSimulationEntries: resources.length, + resourceTransferBytes: resource?.transferSize ?? null, + resourceEncodedBodyBytes: resource?.encodedBodySize ?? null, + resourceDecodedBodyBytes: resource?.decodedBodySize ?? null, + resourceDecodedToEncodedRatio: resource?.encodedBodySize > 0 && resource?.decodedBodySize > 0 ? resource.decodedBodySize / resource.encodedBodySize : null, importToDomMs: delta(m, 'importChange', 'importReadyDom'), importToPaintOpportunityMs: delta(m, 'importChange', 'importReadyPaintOpportunity'), clickToFetchMs: delta(m, 'runClick', 'fetchStart'), fetchToHeadersMs: delta(m, 'fetchStart', 'headers'), @@ -773,6 +818,49 @@ synchronousStreamDecodeMs: trace.profiled ? trace.decodes.reduce((n, r) => n + r.end - r.start, 0) : null, }; } +function classifyFrontendEnvironment(assetUrls, allowDevelopment) { + const environment = assetUrls.some(url => /@vite\/client|@react-refresh|\/src\//.test(url)) ? 'development' : 'production'; + if (environment === 'development' && !allowDevelopment) { + throw new Error('Expected a production build, found Vite development assets. Use --allow-development for a separately labelled development measurement.'); + } + return environment; +} + +function checkFinalizationMeasurementContract() { + const trace = { profiled: true, marks: { completeProgressParsed: 8, completeProgressDom: 10, + resultBoundaryReadAtParseStart: 35, resultParseStart: 40, resultParseEnd: 100, resultReadyDom: 120 }, + requests: [], reads: [], parses: [], decodes: [], workers: [], downloads: [], + simulationResponse: { contentEncoding: 'gzip', contentLength: '100' }, + resources: [{ name: 'http://localhost/api/system-xml/simulate-stream', transferSize: 120, encodedBodySize: 100, decodedBodySize: 1000 }] }; + const row = metrics(trace); + assert.equal(row.completeDomToReadyDomMs, 110); + assert.equal(row.completeParsedToReadyDomMs, 112); + assert.equal(row.completeParsedToDomMs, 2); + assert.equal(row.completeParsedToParseStartMs, 32); + assert.equal(row.completeParsedToParseEndMs, 92); + assert.equal(row.completeParsedToEofMs, null, 'Early result publication must not invent EOF.'); + assert.equal(row.resultBoundaryReadToParseStartMs, 5); + assert.equal(row.responseContentEncoding, 'gzip'); + assert.equal(row.responseContentLengthBytes, 100); + assert.equal(row.resourceEncodedBodyBytes, 100); + assert.equal(row.resourceDecodedBodyBytes, 1000); + assert.equal(row.resourceDecodedToEncodedRatio, 10); + assert.equal(metrics({ ...trace, marks: { ...trace.marks, completeProgressDom: 60 } }).completeDomToParseStartMs, -20); + const control = metrics({ ...trace, profiled: false, marks: { completeProgressDom: 10, resultReadyDom: 120 }, resources: [] }); + assert.equal(control.completeDomToReadyDomMs, 110); + assert.equal(control.completeParsedToReadyDomMs, null); + assert.equal(control.completeDomToParseStartMs, null); + assert.equal(control.streamBytes, null); + assert.equal(control.resourceEncodedBodyBytes, null); + const zero = metrics({ ...trace, resources: [{ ...trace.resources[0], transferSize: 0, encodedBodySize: 0 }] }); + assert.equal(zero.resourceEncodedBodyBytes, 0); + assert.equal(zero.resourceDecodedToEncodedRatio, null, 'Missing wire-size evidence must not be replaced by decoded size.'); + assert.equal(metrics({ ...trace, resources: [...trace.resources, ...trace.resources] }).resourceEncodedBodyBytes, null); + assert.equal(classifyFrontendEnvironment(['http://localhost/assets/index-abc.js'], false), 'production'); + assert.equal(classifyFrontendEnvironment(['http://localhost/@vite/client'], true), 'development'); + assert.throws(() => classifyFrontendEnvironment(['http://localhost/src/main.tsx'], false), /--allow-development/); +} + try { for (const mode of options.mode === 'both' ? ['control', 'profiled'] : [options.mode]) { const context = await browser.newContext({ viewport: { width: 1600, height: 1000 }, acceptDownloads: true }); @@ -781,12 +869,25 @@ const cpuRecorder = options.deep ? await createCpuRecorder(context, page, Number(options.cpuIntervalUs)) : null; page.setDefaultTimeout(30000); const simulationRequests = []; + const requestRecords = new WeakMap(); const workerUrls = new Set(); page.on('worker', worker => workerUrls.add(worker.url())); page.on('request', request => { if (request.url().includes('/api/system-xml/simulate-stream')) { - simulationRequests.push({ url: request.url(), simulationId: request.headers()['x-simulation-id'] ?? null }); + const row = { url: request.url(), simulationId: request.headers()['x-simulation-id'] ?? null }; + simulationRequests.push(row); + requestRecords.set(request, row); } + }); + // Passive header metadata in both modes; never response.body/text/json. + page.on('response', response => { + const row = requestRecords.get(response.request()); + if (!row) return; + const headers = response.headers(); + row.response = { status: response.status(), contentEncoding: headers['content-encoding'] ?? null, + contentLength: headers['content-length'] ?? null, transferEncoding: headers['transfer-encoding'] ?? null, + contentType: headers['content-type'] ?? null, vary: headers.vary ?? null, + fromServiceWorker: response.fromServiceWorker(), source: 'playwright-response-headers' }; }); page.on('pageerror', error => errors.push({ mode, error: String(error) })); page.on('dialog', dialog => { errors.push({ mode, dialog: dialog.message() }); void dialog.dismiss(); }); @@ -797,16 +898,18 @@ appControlObservedAt: performance.now(), navigation: performance.getEntriesByType('navigation').map(e => e.toJSON()) })); const assetUrls = await page.evaluate(() => [...document.querySelectorAll('script[src],link[rel="stylesheet"][href]')] .map(e => e.src || e.href)); - if (assetUrls.some(url => /@vite\/client|\/src\//.test(url))) throw new Error('Expected a production build, found Vite development assets.'); + const frontendEnvironment = classifyFrontendEnvironment(assetUrls, Boolean(options.allowDevelopment)); + if (evidence.frontendEnvironment !== null) assert.equal(evidence.frontendEnvironment, frontendEnvironment, 'Frontend environment changed between modes.'); + evidence.frontendEnvironment = frontendEnvironment; for (const url of assetUrls) { const response = await context.request.get(url); assert.ok(response.ok(), `Asset HTTP ${response.status()}: ${url}`); const bytes = await response.body(); const previous = evidence.servedAssets.find(asset => asset.url === url); - if (previous) assert.equal(previous.sha256, sha(bytes), 'Production asset changed between modes.'); + if (previous) assert.equal(previous.sha256, sha(bytes), 'Frontend asset changed between modes.'); else evidence.servedAssets.push({ url, sha256: sha(bytes), bytes: bytes.length }); } - assert.ok(evidence.servedAssets.length, 'No production assets found.'); + assert.ok(evidence.servedAssets.length, 'No frontend assets found.'); evidence.buildAssetSetSha256 = sha(JSON.stringify(evidence.servedAssets .map(({ url, ...asset }) => ({ path: new URL(url).pathname, ...asset })) .sort((a, b) => a.path.localeCompare(b.path)))); @@ -871,6 +974,11 @@ await saveDownload('下载结果 CSV', 'csvClick', 'csvDownloadSaved', 'result.csv'); await saveDownload('下载结果文件', 'resultFileClick', 'resultFileDownloadSaved', 'result.simresult'); const trace = await page.evaluate(() => window.__stageProfile.snapshot()); + const requests = simulationRequests.slice(requestOffset); + assert.equal(requests.length, 1, 'Expected exactly one real simulation request.'); + assert.ok(requests[0].simulationId, 'Missing X-Simulation-Id correlation key.'); + assert.ok(requests[0].response, 'Missing simulation response-header evidence.'); + trace.simulationResponse = { simulationId: requests[0].simulationId, ...requests[0].response }; if (cpuRecorder) { const capture = await cpuRecorder.stop(); const file = path.join(runDir, 'interaction.cpuprofile'); @@ -908,10 +1016,7 @@ const restoreTrace = await page.evaluate(() => ({ ...window.__stageProfile.snapshot(), navigation: performance.getEntriesByType('navigation').map(e => e.toJSON()) })); await fs.writeFile(path.join(runDir, 'restore-trace.json'), JSON.stringify(restoreTrace, null, 2)); - const requests = simulationRequests.slice(requestOffset); - assert.equal(requests.length, 1, 'Expected exactly one real simulation request.'); - assert.ok(requests[0].simulationId, 'Missing X-Simulation-Id correlation key.'); - const row = { mode, deep: Boolean(options.deep), run, warmup: run === 0, simulationId: requests[0].simulationId, timeOrigin: trace.timeOrigin, ...metrics(trace), + const row = { mode, frontendEnvironment, deep: Boolean(options.deep), run, warmup: run === 0, simulationId: requests[0].simulationId, timeOrigin: trace.timeOrigin, ...metrics(trace), restoreNavigationToDomMs: restoreTrace.marks.restoredResultsDom, restoreNavigationToPaintOpportunityMs: restoreTrace.marks.restoredResultsPaintOpportunity, restoreNavigationToDomStableMs: restoreTrace.marks.restoredResultsStable, --- a/requirements.txt +++ b/requirements.txt @@ -4,3 +4,6 @@ lxml>=5,<7 pydantic>=2,<3 uvicorn[standard] + +# Streaming gzip must flush progress and offload large compression blocks. +starlette>=1.6,<2 --- a/tests/test_native_result_transport.py +++ b/tests/test_native_result_transport.py @@ -1,6 +1,9 @@ """The HTTP fast path must preserve real native values and task semantics.""" from dataclasses import replace +import gzip import json +import struct +import zlib from pathlib import Path import tempfile from types import SimpleNamespace @@ -23,7 +26,7 @@ """Exercise real routing/response bodies without an optional HTTP client dependency.""" def __init__(self, application): self.application = application def post(self, path, *, content=b'', headers=None): return self.request('POST', path, content, headers) - def get(self, path): return self.request('GET', path, b'', None) + def get(self, path, *, headers=None): return self.request('GET', path, b'', headers) def request(self, method, path, content, headers): async def run(): messages = [] @@ -37,7 +40,11 @@ await self.application(scope,receive,send) status = next(m['status'] for m in messages if m['type']=='http.response.start') body = b''.join(m.get('body',b'') for m in messages if m['type']=='http.response.body') - return SimpleNamespace(status_code=status,content=body,json=lambda:json.loads(body)) + response_headers = {key.decode('latin-1'): value.decode('latin-1') + for key, value in next(m['headers'] for m in messages if m['type']=='http.response.start')} + chunks = tuple(m.get('body', b'') for m in messages if m['type']=='http.response.body') + return SimpleNamespace(status_code=status, content=body, headers=response_headers, + chunks=chunks, json=lambda:json.loads(body)) return asyncio.run(run()) @@ -95,6 +102,71 @@ for key in ('series', 'final', 'variables', 'model', 'simulation'): self.assertEqual(synchronous[key], result[key]) + def test_real_http_gzip_stream_and_retained_task_preserve_identity_results(self): + client = AsgiClient(app) + ident = 'transport-gzip-'+uuid4().hex + def load(data): + return json.loads(data, parse_int=lambda token: -0.0 if token == '-0' else int(token)) + def decode(response, *, progress=False): + self.assertEqual(response.status_code, 200) + self.assertEqual(response.headers['content-encoding'], 'gzip') + self.assertIn('accept-encoding', response.headers['vary'].lower()) + decoder = zlib.decompressobj(zlib.MAX_WBITS + 16) + parts = [] + first_progress = False + for index, chunk in enumerate(response.chunks): + part = decoder.decompress(chunk) + parts.append(part) + if progress and not first_progress and b'\n' in part: + self.assertEqual(load(part.split(b'\n', 1)[0])['event'], 'progress') + self.assertLess(index, len(response.chunks)-1) + self.assertFalse(decoder.eof, 'Progress must precede gzip stream completion') + first_progress = True + parts.append(decoder.flush()) + decoded = b''.join(parts) + self.assertTrue(decoder.eof, 'The real HTTP response needs its complete gzip trailer') + self.assertEqual(decoder.unused_data, b'') + self.assertEqual(decoder.unconsumed_tail, b'') + self.assertEqual(gzip.decompress(response.content), decoded) + if progress: + self.assertTrue(first_progress) + return decoded + compressed = client.post('/api/system-xml/simulate-stream', content=self.xml, + headers={'X-Simulation-Id': ident, 'Accept-Encoding': 'gzip'}) + events = [load(line) for line in decode(compressed, progress=True).splitlines()] + results = [event['result'] for event in events if event['event'] == 'result'] + self.assertEqual(len(results), 1) + result = results[0] + self.assertTrue(result['success']) + self.assertEqual(result['simulatedUntil'], .1) + identity = client.post('/api/system-xml/simulate-stream', content=self.xml, + headers={'X-Simulation-Id': ident+'-identity', 'Accept-Encoding': 'identity'}) + self.assertNotIn('content-encoding', identity.headers) + identity_result = next(event['result'] for event in map(load, identity.content.splitlines()) + if event['event'] == 'result') + for key in ('success', 'status', 'partial', 'simulatedUntil', 'requestedStopTime', + 'variables', 'model', 'simulation'): + self.assertEqual(result[key], identity_result[key], key) + self.assertEqual(result['diagnostics']['integration']['totals'], + identity_result['diagnostics']['integration']['totals']) + # Compare numeric bits, including signed zero, independently of gzip's + # byte oracle; timings legitimately vary between these two actual runs. + self.assertEqual(result['series'].keys(), identity_result['series'].keys()) + for key, values in result['series'].items(): + expected = identity_result['series'][key] + self.assertEqual(len(values), len(expected), key) + self.assertEqual(b''.join(struct.pack('!d', value) for value in values), + b''.join(struct.pack('!d', value) for value in expected), key) + self.assertEqual(result['final'].keys(), identity_result['final'].keys()) + for key, value in result['final'].items(): + self.assertEqual(struct.pack('!d', value), struct.pack('!d', identity_result['final'][key]), key) + retained = client.get('/api/system-xml/simulations/'+ident, headers={'Accept-Encoding': 'gzip'}) + retained_bytes = decode(retained) + retained_identity = client.get('/api/system-xml/simulations/'+ident, + headers={'Accept-Encoding': 'identity'}) + self.assertEqual(retained_bytes, retained_identity.content) + self.assertEqual(load(retained_bytes)['result'], result) + def test_cancelled_raw_stream_keeps_partial_result_and_public_status(self): for reason, status in [('user', 'stopped'), ('stalled', 'stalled')]: task = _register_simulation_task('transport-'+uuid4().hex) --- /dev/null +++ b/app/result_compression.py @@ -0,0 +1,84 @@ +"""Lossless, promptly flushed compression for large simulation responses. + +Starlette >= 1.6 flushes each streaming body and moves large compression +blocks off the event loop. Bound its input so transmission can overlap the +encoding of the next block, including when the native series is one big body. +""" + +from __future__ import annotations + +import re + +from starlette.datastructures import Headers +from starlette.middleware.gzip import GZipResponder, IdentityResponder +from starlette.types import ASGIApp, Message, Receive, Scope, Send + + +RESULT_COMPRESSION_CHUNK_BYTES = 256 * 1024 +_QUALITY = re.compile(r"(?:0(?:\.\d{0,3})?|1(?:\.0{0,3})?)\Z") + + +def _accepts_gzip(headers: Headers) -> bool: + qualities: dict[str, float] = {} + for header in headers.getlist("accept-encoding"): + for item in header.split(","): + coding, *parameters = item.lower().strip().split(";") + coding = coding.strip() + quality = 1.0 + for parameter in parameters: + key, separator, value = parameter.strip().partition("=") + if key.strip() == "q": + value = value.strip() + quality = min(quality, float(value) if separator and _QUALITY.fullmatch(value) else 0.0) + # A repeated prohibition takes precedence over another entry. + qualities[coding] = min(qualities.get(coding, 1.0), quality) + gzip_quality = qualities.get("gzip", qualities.get("*", 0.0)) + return gzip_quality > 0.0 and gzip_quality >= qualities.get("identity", 0.0) + + +def _is_result_request(scope: Scope) -> bool: + path = scope.get("path", "") + method = scope.get("method", "") + if method == "POST": + return path in {"/api/system-xml/simulate", "/api/system-xml/simulate-stream"} + prefix = "/api/system-xml/simulations/" + if method == "GET" and path.startswith(prefix): + identifier = path[len(prefix):] + return bool(identifier) and "/" not in identifier + return False + + +class SimulationResultCompressionMiddleware: + def __init__(self, app: ASGIApp) -> None: + self.app = app + + async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: + if scope["type"] != "http" or not _is_result_request(scope): + await self.app(scope, receive, send) + return + if not _accepts_gzip(Headers(scope=scope)): + await IdentityResponder(self.app, minimum_size=500)(scope, receive, send) + return + + async def bounded_app(scope: Scope, receive: Receive, send: Send) -> None: + can_split = True + + async def send_bounded(message: Message) -> None: + nonlocal can_split + if message["type"] == "http.response.start": + can_split = message["status"] != 206 and "content-encoding" not in Headers(raw=message["headers"]) + body = message.get("body", b"") + if message["type"] == "http.response.body" and can_split and len(body) > RESULT_COMPRESSION_CHUNK_BYTES: + for offset in range(0, len(body), RESULT_COMPRESSION_CHUNK_BYTES): + end = offset + RESULT_COMPRESSION_CHUNK_BYTES + await send({ + **message, + "body": body[offset:end], + "more_body": end < len(body) or message.get("more_body", False), + }) + else: + await send(message) + + await self.app(scope, receive, send_bounded) + + await GZipResponder(bounded_app, minimum_size=500, compresslevel=1)(scope, receive, send) --- /dev/null +++ b/frontend/tests/e2e/simulation-stream-terminal.spec.ts @@ -0,0 +1,183 @@ +import { expect, test, type Page } from "@playwright/test"; +import { expandSimulationConsole, prepareApp, resultSnapshot, wideProject } from "./fixtures"; + +async function prepareOpenStream(page: Page, cancelBehavior: "pending" | "reject" = "pending", encoding?: string) { + await prepareApp(page); + await page.addInitScript(({ cancelBehavior, encoding }) => { + const nativeFetch = window.fetch.bind(window); + const probe = { + body: null as ReadableStream | null, + signal: null as AbortSignal | null, + enqueue: (_text: string) => {}, + close: () => {}, + fail: (_message: string) => {}, + closedByServer: false, + cancellations: 0, + unhandled: [] as string[], + }; + (window as any).__terminalStream = probe; + window.addEventListener("unhandledrejection", (event) => probe.unhandled.push(String(event.reason))); + window.fetch = async (input, init) => { + if (!String(input).includes("/api/system-xml/simulate-stream")) return nativeFetch(input, init); + const encoder = new TextEncoder(); + probe.signal = init?.signal ?? null; + probe.body = new ReadableStream({ + start(controller) { + probe.enqueue = (text) => controller.enqueue(encoder.encode(text)); + probe.close = () => { probe.closedByServer = true; controller.close(); }; + probe.fail = (message) => controller.error(new TypeError(message)); + }, + cancel() { + probe.cancellations++; + // The response does not reach EOF; underlying cleanup may also never + // settle or reject. Neither is allowed to block a terminal result. + return cancelBehavior === "pending" + ? new Promise(() => {}) + : Promise.reject(new Error("test transport cancellation rejection")); + }, + }); + const headers = new Headers({ "Content-Type": "application/x-ndjson" }); + if (encoding) headers.set("Content-Encoding", encoding); + return new Response(probe.body, { headers }); + }; + }, { cancelBehavior, encoding }); + await page.goto("/"); + await page.locator('input[type="file"]').setInputFiles({ + name: "terminal-stream.json", mimeType: "application/json", + buffer: Buffer.from(JSON.stringify({ ...wideProject, edges: [...wideProject.edges, { + id: "close-test-loop", source: "generic_sensor_4", target: "generic_sensor_1", + sourceHandle: "port_b", targetHandle: "port_a", data: { isContactEdge: false }, + }] })), + }); + await expect(page.getByRole("textbox", { name: "工程", exact: true })).toHaveValue("terminal-stream"); + await page.getByRole("button", { name: "运行仿真", exact: true }).click(); + await expect.poll(() => page.evaluate(() => Boolean((window as any).__terminalStream.body))).toBe(true); + await expandSimulationConsole(page); +} + +async function expectReleasedStream(page: Page) { + expect(await page.evaluate(() => { + const probe = (window as any).__terminalStream; + return { aborted: probe.signal.aborted, locked: probe.body.locked, + cancellations: probe.cancellations, closedByServer: probe.closedByServer, unhandled: probe.unhandled }; + })).toEqual({ aborted: true, locked: false, cancellations: 1, closedByServer: false, unhandled: [] }); +} + +for (const status of ["completed", "stopped"] as const) { + test(`${status} result is published before EOF and transport cleanup, preserving every value`, async ({ page }) => { + const errors: string[] = []; + page.on("pageerror", (error) => errors.push(error.message)); + await prepareOpenStream(page, status === "completed" ? "pending" : "reject", status === "stopped" ? "identity" : undefined); + const result = { + ...resultSnapshot.result, status, success: status === "completed", partial: status === "stopped", + simulatedUntil: status === "completed" ? 10 : 4, + series: { time: status === "completed" ? [0, 5, 10] : [0, 2, 4], + "generic_sensor_1.value": [Number.MIN_VALUE, -1.2345678901234567, Number.MAX_VALUE] }, + final: { "generic_sensor_1.value": Number.MAX_VALUE }, + }; + const panel = page.getByRole("complementary", { name: "仿真控制台", exact: true }); + const run = page.getByRole("button", { name: "运行仿真", exact: true }); + // The complete JSON text alone is not yet a complete NDJSON record. + await page.evaluate((result) => { + (window as any).__terminalStream.enqueue(JSON.stringify({ event: "progress", phase: "complete", + progress: 100, simulatedTime: result.simulatedUntil, totalTime: 10, message: "正在汇总仿真结果" }) + "\n" + + JSON.stringify({ event: "result", result })); + }, result); + await expect(panel).toContainText("正在汇总仿真结果"); + await expect(run).toBeDisabled(); + await page.evaluate(() => (window as any).__terminalStream.enqueue("\n")); + // This deadline is below the 30-second idle timeout; EOF is never sent. + await expect(run).toBeEnabled({ timeout: 3000 }); + await expect(panel).toContainText(status === "completed" ? "仿真完成,已生成新的结果" : "已手动终止"); + await expect(panel).not.toContainText("仿真失败"); + await expectReleasedStream(page); + await page.getByRole("tab", { name: /^结果/ }).click(); + const pendingDownload = page.waitForEvent("download"); + await page.getByRole("button", { name: "下载结果文件", exact: true }).click(); + const stream = await (await pendingDownload).createReadStream(); + expect(stream).not.toBeNull(); + const chunks: Buffer[] = []; + for await (const chunk of stream!) chunks.push(Buffer.from(chunk)); + const exported = JSON.parse(Buffer.concat(chunks).toString("utf8")); + expect(exported.snapshot.result).toEqual(result); + expect(errors).toEqual([]); + await expectReleasedStream(page); + }); +} + +test("an error before any terminal result releases an open stream without waiting for cleanup", async ({ page }) => { + const errors: string[] = []; + page.on("pageerror", (error) => errors.push(error.message)); + await prepareOpenStream(page); + await page.evaluate(() => (window as any).__terminalStream.enqueue(JSON.stringify({ + event: "error", message: "测试结果生成失败", status: 422, + }) + "\n")); + await expect(page.getByRole("button", { name: "运行仿真", exact: true })).toBeEnabled({ timeout: 3000 }); + await expect(page.getByRole("complementary", { name: "仿真控制台", exact: true })).toContainText("测试结果生成失败"); + await expectReleasedStream(page); + expect(errors).toEqual([]); +}); + +for (const truncated of [false, true]) { + test(`EOF ${truncated ? "rejects truncated JSON" : "accepts a complete result without a trailing newline"}`, async ({ page }) => { + await prepareOpenStream(page); + await page.evaluate(({ result, truncated }) => { + const probe = (window as any).__terminalStream; + const line = JSON.stringify({ event: "result", result }); + probe.enqueue(truncated ? line.slice(0, -1) : line); + probe.close(); + }, { result: resultSnapshot.result, truncated }); + await expect(page.getByRole("button", { name: "运行仿真", exact: true })).toBeEnabled({ timeout: 3000 }); + const panel = page.getByRole("complementary", { name: "仿真控制台", exact: true }); + await expect(panel).toContainText(truncated ? "仿真失败" : "仿真完成,已生成新的结果"); + expect(await page.evaluate(() => { + const probe = (window as any).__terminalStream; + return { aborted: probe.signal.aborted, locked: probe.body.locked, closedByServer: probe.closedByServer }; + })).toEqual({ aborted: true, locked: false, closedByServer: true }); + }); +} + +for (const invalidTrailer of [false, true]) { + test(`encoded result ${invalidTrailer ? "rejects a later decoding error" : "waits for validated EOF before publication"}`, async ({ page }) => { + const errors: string[] = []; + page.on("pageerror", (error) => errors.push(error.message)); + await prepareOpenStream(page, "pending", "gzip"); + // Fetch exposes decoded bytes while retaining Content-Encoding. This mock + // represents the browser delivering JSON before validating gzip's trailer; + // gzip wire encoding itself belongs to the HTTP/backend integration tests. + await page.evaluate((result) => { + (window as any).__terminalStream.enqueue(JSON.stringify({ event: "progress", phase: "complete", + progress: 100, simulatedTime: 10, totalTime: 10, message: "正在汇总仿真结果" }) + "\n" + + JSON.stringify({ event: "result", result }) + "\n"); + }, resultSnapshot.result); + const panel = page.getByRole("complementary", { name: "仿真控制台", exact: true }); + const run = page.getByRole("button", { name: "运行仿真", exact: true }); + await expect(panel).toContainText("正在汇总仿真结果"); + await expect(run).toBeDisabled(); + expect(await page.evaluate(() => { + const probe = (window as any).__terminalStream; + return { locked: probe.body.locked, aborted: probe.signal.aborted }; + })).toEqual({ locked: true, aborted: false }); + await page.evaluate((invalidTrailer) => { + const probe = (window as any).__terminalStream; + if (invalidTrailer) probe.fail("gzip trailer checksum failed"); + else probe.close(); + }, invalidTrailer); + await expect(run).toBeEnabled({ timeout: 3000 }); + if (invalidTrailer) { + await expect(panel).toContainText("gzip trailer checksum failed"); + await expect(panel).not.toContainText("仿真完成,已生成新的结果"); + expect(await page.evaluate(() => sessionStorage.getItem("system-simulation-flow:latest-result"))).toBeNull(); + } else { + await expect(panel).toContainText("仿真完成,已生成新的结果"); + await expect(panel).not.toContainText("仿真失败"); + await page.getByRole("tab", { name: /^结果/ }).click(); + await expect(page.getByRole("button", { name: "下载结果文件", exact: true })).toBeVisible(); + } + expect(await page.evaluate(() => { + const probe = (window as any).__terminalStream; + return { locked: probe.body.locked, aborted: probe.signal.aborted, unhandled: probe.unhandled }; + })).toEqual({ locked: false, aborted: true, unhandled: [] }); + expect(errors).toEqual([]); + }); +} --- /dev/null +++ b/tests/test_simulation_result_compression.py @@ -0,0 +1,251 @@ +"""Lossless, live ASGI result compression without a model or HTTP client. + +The wire oracle is Python's independent gzip/zlib decoder. Tests assert emitted +bytes and request/response semantics, not the compressor's internal algorithm. +""" +from __future__ import annotations + +import asyncio +import gzip +import hashlib +import unittest +import zlib + +from app.result_compression import SimulationResultCompressionMiddleware + + +STREAM_PATH = '/api/system-xml/simulate-stream' +CHUNK_BYTES = 256 * 1024 + + +def http_scope(method='POST', path=STREAM_PATH, accept_encoding='gzip'): + return { + 'type': 'http', 'asgi': {'version': '3.0', 'spec_version': '2.4'}, + 'http_version': '1.1', 'method': method, 'scheme': 'http', + 'path': path, 'raw_path': path.encode(), 'query_string': b'', + 'root_path': '', 'server': ('testserver', 80), + 'client': ('127.0.0.1', 1234), + 'headers': ([] if accept_encoding is None else + [(b'accept-encoding', value.encode('ascii')) + for value in ([accept_encoding] if isinstance(accept_encoding, str) else accept_encoding)]), + } + + +def copied_message(message): + result = dict(message) + if 'headers' in result: + result['headers'] = list(result['headers']) + return result + + +def response_headers(messages): + return dict(next(m['headers'] for m in messages if m['type'] == 'http.response.start')) + + +def response_bodies(messages): + return [m for m in messages if m['type'] == 'http.response.body'] + + +async def receive(): + return {'type': 'http.request', 'body': b'', 'more_body': False} + + +class SimulationResultCompressionTests(unittest.IsolatedAsyncioTestCase): + async def request(self, body, *, method='POST', path=STREAM_PATH, + accept_encoding='gzip', status=200, headers=None): + original = [ + {'type': 'http.response.start', 'status': status, + 'headers': list(headers if headers is not None else [ + (b'content-type', b'application/json'), + (b'content-length', str(len(body)).encode()), + ])}, + {'type': 'http.response.body', 'body': body, 'more_body': False}, + ] + async def application(scope, incoming, send): + self.assertEqual(scope['method'], method) + for message in original: + await send(copied_message(message)) + messages = [] + async def send(message): + messages.append(copied_message(message)) + await SimulationResultCompressionMiddleware(application)( + http_scope(method, path, accept_encoding), receive, send) + return messages, original + + def assert_complete_gzip(self, messages, expected): + encoded = b''.join(m.get('body', b'') for m in response_bodies(messages)) + self.assertEqual(gzip.decompress(encoded), expected) + decoder = zlib.decompressobj(zlib.MAX_WBITS + 16) + self.assertEqual(decoder.decompress(encoded) + decoder.flush(), expected) + self.assertTrue(decoder.eof, 'The gzip trailer must be complete') + self.assertEqual(decoder.unused_data, b'', 'No bytes may follow the gzip stream') + self.assertEqual(decoder.unconsumed_tail, b'') + headers = response_headers(messages) + self.assertEqual(headers[b'content-encoding'], b'gzip') + vary = [part.strip().lower() for part in headers.get(b'vary', b'').split(b',')] + self.assertIn(b'accept-encoding', vary) + if b'content-length' in headers: + self.assertEqual(int(headers[b'content-length']), len(encoded)) + bodies = response_bodies(messages) + self.assertTrue(bodies) + self.assertFalse(bodies[-1].get('more_body', False)) + self.assertTrue(all(m.get('more_body', False) for m in bodies[:-1])) + + async def test_progress_is_decodable_before_application_sends_result(self): + progress = '{"event":"progress","message":"正在计算雪的温度","progress":20}\n'.encode() + result = ('{"event":"result","series":{"x":[5e-324,-0,-0.0,1e-300]},' + '"message":"温度与压力\\n已完成","padding":"' + 'x' * (CHUNK_BYTES + 200) + '"}\n').encode() + decoder = zlib.decompressobj(zlib.MAX_WBITS + 16) + received = bytearray() + messages = [] + result_started = False + async def send(message): + messages.append(copied_message(message)) + if message['type'] == 'http.response.body': + received.extend(decoder.decompress(message.get('body', b''))) + if not result_started: + self.assertFalse(decoder.eof) + async def application(scope, incoming, send): + await send({'type': 'http.response.start', 'status': 200, 'headers': [ + (b'content-type', b'application/x-ndjson'), + (b'cache-control', b'no-cache, no-transform'), + (b'x-accel-buffering', b'no'), (b'vary', b'Origin'), + ]}) + await send({'type': 'http.response.body', 'body': progress, 'more_body': True}) + self.assertEqual(bytes(received), progress, + 'Progress must be readable before the app computes/sends its result') + nonlocal result_started + result_started = True + # Arbitrary UTF-8/JSON boundaries must not change a byte. + utf8_split = result.index('温度'.encode()) + 1 + for body in (result[:7], result[7:utf8_split], result[utf8_split:]): + await send({'type': 'http.response.body', 'body': body, 'more_body': True}) + await send({'type': 'http.response.body', 'body': b'', 'more_body': False}) + await SimulationResultCompressionMiddleware(application)(http_scope(), receive, send) + self.assertEqual(bytes(received), progress + result) + self.assertTrue(decoder.eof) + self.assert_complete_gzip(messages, progress + result) + headers = response_headers(messages) + self.assertNotIn(b'content-length', headers) + self.assertEqual(headers[b'cache-control'], b'no-cache, no-transform') + self.assertEqual(headers[b'x-accel-buffering'], b'no') + self.assertIn(b'origin', headers[b'vary'].lower()) + + async def test_large_single_body_is_split_and_preserves_entire_trailer(self): + # Deterministic high-entropy ASCII avoids a trivially tiny compressed + # result hiding an accidental one-shot write of the whole large body. + body = b'{"opaque":"' + hashlib.shake_256(b'asgi-result').hexdigest(600 * 1024).encode() + b'"}' + messages, _ = await self.request(body) + self.assert_complete_gzip(messages, body) + chunks = [m['body'] for m in response_bodies(messages) if m.get('body')] + self.assertGreaterEqual(len(chunks), 4) + # DEFLATE may enlarge incompressible input slightly, so compressed + # messages are allowed framing overhead beyond the 256 KiB input cap. + self.assertLessEqual(max(map(len, chunks)), CHUNK_BYTES + 4096) + self.assertNotIn(b'content-length', response_headers(messages)) + + async def test_gzip_negotiation_and_supported_result_routes(self): + body = b'{"result":"' + b'large-result,' * 100 + b'"}' + for method, path in [('POST', STREAM_PATH), ('POST', '/api/system-xml/simulate'), + ('GET', '/api/system-xml/simulations/task-123')]: + for encoding in ('gzip', 'br, gzip;q=0.5', '*', '*;q=0.5', 'GZip;Q=0.7', + 'gzip;q=0.5, *;q=0', 'gzip;q=0.8, identity;q=0.5', + ['br', 'gzip;q=0.5']): + with self.subTest(method=method, path=path, encoding=encoding): + messages, _ = await self.request(body, method=method, path=path, + accept_encoding=encoding) + self.assert_complete_gzip(messages, body) + + async def test_nonaccepting_clients_and_unrelated_routes_are_unchanged(self): + body = b'{"result":"' + b'x' * 2000 + b'"}' + for encoding in (None, '', 'identity', 'br', 'gzip;q=0', 'gzip;q=0.000', + '*;q=0', 'gzip;q=0, *;q=1', 'GZIP;Q=0', 'notgzip', + 'gzip;q=bogus', 'gzip;q=1.001', 'gzip;q=0.1234', + 'gzip;q=0.8, identity;q=1', ['gzip', 'gzip;q=0']): + with self.subTest(encoding=encoding): + messages, original = await self.request(body, accept_encoding=encoding) + # Identity negotiation still varies by Accept-Encoding for + # shared caches; its status, existing headers and bytes stay exact. + headers = response_headers(messages) + self.assertNotIn(b'content-encoding', headers) + self.assertIn(b'accept-encoding', headers[b'vary'].lower()) + normalized = [copied_message(message) for message in messages] + start = next(message for message in normalized if message['type'] == 'http.response.start') + start['headers'] = [(key, value) for key, value in start['headers'] if key.lower() != b'vary'] + self.assertEqual(normalized, original) + for method, path in [('GET', STREAM_PATH), ('POST', '/api/system-xml/validate'), + ('POST', '/api/system-xml/simulations/task/cancel'), + ('GET', '/api/system-xml/simulations/task/cancel'), + ('GET', '/api/system-xml/simulations'), ('GET', '/')]: + with self.subTest(method=method, path=path): + messages, original = await self.request(body, method=method, path=path) + self.assertEqual(messages, original) + + async def test_small_preencoded_and_partial_responses_are_not_compressed(self): + tiny = b'{"status":"queued"}' + messages, original = await self.request(tiny) + self.assertEqual(messages, original) + body = b'{"data":"' + b'x' * (CHUNK_BYTES + 50) + b'"}' + for status, extra in [(200, [(b'content-encoding', b'br')]), + (200, [(b'content-encoding', b'gzip')]), + (206, [(b'content-range', b'bytes 0-100/2000')])]: + with self.subTest(status=status, extra=extra): + headers = [(b'content-type', b'application/json'), + (b'content-length', str(len(body)).encode()), *extra] + messages, original = await self.request(body, status=status, headers=headers) + self.assertEqual(messages, original) + + async def test_error_and_cancelled_results_preserve_status_and_numeric_text(self): + for code, status in [(422, 'failed'), (200, 'stopped'), (200, 'stalled')]: + with self.subTest(code=code, status=status): + body = ('{"status":"' + status + '","partial":true,"series":' + '{"温度":[-0,5e-324,-0.0,1.7976931348623157e308]},' + '"message":"' + '保留已接受的部分结果。' * 80 + '"}').encode() + messages, _ = await self.request(body, status=code) + self.assertEqual(next(m['status'] for m in messages if m['type'] == 'http.response.start'), code) + self.assert_complete_gzip(messages, body) + + async def test_application_failures_and_cancellation_are_not_swallowed(self): + progress = b'{"event":"progress","progress":20}\n' + for exception in (RuntimeError('upstream failed'), asyncio.CancelledError()): + with self.subTest(exception=type(exception).__name__): + messages = [] + async def send(message): + messages.append(copied_message(message)) + async def application(scope, incoming, send): + await send({'type': 'http.response.start', 'status': 200, + 'headers': [(b'content-type', b'application/x-ndjson')]}) + await send({'type': 'http.response.body', 'body': progress, 'more_body': True}) + raise exception + with self.assertRaises(type(exception)) as caught: + await SimulationResultCompressionMiddleware(application)(http_scope(), receive, send) + self.assertIs(caught.exception, exception) + decoder = zlib.decompressobj(zlib.MAX_WBITS + 16) + partial = b''.join(m.get('body', b'') for m in response_bodies(messages)) + self.assertEqual(decoder.decompress(partial), progress) + self.assertFalse(decoder.eof, 'A failed stream must not masquerade as normally completed') + + async def test_downstream_disconnect_propagates_and_unwinds_application(self): + unwound = False + failure = BrokenPipeError('client disconnected') + async def application(scope, incoming, send): + nonlocal unwound + try: + await send({'type': 'http.response.start', 'status': 200, + 'headers': [(b'content-type', b'application/x-ndjson')]}) + await send({'type': 'http.response.body', 'body': b'{"event":"progress"}\n', + 'more_body': True}) + self.fail('The application must observe the failed downstream send') + finally: + unwound = True + async def send(message): + if message['type'] == 'http.response.body': + raise failure + with self.assertRaises(BrokenPipeError) as caught: + await SimulationResultCompressionMiddleware(application)(http_scope(), receive, send) + self.assertIs(caught.exception, failure) + self.assertTrue(unwound) + + +if __name__ == '__main__': + unittest.main() --- /dev/null +++ b/tests/manual/result_transfer_profile.py @@ -0,0 +1,250 @@ +"""Serve a real app with optional throttling AFTER its content compression. + + .venv/bin/python tests/manual/result_transfer_profile.py \ + --source-root . --frontend-dist frontend/dist --port 8012 \ + --output-dir test/result-transfer/new --download-bytes-per-second 1600000 + +Only /api/system-xml/simulate-stream response bodies are split into <=64 KiB +chunks. Each encoded chunk is delayed before the original ASGI send using +max(now, previous_deadline) + chunk_bytes/rate; solver idle time earns no credit. +Rate 0 preserves the same chunking without deliberate delays. No solver hooks, +request-body copies, response parsing, browser launch, or compilation occurs. +""" +from __future__ import annotations + +import argparse +import asyncio +from hashlib import sha256 +import importlib +import json +import logging +import math +import os +from pathlib import Path +import re +import shutil +import sys +import time +from uuid import uuid4 + +ROOT = Path(__file__).resolve().parents[2] +STREAM_PATH = "/api/system-xml/simulate-stream" +CHUNK_BYTES = 64 * 1024 + + +def write_json(path: Path, value: dict) -> None: + path.write_text(json.dumps(value, ensure_ascii=False, indent=2, allow_nan=False) + "\n", encoding="utf-8") + + +def digest(path: Path) -> str: + return sha256(path.read_bytes()).hexdigest() + + +class TransferProfile: + """An outer ASGI wrapper; outgoing body bytes are already content-encoded.""" + + def __init__(self, app, output: Path, rate: int, *, clock=time.perf_counter, sleep=asyncio.sleep): + if type(rate) is not int or rate < 0: + raise ValueError("download bytes per second must be a nonnegative integer") + self.app = app + self.output = output + self.rate = rate + self.clock = clock + self.sleep = sleep + + async def __call__(self, scope, receive, send): + if scope["type"] != "http" or scope.get("path") != STREAM_PATH: + return await self.app(scope, receive, send) + request_headers = dict(scope.get("headers", [])) + sid = request_headers.get(b"x-simulation-id", b"").decode("latin-1") + request_id = sid if re.fullmatch(r"[A-Za-z0-9_-]{1,128}", sid) else uuid4().hex + directory = self.output / "requests" / request_id + try: + directory.mkdir(parents=True, exist_ok=False) + except FileExistsError: + request_id += "-" + uuid4().hex + directory = self.output / "requests" / request_id + directory.mkdir(parents=True, exist_ok=False) + started = self.clock() + deadline = started + record = { + "schemaVersion": 1, "requestId": request_id, "simulationId": sid or None, + "path": scope["path"], "method": scope.get("method"), "httpVersion": scope.get("http_version"), + "asgiSpecVersion": scope.get("asgi", {}).get("spec_version"), + "downloadBytesPerSecond": self.rate, "maxChunkBytes": CHUNK_BYTES, + "requestAcceptEncoding": request_headers.get(b"accept-encoding", b"").decode("latin-1"), + "status": "incomplete", "responseHeaders": [], "httpStatus": None, + "encodedBodyBytes": 0, "attemptedEncodedBodyBytes": 0, + "sourceBodyMessages": 0, "chunkCount": 0, "nonEmptyChunkCount": 0, + "maxObservedChunkBytes": 0, "throttleSleepSeconds": 0.0, "sendAwaitSeconds": 0.0, + "responseHeadersSeconds": None, "firstBodyAvailableSeconds": None, + "firstNonemptyBodyAvailableSeconds": None, "firstBodySendStartSeconds": None, + "firstBodySendEndSeconds": None, "lastBodySendStartSeconds": None, + "lastBodySendEndSeconds": None, "responseBodyCompleteSeconds": None, + "disconnectObserved": False, "disconnectBeforeBodyComplete": False, + "disconnectObservedSeconds": None, "sendDisconnectException": None, + "applicationReturned": False, "exception": None, + } + body_complete = False + active_exception = None + + def elapsed() -> float: + return self.clock() - started + + async def observed_receive(): + message = await receive() + if message["type"] == "http.disconnect": + record["disconnectObserved"] = True + if record["disconnectObservedSeconds"] is None: + record["disconnectObservedSeconds"] = elapsed() + # Uvicorn may report http.disconnect after a normally finished response. + if not body_complete: + record["disconnectBeforeBodyComplete"] = True + return message + + async def send_chunk(message, chunk: bytes, more_body: bool): + nonlocal deadline, body_complete + length = len(chunk) + if self.rate and length: + now = self.clock() + deadline = max(now, deadline) + length / self.rate + pause_start = self.clock() + try: + await self.sleep(max(0.0, deadline - pause_start)) + finally: + record["throttleSleepSeconds"] += self.clock() - pause_start + start = elapsed() + if length and record["firstBodySendStartSeconds"] is None: + record["firstBodySendStartSeconds"] = start + record["lastBodySendStartSeconds"] = start + record["attemptedEncodedBodyBytes"] += length + try: + await send({**message, "body": chunk, "more_body": more_body}) + except OSError as exc: + record["sendDisconnectException"] = {"type": type(exc).__name__, "message": str(exc)[:1000]} + record["disconnectBeforeBodyComplete"] = not body_complete + raise + finally: + record["sendAwaitSeconds"] += elapsed() - start + end = elapsed() + record["encodedBodyBytes"] += length + record["chunkCount"] += 1 + record["nonEmptyChunkCount"] += int(length > 0) + record["maxObservedChunkBytes"] = max(record["maxObservedChunkBytes"], length) + if length and record["firstBodySendEndSeconds"] is None: + record["firstBodySendEndSeconds"] = end + record["lastBodySendEndSeconds"] = end + if not more_body: + body_complete = True + record["responseBodyCompleteSeconds"] = end + + async def observed_send(message): + if message["type"] == "http.response.start": + record["responseHeadersSeconds"] = elapsed() + record["httpStatus"] = message["status"] + record["responseHeaders"] = [[key.decode("latin-1"), value.decode("latin-1")] + for key, value in message.get("headers", [])] + headers = dict(message.get("headers", [])) + record["contentEncoding"] = headers.get(b"content-encoding", b"").decode("latin-1") or None + record["contentType"] = headers.get(b"content-type", b"").decode("latin-1") or None + return await send(message) + if message["type"] != "http.response.body": + return await send(message) + body = message.get("body", b"") + record["sourceBodyMessages"] += 1 + if record["firstBodyAvailableSeconds"] is None: + record["firstBodyAvailableSeconds"] = elapsed() + if body and record["firstNonemptyBodyAvailableSeconds"] is None: + record["firstNonemptyBodyAvailableSeconds"] = elapsed() + more_body = message.get("more_body", False) + if not body: + await send_chunk(message, b"", more_body) + return + for offset in range(0, len(body), CHUNK_BYTES): + end = min(offset + CHUNK_BYTES, len(body)) + await send_chunk(message, body[offset:end], more_body or end < len(body)) + + try: + await self.app(scope, observed_receive, observed_send) + record["applicationReturned"] = True + except BaseException as exc: + active_exception = exc + record["exception"] = {"type": type(exc).__name__, "message": str(exc)[:1000]} + raise # Preserve disconnect, cancellation and application failures. + finally: + record["responseSeconds"] = elapsed() + record["responseBodyComplete"] = body_complete + if record["disconnectBeforeBodyComplete"] or record["sendDisconnectException"]: + record["status"] = "disconnected" + elif active_exception is not None: + record["status"] = "cancelled" if isinstance(active_exception, asyncio.CancelledError) else "failed" + elif body_complete: + record["status"] = "completed" + try: + write_json(directory / "transfer.json", record) + except OSError: + if active_exception is None: + raise + logging.exception("Could not save transfer metadata while propagating the original exception") + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--source-root", type=Path, default=ROOT, help="Repository or frozen source snapshot containing app/main.py") + parser.add_argument("--frontend-dist", type=Path, help="Default: SOURCE_ROOT/frontend/dist; copied to the fresh output directory") + parser.add_argument("--port", type=int, default=8012) + parser.add_argument("--output-dir", type=Path, required=True) + parser.add_argument("--download-bytes-per-second", type=int, default=0, help="Encoded HTTP body bytes/s; 0 disables deliberate delays") + args = parser.parse_args() + source, output = args.source_root.resolve(), args.output_dir.resolve() + frontend = (args.frontend_dist or source / "frontend/dist").resolve() + if not (source / "app/main.py").is_file(): + parser.error("--source-root must contain app/main.py") + if not (frontend / "index.html").is_file(): + parser.error("--frontend-dist must contain index.html") + if output.exists() or output.is_relative_to(frontend): + parser.error("Choose a fresh output directory outside --frontend-dist") + if not 1 <= args.port <= 65535 or args.download_bytes_per_second < 0: + parser.error("Require port 1..65535 and download bytes per second >= 0") + if not math.isfinite(float(args.download_bytes_per_second)): + parser.error("Download rate must be finite") + output.mkdir(parents=True) + shutil.copytree(frontend, output / "frontend") + # These timing-only inherited switches must not accidentally instrument a run. + for name in ("SIMULATIONAPP_PROFILE", "NATIVE_COMPUTE_PROFILE", "NATIVE_STAGE_PROFILE"): + os.environ.pop(name, None) + sys.path.insert(0, str(source)) + api = importlib.import_module("app.main") + if Path(api.__file__).resolve() != source / "app/main.py": + raise RuntimeError("Imported app.main does not belong to the requested source root") + api.FRONTEND_DIST_DIR = output / "frontend" + import uvicorn + + metadata = { + "schemaVersion": 1, "mode": "encoded-body-rate-limit", "sourceRoot": str(source), + "loadedApp": str(Path(api.__file__).resolve()), "appMainSha256": digest(source / "app/main.py"), + "scriptSha256": digest(Path(__file__)), "frontendDist": str(frontend), + "frontendFiles": {str(p.relative_to(output / "frontend")): digest(p) + for p in sorted((output / "frontend").rglob("*")) if p.is_file()}, + "python": sys.version, "platform": sys.platform, "uvicorn": uvicorn.__version__, + "host": "127.0.0.1", "port": args.port, + "downloadBytesPerSecond": args.download_bytes_per_second, "maxChunkBytes": CHUNK_BYTES, + "limitedPath": STREAM_PATH, "solverInstrumented": False, + "definitions": { + "rate": "Each nonempty encoded chunk is delayed before ASGI send with deadline=max(now,last_deadline)+chunk_bytes/rate; no accumulated credit during computation or socket backpressure. 0 means no deliberate delay, with the same <=64KiB chunking.", + "encodedBodyBytes": "Body bytes after application content compression whose original ASGI send returned successfully. Includes progress/result/newline bytes; excludes HTTP framing, response headers, TCP/TLS and port-forward overhead. Not proof of client receipt.", + "chunkCount": "Successfully sent ASGI body messages, including empty messages. nonEmptyChunkCount excludes them.", + "bodyTimes": "Seconds since this request entered the outer middleware. firstBodySend* refers to first nonempty chunk; lastBodySend* includes an empty terminal body. responseSeconds ends when the application returns/raises, before metadata file I/O.", + "completion": "completed means the final more_body=false ASGI send returned and the app returned normally. It does not mean the browser parsed or rendered the result.", + "disconnect": "Original receive messages and raised exceptions are preserved. Disconnect observed after a completed body is separately recorded but does not relabel a normal response. Before-completion disconnect, cancellation and failure are distinct statuses.", + "scope": "Only the simulation streaming route is throttled. This harness models encoded HTTP body throughput, not latency, packet loss or an actual remote tunnel. Current/frozen applications retain their own compression policy.", + }, + } + write_json(output / "environment.json", metadata) + print(json.dumps({"url": f"http://127.0.0.1:{args.port}", "output": str(output), + "downloadBytesPerSecond": args.download_bytes_per_second, "sourceRoot": str(source)}), flush=True) + uvicorn.run(TransferProfile(api.app, output, args.download_bytes_per_second), host="127.0.0.1", port=args.port) + + +if __name__ == "__main__": + main()