整合求解器活动监控与步长回归证据

同步远端 PNL0003 诊断和大采样网格能力,语义合并活动感知的 60 秒真停滞判定与旧后端 15 分钟兼容兜底。

纳管热路径优化、15 单元运行证据、浏览器与 API 报告,并补充北京时间更新日志和遗留问题。
This commit is contained in:
lujingze committed 2026-08-19 16:24:31 +00:00
1 parent c19cf77aee
commit e18399c022
46 files changed
+181589 -170

No files matched your search

+59
View File
@@ -0,0 +1,59 @@
import { defineConfig } from "@playwright/test";
import { fileURLToPath } from "node:url";
const channel = process.env.PLAYWRIGHT_CHANNEL?.trim();
const executablePath = process.env.PLAYWRIGHT_EXECUTABLE_PATH?.trim();
const frontendRoot = fileURLToPath(new URL(".", import.meta.url));
const repositoryRoot = fileURLToPath(new URL("..", import.meta.url));
const simulationEnvironment = {
...process.env,
SIMULATION_CAUSAL_EXECUTOR_V2: "1",
SIMULATION_CAUSAL_COORDINATE_KERNEL: "1",
SIMULATION_CAUSAL_DIRECT_SUM_ASSIGNMENTS: "1",
SIMULATION_CAUSAL_DIRECT_EQUATION_READERS: "1",
SIMULATION_CAUSAL_FAST_PATH: "1",
SIMULATION_MECHANICAL_ATOL_MODE: "legacy",
SIMULATION_ODE_JACOBIAN_MODE: "scipy",
SIMULATIONAPP_PROPERTY_CACHE: "on",
};
export default defineConfig({
testDir: "./tests/e2e",
fullyParallel: false,
workers: 1,
timeout: 600_000,
expect: {
timeout: 15_000,
},
outputDir: "test-results/live-mql8",
use: {
baseURL: "http://127.0.0.1:14173",
...(channel ? { channel } : {}),
...(executablePath ? { launchOptions: { executablePath } } : {}),
headless: true,
viewport: { width: 1440, height: 900 },
screenshot: "only-on-failure",
trace: "on",
},
webServer: [
{
command:
".venv/bin/python -m uvicorn app.main:app --host 127.0.0.1 --port 18181",
cwd: repositoryRoot,
env: simulationEnvironment,
url: "http://127.0.0.1:18181/api/components/catalog",
reuseExistingServer: false,
timeout: 120_000,
gracefulShutdown: { signal: "SIGTERM", timeout: 10_000 },
},
{
command:
"node --input-type=module -e \"import { createServer } from 'vite'; import react from '@vitejs/plugin-react'; const server = await createServer({ configFile: false, root: process.cwd(), plugins: [react()], server: { host: '127.0.0.1', port: 14173, strictPort: true, proxy: { '/api': 'http://127.0.0.1:18181' } } }); await server.listen();\"",
cwd: frontendRoot,
url: "http://127.0.0.1:14173",
reuseExistingServer: false,
timeout: 120_000,
gracefulShutdown: { signal: "SIGTERM", timeout: 10_000 },
},
],
});
+223 -22
View File
@@ -109,8 +109,12 @@ import {
lmechn1RightPortCount,
} from "./componentSymbols/mechanical";
import {
SIMULATION_ACTIVITY_STALL_TIMEOUT_MS,
SIMULATION_ACTIVITY_STALL_TIMEOUT_SECONDS,
SIMULATION_SOLVER_STALL_TIMEOUT_MINUTES,
SIMULATION_SOLVER_STALL_TIMEOUT_MS,
solverActivityStallTimeoutMessage,
solverStallTimeoutMessage,
solverStallTimeoutReached,
} from "./simulationTimeout";
import {
CONTACT_AWARE_EDGE_TYPE,
@@ -1053,6 +1057,37 @@ type SimulationStreamIssue = {
message?: string;
};
type SimulationActivityTelemetry = {
activitySequence?: number;
activityKind?: string;
currentTrialTime?: number | null;
rhsCallCount?: number;
acceptedStepSequence?: number;
acceptedTime?: number | null;
};
export type SimulationActivityWatchdogStatus =
| "progressing"
| "slow-active"
| "slow-telemetry-unavailable"
| "stalled";
export type SimulationActivityWatchdogState = {
lastAcceptedProgressAt: number;
lastActivityChangeAt: number | null;
lastActivitySequence: number | null;
lastAcceptedStepSequence: number | null;
lastAcceptedTime: number | null;
integratingPhaseActive: boolean;
};
export type SimulationActivityWatchdogObservation = {
status: SimulationActivityWatchdogStatus;
acceptedProgressIdleMs: number;
activityIdleMs: number | null;
activityTelemetryAvailable: boolean;
};
type SimulationCancelReason = "user" | "stalled";
type SimulationTaskResponse = {
@@ -1064,7 +1099,7 @@ type SimulationTaskResponse = {
};
type SimulationStreamEvent =
| {
| ({
event: "progress";
progress: number;
phase: string;
@@ -1072,7 +1107,7 @@ type SimulationStreamEvent =
heartbeat?: boolean;
simulatedTime?: number;
totalTime?: number;
}
} & SimulationActivityTelemetry)
| {
event: "result";
progress: number;
@@ -1092,6 +1127,137 @@ type SimulationStreamEvent =
simulatedTime?: number;
totalTime?: number;
};
export function createSimulationActivityWatchdog(
now = Date.now(),
): SimulationActivityWatchdogState {
return {
lastAcceptedProgressAt: now,
lastActivityChangeAt: null,
lastActivitySequence: null,
lastAcceptedStepSequence: null,
lastAcceptedTime: null,
integratingPhaseActive: false,
};
}
export function isSimulationSolverActivityPhase(phase: string): boolean {
return phase === "integrating";
}
export function observeSimulationActivityWatchdog(
current: SimulationActivityWatchdogState,
event: SimulationActivityTelemetry & { heartbeat?: boolean; phase?: string },
now = Date.now(),
stallTimeoutMs = SIMULATION_ACTIVITY_STALL_TIMEOUT_MS,
legacyStallTimeoutMs = SIMULATION_SOLVER_STALL_TIMEOUT_MS,
): {
state: SimulationActivityWatchdogState;
observation: SimulationActivityWatchdogObservation;
} {
const reportsPhase = typeof event.phase === "string";
if (reportsPhase && !isSimulationSolverActivityPhase(event.phase!)) {
return {
state: createSimulationActivityWatchdog(now),
observation: {
status: "progressing",
acceptedProgressIdleMs: 0,
activityIdleMs: null,
activityTelemetryAvailable: false,
},
};
}
const baseline =
reportsPhase && !current.integratingPhaseActive
? {
...createSimulationActivityWatchdog(now),
integratingPhaseActive: true,
}
: current;
let acceptedProgress = event.heartbeat !== true;
let lastAcceptedStepSequence = baseline.lastAcceptedStepSequence;
let lastAcceptedTime = baseline.lastAcceptedTime;
if (
typeof event.acceptedStepSequence === "number" &&
Number.isFinite(event.acceptedStepSequence)
) {
if (
lastAcceptedStepSequence !== null &&
event.acceptedStepSequence !== lastAcceptedStepSequence
) {
acceptedProgress = true;
}
lastAcceptedStepSequence = event.acceptedStepSequence;
}
if (typeof event.acceptedTime === "number" && Number.isFinite(event.acceptedTime)) {
if (lastAcceptedTime !== null && event.acceptedTime !== lastAcceptedTime) {
acceptedProgress = true;
}
lastAcceptedTime = event.acceptedTime;
}
let lastActivityChangeAt = baseline.lastActivityChangeAt;
let lastActivitySequence = baseline.lastActivitySequence;
const reportsActivitySequence =
typeof event.activitySequence === "number" &&
Number.isFinite(event.activitySequence);
if (reportsActivitySequence) {
if (
lastActivitySequence === null ||
event.activitySequence !== lastActivitySequence
) {
lastActivityChangeAt = now;
}
lastActivitySequence = event.activitySequence!;
}
const lastAcceptedProgressAt = acceptedProgress
? now
: baseline.lastAcceptedProgressAt;
const acceptedProgressIdleMs = Math.max(0, now - lastAcceptedProgressAt);
const activityIdleMs =
lastActivityChangeAt === null
? null
: Math.max(0, now - lastActivityChangeAt);
const effectiveStallTimeoutMs = Math.max(0, stallTimeoutMs);
const effectiveLegacyStallTimeoutMs = Math.max(
effectiveStallTimeoutMs,
legacyStallTimeoutMs,
);
let status: SimulationActivityWatchdogStatus = "progressing";
if (acceptedProgressIdleMs >= effectiveStallTimeoutMs) {
if (!reportsActivitySequence || activityIdleMs === null) {
status =
acceptedProgressIdleMs >= effectiveLegacyStallTimeoutMs
? "stalled"
: "slow-telemetry-unavailable";
} else if (activityIdleMs >= effectiveStallTimeoutMs) {
status = "stalled";
} else {
status = "slow-active";
}
}
return {
state: {
lastAcceptedProgressAt,
lastActivityChangeAt,
lastActivitySequence,
lastAcceptedStepSequence,
lastAcceptedTime,
integratingPhaseActive:
reportsPhase || baseline.integratingPhaseActive,
},
observation: {
status,
acceptedProgressIdleMs,
activityIdleMs,
activityTelemetryAvailable: reportsActivitySequence,
},
};
}
const SUPPORTED_SOLVER_METHODS = ["BDF", "Radau", "LSODA", "RK45", "RK23", "DOP853"];
const simulationConfigLabels: Record<SimulationNumericKey, string> = {
t_start: "起始时间",
@@ -7355,10 +7521,11 @@ function FlowWorkbench() {
setSimulationStopPending(false);
let lastLoggedPhase = "";
let lastLoggedBucket = -1;
let lastWatchdogStatus: SimulationActivityWatchdogStatus = "progressing";
try {
const result = await streamSystemSimulation(
xml,
(event) => {
(event, watchdog) => {
if (event.event !== "progress") {
return;
}
@@ -7373,14 +7540,38 @@ function FlowWorkbench() {
"initialization",
"initializing",
].includes(event.phase);
const progressMessage =
watchdog?.status === "slow-active"
? `正在处理耗时积分步,内部计算仍活跃${
event.activityKind ? `(${event.activityKind})` : ""
},可手动停止`
: watchdog?.status === "slow-telemetry-unavailable"
? "积分步耗时较长,后端未提供活动遥测;任务保持运行,可手动停止"
: event.message;
setSimulationProgress((current) => ({
...current,
state: "running",
percent: normalizedProgress.percent,
message: event.message,
message: progressMessage,
simulatedTime: normalizedProgress.simulatedTime,
totalTime: normalizedProgress.totalTime,
}));
if (watchdog && watchdog.status !== lastWatchdogStatus) {
if (watchdog.status === "slow-active") {
appendConsoleEntry(
"warning",
`积分步超过 ${SIMULATION_ACTIVITY_STALL_TIMEOUT_SECONDS} 秒尚未接受,但后端内部活动仍在推进;继续等待,可随时手动停止`,
);
} else if (watchdog.status === "slow-telemetry-unavailable") {
appendConsoleEntry(
"warning",
`积分步超过 ${SIMULATION_ACTIVITY_STALL_TIMEOUT_SECONDS} 秒尚未接受,且后端未提供活动遥测;将按 ${SIMULATION_SOLVER_STALL_TIMEOUT_MINUTES} 分钟保守上限继续等待,可手动停止`,
);
} else if (lastWatchdogStatus !== "progressing") {
appendConsoleEntry("info", "求解器已接受新的积分步,仿真继续推进");
}
lastWatchdogStatus = watchdog.status;
}
const progressBucket = Math.floor(normalizedProgress.percent / 10);
if (event.phase !== lastLoggedPhase || progressBucket !== lastLoggedBucket) {
const progressDetail = reportsSimulationTime
@@ -7404,7 +7595,9 @@ function FlowWorkbench() {
appendConsoleEntry(
"error",
error.code === "SOLVER_STALLED"
? solverStallTimeoutMessage("recovering")
? error.message.includes("内部活动")
? solverActivityStallTimeoutMessage("recovering")
: solverStallTimeoutMessage("recovering")
: "超过 30 秒未收到后端数据,正在终止任务并恢复部分结果",
);
try {
@@ -10882,7 +11075,10 @@ async function readSimulationStreamChunk(
async function streamSystemSimulation(
xml: string,
onEvent: (event: SimulationStreamEvent) => void,
onEvent: (
event: SimulationStreamEvent,
watchdog?: SimulationActivityWatchdogObservation,
) => void,
simulationId: string,
): Promise<SimulationResult> {
const abortController = new AbortController();
@@ -10920,7 +11116,7 @@ async function streamSystemSimulation(
const decoder = new TextDecoder();
let buffer = "";
let result: SimulationResult | null = null;
let lastSolverProgressAt = Date.now();
let activityWatchdog = createSimulationActivityWatchdog();
const consumeLine = (line: string) => {
const trimmed = line.trim();
@@ -10936,23 +11132,28 @@ async function streamSystemSimulation(
throw new SimulationStreamError("仿真服务返回了无法识别的进度消息");
}
const event = parsed as SimulationStreamEvent;
let watchdogObservation: SimulationActivityWatchdogObservation | undefined;
if (event.event === "progress") {
if (
event.heartbeat === true &&
solverStallTimeoutReached(lastSolverProgressAt)
) {
throw new SimulationStreamError(
solverStallTimeoutMessage("detected"),
[],
undefined,
"SOLVER_STALLED",
);
}
if (event.heartbeat !== true) {
lastSolverProgressAt = Date.now();
const watched = observeSimulationActivityWatchdog(
activityWatchdog,
event,
);
activityWatchdog = watched.state;
if (isSimulationSolverActivityPhase(event.phase)) {
watchdogObservation = watched.observation;
if (watchdogObservation.status === "stalled") {
throw new SimulationStreamError(
watchdogObservation.activityTelemetryAvailable
? solverActivityStallTimeoutMessage("detected")
: solverStallTimeoutMessage("detected"),
[],
undefined,
"SOLVER_STALLED",
);
}
}
}
onEvent(event);
onEvent(event, watchdogObservation);
if (event.event === "result") {
result = event.result;
return;
+12
View File
@@ -1,6 +1,9 @@
export const SIMULATION_SOLVER_STALL_TIMEOUT_MINUTES = 15;
export const SIMULATION_SOLVER_STALL_TIMEOUT_MS =
SIMULATION_SOLVER_STALL_TIMEOUT_MINUTES * 60_000;
export const SIMULATION_ACTIVITY_STALL_TIMEOUT_SECONDS = 60;
export const SIMULATION_ACTIVITY_STALL_TIMEOUT_MS =
SIMULATION_ACTIVITY_STALL_TIMEOUT_SECONDS * 1_000;
export function solverStallTimeoutReached(
lastSolverProgressAt: number,
@@ -17,3 +20,12 @@ export function solverStallTimeoutMessage(
? `${prefix},正在终止任务并恢复部分结果`
: `${prefix},任务可能已经卡死`;
}
export function solverActivityStallTimeoutMessage(
context: "detected" | "recovering",
) {
const prefix = `求解器的接受步和内部活动均连续 ${SIMULATION_ACTIVITY_STALL_TIMEOUT_SECONDS} 秒没有变化`;
return context === "recovering"
? `${prefix},正在终止任务并恢复部分结果`
: `${prefix},任务可能已经卡死`;
}
@@ -0,0 +1,252 @@
import { expect, test } from "@playwright/test";
import {
createSimulationActivityWatchdog,
isSimulationSolverActivityPhase,
observeSimulationActivityWatchdog,
} from "../../src/App";
const TEST_STALL_TIMEOUT_MS = 1_000;
test("首个含 activitySequence 的慢步心跳只建立活动基线", () => {
const state = createSimulationActivityWatchdog(0);
const watched = observeSimulationActivityWatchdog(
state,
{ heartbeat: true, activitySequence: 42 },
TEST_STALL_TIMEOUT_MS,
TEST_STALL_TIMEOUT_MS,
);
expect(watched.observation.status).toBe("slow-active");
expect(watched.observation.acceptedProgressIdleMs).toBe(TEST_STALL_TIMEOUT_MS);
expect(watched.observation.activityIdleMs).toBe(0);
});
test("accepted time 停滞但 activitySequence 持续增长时保持慢步运行", () => {
let state = createSimulationActivityWatchdog(0);
({ state } = observeSimulationActivityWatchdog(
state,
{
heartbeat: true,
activitySequence: 1,
acceptedStepSequence: 10,
acceptedTime: 0.048,
},
100,
TEST_STALL_TIMEOUT_MS,
));
let watched = observeSimulationActivityWatchdog(
state,
{
heartbeat: true,
activitySequence: 2,
acceptedStepSequence: 10,
acceptedTime: 0.048,
},
1_000,
TEST_STALL_TIMEOUT_MS,
);
state = watched.state;
expect(watched.observation.status).toBe("slow-active");
watched = observeSimulationActivityWatchdog(
state,
{
heartbeat: true,
activitySequence: 3,
acceptedStepSequence: 10,
acceptedTime: 0.048,
},
2_500,
TEST_STALL_TIMEOUT_MS,
);
expect(watched.observation.status).toBe("slow-active");
expect(watched.observation.acceptedProgressIdleMs).toBe(2_500);
expect(watched.observation.activityIdleMs).toBe(0);
});
test("旧后端缺少活动字段时只报告遥测不可用且不误判 stalled", () => {
const state = createSimulationActivityWatchdog(0);
const watched = observeSimulationActivityWatchdog(
state,
{ heartbeat: true },
20_000,
TEST_STALL_TIMEOUT_MS,
);
expect(watched.observation.status).toBe("slow-telemetry-unavailable");
expect(watched.observation.activityIdleMs).toBeNull();
expect(watched.observation.activityTelemetryAvailable).toBe(false);
});
test("旧后端缺少活动字段时使用 15 分钟兼容兜底", () => {
const state = createSimulationActivityWatchdog(0);
const watched = observeSimulationActivityWatchdog(
state,
{ heartbeat: true },
900_000,
TEST_STALL_TIMEOUT_MS,
);
expect(watched.observation.status).toBe("stalled");
expect(watched.observation.activityTelemetryAvailable).toBe(false);
});
test("接受步和明确可用的活动序号都长期不变时才报告 stalled", () => {
let state = createSimulationActivityWatchdog(0);
({ state } = observeSimulationActivityWatchdog(
state,
{ heartbeat: true, activitySequence: 7 },
0,
TEST_STALL_TIMEOUT_MS,
));
const beforeDeadline = observeSimulationActivityWatchdog(
state,
{ heartbeat: true, activitySequence: 7 },
999,
TEST_STALL_TIMEOUT_MS,
);
expect(beforeDeadline.observation.status).toBe("progressing");
const atDeadline = observeSimulationActivityWatchdog(
beforeDeadline.state,
{ heartbeat: true, activitySequence: 7 },
1_000,
TEST_STALL_TIMEOUT_MS,
);
expect(atDeadline.observation.status).toBe("stalled");
});
test("heartbeat 报告新的 acceptedStepSequence 时重置接受步计时", () => {
let state = createSimulationActivityWatchdog(0);
({ state } = observeSimulationActivityWatchdog(
state,
{
heartbeat: true,
activitySequence: 1,
acceptedStepSequence: 10,
acceptedTime: 0.048,
},
0,
TEST_STALL_TIMEOUT_MS,
));
const watched = observeSimulationActivityWatchdog(
state,
{
heartbeat: true,
activitySequence: 2,
acceptedStepSequence: 11,
acceptedTime: 0.049,
},
5_000,
TEST_STALL_TIMEOUT_MS,
);
expect(watched.observation.status).toBe("progressing");
expect(watched.observation.acceptedProgressIdleMs).toBe(0);
expect(watched.state.lastAcceptedProgressAt).toBe(5_000);
});
test("validation 和 compilation 阶段不参与求解器停滞判定", () => {
let state = createSimulationActivityWatchdog(0);
let watched = observeSimulationActivityWatchdog(
state,
{ phase: "validation", heartbeat: true },
90_000,
TEST_STALL_TIMEOUT_MS,
);
state = watched.state;
expect(watched.observation.status).toBe("progressing");
expect(watched.observation.acceptedProgressIdleMs).toBe(0);
expect(watched.state.integratingPhaseActive).toBe(false);
watched = observeSimulationActivityWatchdog(
state,
{ phase: "compilation", heartbeat: true, activitySequence: 7 },
180_000,
TEST_STALL_TIMEOUT_MS,
);
expect(watched.observation.status).toBe("progressing");
expect(watched.observation.activityIdleMs).toBeNull();
expect(isSimulationSolverActivityPhase("compilation")).toBe(false);
});
test("进入 integrating 时重新建立计时基线", () => {
let state = createSimulationActivityWatchdog(0);
({ state } = observeSimulationActivityWatchdog(
state,
{ phase: "initialization", heartbeat: true },
120_000,
TEST_STALL_TIMEOUT_MS,
));
let watched = observeSimulationActivityWatchdog(
state,
{
phase: "integrating",
heartbeat: true,
activitySequence: 11,
acceptedStepSequence: 3,
acceptedTime: 0.048,
},
180_000,
TEST_STALL_TIMEOUT_MS,
);
state = watched.state;
expect(watched.observation.status).toBe("progressing");
expect(watched.observation.acceptedProgressIdleMs).toBe(0);
expect(watched.observation.activityIdleMs).toBe(0);
expect(watched.state.integratingPhaseActive).toBe(true);
expect(isSimulationSolverActivityPhase("integrating")).toBe(true);
watched = observeSimulationActivityWatchdog(
state,
{
phase: "integrating",
heartbeat: true,
activitySequence: 11,
acceptedStepSequence: 3,
acceptedTime: 0.048,
},
181_000,
TEST_STALL_TIMEOUT_MS,
);
expect(watched.observation.status).toBe("stalled");
});
test("离开 integrating 后清除旧积分阶段的停滞状态", () => {
let state = createSimulationActivityWatchdog(0);
({ state } = observeSimulationActivityWatchdog(
state,
{
phase: "integrating",
heartbeat: true,
activitySequence: 5,
},
0,
TEST_STALL_TIMEOUT_MS,
));
({ state } = observeSimulationActivityWatchdog(
state,
{
phase: "integrating",
heartbeat: true,
activitySequence: 5,
},
TEST_STALL_TIMEOUT_MS,
TEST_STALL_TIMEOUT_MS,
));
const watched = observeSimulationActivityWatchdog(
state,
{ phase: "postprocessing", heartbeat: true },
60_000,
TEST_STALL_TIMEOUT_MS,
);
expect(watched.observation.status).toBe("progressing");
expect(watched.observation.acceptedProgressIdleMs).toBe(0);
expect(watched.state.integratingPhaseActive).toBe(false);
expect(watched.state.lastActivitySequence).toBeNull();
});
@@ -0,0 +1,578 @@
import { expect, test, type Page } from "@playwright/test";
import { createHash } from "node:crypto";
import { readFileSync, statSync, writeFileSync } from "node:fs";
import { fileURLToPath } from "node:url";
type CompactProgressEvent = {
receivedAtMs: number;
phase?: string;
heartbeat: boolean;
progress?: number;
simulatedTime?: number;
totalTime?: number;
activitySequence?: number;
activityKind?: string;
currentTrialTime?: number | null;
rhsCallCount?: number;
acceptedStepSequence?: number;
acceptedTime?: number | null;
};
type CompactResult = {
success?: boolean;
status?: string;
partial?: boolean;
simulatedUntil?: number;
requestedStopTime?: number;
sampleCount?: number;
timePointCount?: number;
finalTime?: number;
};
type BrowserStreamEvidence = {
lineCount: number;
progressEvents: CompactProgressEvent[];
result: CompactResult | null;
streamErrors: Array<{
receivedAtMs: number;
status?: number;
message?: string;
detail?: unknown;
}>;
parseErrors: string[];
};
const PROJECT_PATH = fileURLToPath(
new URL("../../../tests/data/test-mql-8.json", import.meta.url),
);
const EVIDENCE_PATH = fileURLToPath(
new URL(
"../../../tests/baselines/simulation/test_mql_8/runs/2026-08-18-production-browser-live-activity-v1-0.2.json",
import.meta.url,
),
);
const EXPECTED_RUNTIME_ENVIRONMENT = {
SIMULATION_CAUSAL_EXECUTOR_V2: "1",
SIMULATION_CAUSAL_COORDINATE_KERNEL: "1",
SIMULATION_CAUSAL_DIRECT_SUM_ASSIGNMENTS: "1",
SIMULATION_CAUSAL_DIRECT_EQUATION_READERS: "1",
SIMULATION_CAUSAL_FAST_PATH: "1",
SIMULATION_MECHANICAL_ATOL_MODE: "legacy",
SIMULATION_ODE_JACOBIAN_MODE: "scipy",
SIMULATIONAPP_PROPERTY_CACHE: "on",
} as const;
function extractSimulationAttribute(xml: string, name: string) {
const match = xml.match(new RegExp(`\\b${name}="([^"]+)"`));
return match?.[1] ?? null;
}
function buildActivitySummary(events: CompactProgressEvent[]) {
const heartbeats = events.filter((event) => event.heartbeat);
const activityHeartbeats = heartbeats.filter(
(event) => typeof event.activitySequence === "number",
);
const activitySequences = activityHeartbeats.map(
(event) => event.activitySequence as number,
);
const acceptedTransitions: CompactProgressEvent[] = [];
let lastAcceptedStepSequence: number | undefined;
for (const event of events) {
if (
event.phase !== "integrating" ||
typeof event.acceptedStepSequence !== "number"
) {
continue;
}
if (event.acceptedStepSequence === lastAcceptedStepSequence) {
continue;
}
lastAcceptedStepSequence = event.acceptedStepSequence;
acceptedTransitions.push(event);
}
let maximumAcceptedPlateau:
| {
durationMs: number;
fromAcceptedStepSequence?: number;
toAcceptedStepSequence?: number;
fromAcceptedTime?: number | null;
toAcceptedTime?: number | null;
heartbeatCount: number;
activitySequenceStart: number | null;
activitySequenceEnd: number | null;
activitySequenceAdvanced: boolean;
}
| undefined;
for (let index = 1; index < acceptedTransitions.length; index += 1) {
const previous = acceptedTransitions[index - 1];
const current = acceptedTransitions[index];
const plateauHeartbeats = activityHeartbeats.filter(
(event) =>
event.receivedAtMs >= previous.receivedAtMs &&
event.receivedAtMs <= current.receivedAtMs,
);
const plateauActivitySequences = plateauHeartbeats.map(
(event) => event.activitySequence as number,
);
const candidate = {
durationMs: current.receivedAtMs - previous.receivedAtMs,
fromAcceptedStepSequence: previous.acceptedStepSequence,
toAcceptedStepSequence: current.acceptedStepSequence,
fromAcceptedTime: previous.acceptedTime,
toAcceptedTime: current.acceptedTime,
heartbeatCount: plateauHeartbeats.length,
activitySequenceStart: plateauActivitySequences.at(0) ?? null,
activitySequenceEnd: plateauActivitySequences.at(-1) ?? null,
activitySequenceAdvanced:
plateauActivitySequences.length >= 2 &&
(plateauActivitySequences.at(-1) as number) >
(plateauActivitySequences.at(0) as number),
};
if (
!maximumAcceptedPlateau ||
candidate.durationMs > maximumAcceptedPlateau.durationMs
) {
maximumAcceptedPlateau = candidate;
}
}
const slowRegionEvents = events.filter((event) => {
const reportedTimes = [
event.acceptedTime,
event.currentTrialTime,
event.simulatedTime,
].filter((value): value is number => typeof value === "number");
return reportedTimes.some((value) => value >= 0.0475 && value <= 0.0505);
});
return {
progressEventCount: events.length,
heartbeatCount: heartbeats.length,
activityTelemetryHeartbeatCount: activityHeartbeats.length,
firstActivitySequence: activitySequences.at(0) ?? null,
lastActivitySequence: activitySequences.at(-1) ?? null,
activitySequenceAdvanced:
activitySequences.length >= 2 &&
(activitySequences.at(-1) as number) >
(activitySequences.at(0) as number),
acceptedTransitionCount: acceptedTransitions.length,
maximumAcceptedPlateau: maximumAcceptedPlateau ?? null,
slowRegionEvents,
};
}
async function readBrowserStreamEvidence(page: Page): Promise<BrowserStreamEvidence> {
return page.evaluate(() => {
const state = (
window as typeof window & { __mql8LiveStreamEvidence?: BrowserStreamEvidence }
).__mql8LiveStreamEvidence;
return (
state ?? {
lineCount: 0,
progressEvents: [],
result: null,
streamErrors: [],
parseErrors: ["Browser stream evidence was not initialized."],
}
);
});
}
function emptyBrowserStreamEvidence(): BrowserStreamEvidence {
return {
lineCount: 0,
progressEvents: [],
result: null,
streamErrors: [],
parseErrors: [],
};
}
function retainMoreCompleteEvidence(
retained: BrowserStreamEvidence,
candidate: BrowserStreamEvidence,
) {
if (
candidate.lineCount > retained.lineCount ||
(candidate.lineCount === retained.lineCount &&
candidate.result !== null &&
retained.result === null)
) {
return candidate;
}
return retained;
}
test("test-mql-8 在真实浏览器流式链路完成 0.2 s BDF 仿真", async ({
page,
}, testInfo) => {
const startedAt = Date.now();
const startedAtIso = new Date(startedAt).toISOString();
const cancellationRequests: string[] = [];
const browserConsoleErrors: string[] = [];
const pageErrors: string[] = [];
let simulationXml = "";
let simulationId: string | undefined;
let workflowCompleted = false;
let retainedStreamEvidence = emptyBrowserStreamEvidence();
let evidenceRetentionTimer: ReturnType<typeof setInterval> | undefined;
page.on("request", (request) => {
const url = request.url();
if (url.includes("/api/system-xml/simulate-stream")) {
simulationXml = request.postData() ?? "";
simulationId = request.headers()["x-simulation-id"];
}
if (/\/api\/system-xml\/simulations\/[^/]+\/cancel(?:\?|$)/.test(url)) {
cancellationRequests.push(url);
}
});
page.on("console", (message) => {
if (message.type() === "error") {
browserConsoleErrors.push(message.text());
}
});
page.on("pageerror", (error) => pageErrors.push(error.message));
await page.addInitScript(() => {
window.localStorage.clear();
window.sessionStorage.clear();
window.location.hash = "#/modeling";
const evidence: BrowserStreamEvidence = {
lineCount: 0,
progressEvents: [],
result: null,
streamErrors: [],
parseErrors: [],
};
(
window as typeof window & { __mql8LiveStreamEvidence?: BrowserStreamEvidence }
).__mql8LiveStreamEvidence = evidence;
const nativeFetch = window.fetch.bind(window);
window.fetch = async (...args: Parameters<typeof window.fetch>) => {
const response = await nativeFetch(...args);
const input = args[0];
const requestUrl =
typeof input === "string"
? input
: input instanceof URL
? input.href
: input.url;
if (!requestUrl.includes("/api/system-xml/simulate-stream") || !response.body) {
return response;
}
const decoder = new TextDecoder();
let buffer = "";
const recordLine = (line: string) => {
const trimmed = line.trim();
if (!trimmed) {
return;
}
evidence.lineCount += 1;
try {
const parsed = JSON.parse(trimmed) as Record<string, unknown>;
const receivedAtMs = performance.now();
if (parsed.event === "progress") {
evidence.progressEvents.push({
receivedAtMs,
phase: typeof parsed.phase === "string" ? parsed.phase : undefined,
heartbeat: parsed.heartbeat === true,
progress:
typeof parsed.progress === "number" ? parsed.progress : undefined,
simulatedTime:
typeof parsed.simulatedTime === "number"
? parsed.simulatedTime
: undefined,
totalTime:
typeof parsed.totalTime === "number" ? parsed.totalTime : undefined,
activitySequence:
typeof parsed.activitySequence === "number"
? parsed.activitySequence
: undefined,
activityKind:
typeof parsed.activityKind === "string"
? parsed.activityKind
: undefined,
currentTrialTime:
typeof parsed.currentTrialTime === "number" ||
parsed.currentTrialTime === null
? parsed.currentTrialTime
: undefined,
rhsCallCount:
typeof parsed.rhsCallCount === "number"
? parsed.rhsCallCount
: undefined,
acceptedStepSequence:
typeof parsed.acceptedStepSequence === "number"
? parsed.acceptedStepSequence
: undefined,
acceptedTime:
typeof parsed.acceptedTime === "number" || parsed.acceptedTime === null
? parsed.acceptedTime
: undefined,
});
} else if (parsed.event === "result") {
const result = parsed.result as Record<string, unknown> | undefined;
const diagnostics = result?.diagnostics as
| Record<string, unknown>
| undefined;
const series = result?.series as Record<string, unknown> | undefined;
const times = Array.isArray(series?.time)
? (series.time as unknown[]).filter(
(value): value is number => typeof value === "number",
)
: [];
evidence.result = {
success:
typeof result?.success === "boolean" ? result.success : undefined,
status: typeof result?.status === "string" ? result.status : undefined,
partial:
typeof result?.partial === "boolean" ? result.partial : undefined,
simulatedUntil:
typeof result?.simulatedUntil === "number"
? result.simulatedUntil
: undefined,
requestedStopTime:
typeof result?.requestedStopTime === "number"
? result.requestedStopTime
: undefined,
sampleCount:
typeof diagnostics?.sampleCount === "number"
? diagnostics.sampleCount
: undefined,
timePointCount: times.length,
finalTime: times.at(-1),
};
} else if (parsed.event === "error") {
evidence.streamErrors.push({
receivedAtMs,
status: typeof parsed.status === "number" ? parsed.status : undefined,
message:
typeof parsed.message === "string" ? parsed.message : undefined,
detail: parsed.detail,
});
}
} catch (error) {
evidence.parseErrors.push(
error instanceof Error ? error.message : String(error),
);
}
};
const flushLines = () => {
let newlineIndex = buffer.indexOf("\n");
while (newlineIndex >= 0) {
recordLine(buffer.slice(0, newlineIndex));
buffer = buffer.slice(newlineIndex + 1);
newlineIndex = buffer.indexOf("\n");
}
};
const monitoredBody = response.body.pipeThrough(
new TransformStream<Uint8Array, Uint8Array>({
transform(chunk, controller) {
buffer += decoder.decode(chunk, { stream: true });
flushLines();
controller.enqueue(chunk);
},
flush() {
buffer += decoder.decode();
flushLines();
recordLine(buffer);
buffer = "";
},
}),
);
return new Response(monitoredBody, {
status: response.status,
statusText: response.statusText,
headers: response.headers,
});
};
});
const retainCurrentStreamEvidence = async () => {
const candidate = await readBrowserStreamEvidence(page);
retainedStreamEvidence = retainMoreCompleteEvidence(
retainedStreamEvidence,
candidate,
);
};
evidenceRetentionTimer = setInterval(() => {
void retainCurrentStreamEvidence().catch(() => undefined);
}, 1_000);
try {
await page.goto("/");
await expect(page.locator(".catalog-source.ready")).toBeVisible({
timeout: 30_000,
});
await page.locator('input[type="file"]').setInputFiles(PROJECT_PATH);
const consolePanel = page.getByRole("complementary", {
name: "仿真控制台",
exact: true,
});
const consoleSummary = consolePanel.locator(
".simulation-console-dock-summary",
);
await expect(consoleSummary).toContainText(
"已导入工程:test-mql-8.json",
{ timeout: 30_000 },
);
const stopTimeInput = page.getByLabel("结束时间", { exact: true });
await stopTimeInput.fill("0.2");
await stopTimeInput.press("Enter");
await expect(stopTimeInput).toHaveValue("0.2");
await expect(page.getByLabel("采样步长", { exact: true })).toHaveValue(
"0.01",
);
await expect(
page.getByLabel("最大积分步长", { exact: true }),
).toHaveValue("0.001");
await expect(page.getByLabel("求解器", { exact: true })).toHaveValue("BDF");
await page.getByRole("button", { name: "运行仿真", exact: true }).click();
await expect.poll(() => simulationXml.length, { timeout: 60_000 }).toBeGreaterThan(0);
expect(extractSimulationAttribute(simulationXml, "tStop")).toBe("0.2");
expect(extractSimulationAttribute(simulationXml, "sampleStep")).toBe("0.01");
expect(extractSimulationAttribute(simulationXml, "maxStep")).toBe("0.001");
expect(extractSimulationAttribute(simulationXml, "method")).toBe("BDF");
await expect(consoleSummary).toContainText(
"仿真完成,已生成新的结果,共 21 个采样点",
{ timeout: 480_000 },
);
await expect
.poll(async () => (await readBrowserStreamEvidence(page)).result?.status, {
timeout: 30_000,
})
.toBe("completed");
const streamEvidence = await readBrowserStreamEvidence(page);
expect(streamEvidence.parseErrors).toEqual([]);
expect(streamEvidence.streamErrors).toEqual([]);
expect(streamEvidence.result).toMatchObject({
success: true,
status: "completed",
partial: false,
simulatedUntil: 0.2,
requestedStopTime: 0.2,
sampleCount: 21,
timePointCount: 21,
finalTime: 0.2,
});
const activitySummary = buildActivitySummary(streamEvidence.progressEvents);
expect(activitySummary.heartbeatCount).toBeGreaterThan(0);
expect(activitySummary.activityTelemetryHeartbeatCount).toBeGreaterThan(1);
expect(activitySummary.activitySequenceAdvanced).toBe(true);
expect(cancellationRequests).toEqual([]);
expect(pageErrors).toEqual([]);
await page.getByRole("button", { name: "展开仿真控制台" }).click();
const consoleLog = consolePanel.getByRole("region", { name: "控制台日志" });
await expect(consoleLog).toContainText(
"仿真完成,已生成新的结果,共 21 个采样点",
);
await expect(consoleLog).not.toContainText("仿真失败");
await expect(consoleLog).not.toContainText("仿真因进度异常而终止");
if (
activitySummary.maximumAcceptedPlateau &&
activitySummary.maximumAcceptedPlateau.durationMs >= 60_000
) {
expect(
activitySummary.maximumAcceptedPlateau.activitySequenceAdvanced,
).toBe(true);
await expect(consoleLog).toContainText("后端内部活动仍在推进");
}
workflowCompleted = true;
} finally {
if (evidenceRetentionTimer !== undefined) {
clearInterval(evidenceRetentionTimer);
}
await retainCurrentStreamEvidence().catch(() => undefined);
const streamEvidence = retainedStreamEvidence;
const activitySummary = buildActivitySummary(streamEvidence.progressEvents);
const finishedAt = Date.now();
const projectBytes = readFileSync(PROJECT_PATH);
const evidence = {
schemaVersion: 1,
kind: "live-browser-stream-verification",
generatedAt: new Date(finishedAt).toISOString(),
startedAt: startedAtIso,
wallSeconds: (finishedAt - startedAt) / 1_000,
workflowCompleted,
topology: {
browserOrigin: "http://127.0.0.1:14173",
backendOrigin: "http://127.0.0.1:18181",
servingMode: "isolated-vite-dev-proxy",
},
source: {
path: "tests/data/test-mql-8.json",
bytes: statSync(PROJECT_PATH).size,
sha256: createHash("sha256").update(projectBytes).digest("hex"),
},
target: {
tStart: 0,
tStop: 0.2,
sampleStep: 0.01,
maxStep: 0.001,
method: "BDF",
},
runtimeEnvironment: EXPECTED_RUNTIME_ENVIRONMENT,
request: {
simulationId: simulationId ?? null,
xmlBytes: Buffer.byteLength(simulationXml),
simulationAttributes: {
tStart: extractSimulationAttribute(simulationXml, "tStart"),
tStop: extractSimulationAttribute(simulationXml, "tStop"),
sampleStep: extractSimulationAttribute(simulationXml, "sampleStep"),
maxStep: extractSimulationAttribute(simulationXml, "maxStep"),
method: extractSimulationAttribute(simulationXml, "method"),
},
},
ndjson: {
lineCount: streamEvidence.lineCount,
parseErrors: streamEvidence.parseErrors,
streamErrors: streamEvidence.streamErrors,
result: streamEvidence.result,
activity: activitySummary,
},
browser: {
cancellationRequests,
pageErrors,
consoleErrors: browserConsoleErrors,
},
acceptance: {
completedWith21Samples:
workflowCompleted &&
streamEvidence.result?.status === "completed" &&
streamEvidence.result?.sampleCount === 21 &&
streamEvidence.result?.timePointCount === 21,
noCancellationRequest: cancellationRequests.length === 0,
noStreamError: streamEvidence.streamErrors.length === 0,
noPageError: pageErrors.length === 0,
activityTelemetryAdvanced: activitySummary.activitySequenceAdvanced,
},
};
const serializedEvidence = `${JSON.stringify(evidence, null, 2)}\n`;
writeFileSync(EVIDENCE_PATH, serializedEvidence, "utf8");
await testInfo.attach("mql8-live-browser-evidence", {
body: Buffer.from(serializedEvidence),
contentType: "application/json",
});
const screenshot = await page.screenshot({ fullPage: true }).catch(() => null);
if (screenshot) {
await testInfo.attach("mql8-live-browser-final-page", {
body: screenshot,
contentType: "image/png",
});
}
}
});
@@ -1,8 +1,11 @@
import { expect, test } from "@playwright/test";
import {
SIMULATION_ACTIVITY_STALL_TIMEOUT_MS,
SIMULATION_ACTIVITY_STALL_TIMEOUT_SECONDS,
SIMULATION_SOLVER_STALL_TIMEOUT_MINUTES,
SIMULATION_SOLVER_STALL_TIMEOUT_MS,
solverActivityStallTimeoutMessage,
solverStallTimeoutMessage,
solverStallTimeoutReached,
} from "../../src/simulationTimeout";
@@ -19,3 +22,11 @@ test("solver stall messages use the configured duration", () => {
expect(solverStallTimeoutMessage("recovering")).toContain("连续 15 分钟");
expect(solverStallTimeoutMessage("recovering")).toContain("恢复部分结果");
});
test("activity-aware watchdog keeps its shorter true-stall threshold", () => {
expect(SIMULATION_ACTIVITY_STALL_TIMEOUT_SECONDS).toBe(60);
expect(SIMULATION_ACTIVITY_STALL_TIMEOUT_MS).toBe(60_000);
expect(solverActivityStallTimeoutMessage("detected")).toContain(
"接受步和内部活动均连续 60 秒",
);
});