plugin: welcome-timeout + stall watchdog (bulletproof review: fixes handshake hang + wedged-socket unbounded buffering); smoke 27 checks, e2e 13/13

This commit is contained in:
Raphael Westphal
2026-08-18 15:31:42 +02:00
parent 49840225e2
commit 1a267bf489
3 changed files with 1377 additions and 925 deletions
+258 -45
View File
@@ -20,6 +20,12 @@
* events are additionally kept in a bounded replay buffer and resent
* after welcome.lastSeq on reconnect
* - reconnect with exponential backoff + jitter, capped, forever
* - handshake watchdog: if welcome does not arrive within
* LVMH_WELCOME_TIMEOUT_MS (default 15s) of connecting, the socket is
* abandoned and retried (covers a daemon that accepts TCP then hangs)
* - stall watchdog: undici exposes no ping(), so a daemon that dies
* without a TCP close is detected via bufferedAmount never draining
* (two LVMH_STALL_CHECK_MS ticks over LVMH_STALL_BYTES, default 30s/16MB)
* - errors are logged to ~/.pi/lvmh-agent.log only; stdout stays clean
* - session_shutdown closes the socket and stops all timers immediately
*/
@@ -27,7 +33,10 @@
import * as fs from "node:fs";
import * as os from "node:os";
import * as path from "node:path";
import type { ExtensionAPI, ExtensionContext } from "@earendil-works/pi-coding-agent";
import type {
ExtensionAPI,
ExtensionContext,
} from "@earendil-works/pi-coding-agent";
const PROTOCOL_VERSION: number = 1;
const ENV_URL: string = "LVMH_URL";
@@ -43,6 +52,20 @@ const BACKOFF_JITTER_MS: number = 1000;
const RESULT_PREVIEW_MAX_CHARS: number = 2000;
const LOG_LINE_MAX_CHARS: number = 500;
function envPositiveInt(name: string, fallback: number): number {
const raw: string | undefined = process.env[name];
if (raw === undefined) return fallback;
const n: number = Number.parseInt(raw, 10);
return Number.isFinite(n) && n > 0 ? n : fallback;
}
// Test/debug knobs; production never sets them. Read lazily so harnesses
// can reconfigure per scenario after the module is already imported.
const welcomeTimeoutMs = (): number =>
envPositiveInt("LVMH_WELCOME_TIMEOUT_MS", 15000);
const stallCheckMs = (): number => envPositiveInt("LVMH_STALL_CHECK_MS", 30000);
const stallBytes = (): number => envPositiveInt("LVMH_STALL_BYTES", 16777216);
const TRANSIENT_TYPES: ReadonlySet<string> = new Set(["message_update"]);
interface Frame {
@@ -93,7 +116,11 @@ function textOfContent(content: unknown): string {
if (!Array.isArray(content)) return "";
let out = "";
for (const block of content) {
if (block !== null && typeof block === "object" && (block as { type?: unknown }).type === "text") {
if (
block !== null &&
typeof block === "object" &&
(block as { type?: unknown }).type === "text"
) {
const text = (block as { text?: unknown }).text;
if (typeof text === "string") out += text;
}
@@ -125,14 +152,22 @@ function mapMessage(message: unknown): Record<string, unknown> | null {
if (Array.isArray(content)) {
for (const block of content) {
if (block === null || typeof block !== "object") continue;
const b = block as { type?: unknown; id?: unknown; name?: unknown; arguments?: unknown };
const b = block as {
type?: unknown;
id?: unknown;
name?: unknown;
arguments?: unknown;
};
if (b.type === "toolCall") {
toolCalls.push({
id: String(b.id ?? ""),
name: String(b.name ?? ""),
argsJson: safeJsonStringify(b.arguments),
});
} else if (b.type === "thinking" && typeof (b as { thinking?: unknown }).thinking === "string") {
} else if (
b.type === "thinking" &&
typeof (b as { thinking?: unknown }).thinking === "string"
) {
thinking += (b as { thinking: string }).thinking;
}
}
@@ -143,7 +178,10 @@ function mapMessage(message: unknown): Record<string, unknown> | null {
text: textOfContent(content),
thinking: thinking.length > 0 ? thinking : null,
toolCalls,
toolCallId: role === "toolResult" && typeof msg.toolCallId === "string" ? msg.toolCallId : null,
toolCallId:
role === "toolResult" && typeof msg.toolCallId === "string"
? msg.toolCallId
: null,
};
}
@@ -164,6 +202,9 @@ export default function (pi: ExtensionAPI): void {
let greeted: boolean = false;
let reconnectTimer: ReturnType<typeof setTimeout> | null = null;
let backoffAttempt: number = 0;
let welcomeTimer: ReturnType<typeof setTimeout> | null = null;
let stallTimer: ReturnType<typeof setInterval> | null = null;
let lastStallBytes: number | null = null;
let currentSessionId: string | null = null;
let snapshot: SessionSnapshot | null = null;
@@ -184,13 +225,22 @@ export default function (pi: ExtensionAPI): void {
event: string,
handler: (event: any, ctx: ExtensionContext) => unknown,
): void {
(pi as { on: (e: string, h: (event: any, ctx: ExtensionContext) => unknown) => void }).on(
event,
(evt: any, ctx: ExtensionContext): undefined => {
(
pi as {
on: (
e: string,
h: (event: any, ctx: ExtensionContext) => unknown,
) => void;
}
).on(event, (evt: any, ctx: ExtensionContext): undefined => {
lastCtx = ctx;
try {
const result = handler(evt, ctx);
if (result !== null && typeof result === "object" && typeof (result as Promise<unknown>).then === "function") {
if (
result !== null &&
typeof result === "object" &&
typeof (result as Promise<unknown>).then === "function"
) {
(result as Promise<unknown>).then(undefined, (err: unknown) => {
log(`async handler error (${event}): ${err}`);
});
@@ -199,8 +249,7 @@ export default function (pi: ExtensionAPI): void {
log(`handler error (${event}): ${err}`);
}
return undefined;
},
);
});
}
function enqueue(frame: Frame): void {
@@ -216,7 +265,13 @@ export default function (pi: ExtensionAPI): void {
}
function flush(): void {
while (!stopped && ws !== null && wsOpen && greeted && sendQueue.length > 0) {
while (
!stopped &&
ws !== null &&
wsOpen &&
greeted &&
sendQueue.length > 0
) {
const frame: Frame | undefined = sendQueue.shift();
if (frame === undefined) return;
let text: string;
@@ -253,7 +308,9 @@ export default function (pi: ExtensionAPI): void {
while (replayBuf.length > REPLAY_BUFFER_MAX) {
replayBuf.shift();
droppedEvents++;
log(`replay buffer overflow (cap ${REPLAY_BUFFER_MAX}), dropped oldest event`);
log(
`replay buffer overflow (cap ${REPLAY_BUFFER_MAX}), dropped oldest event`,
);
}
}
enqueue(frame);
@@ -264,6 +321,8 @@ export default function (pi: ExtensionAPI): void {
ws = null;
wsOpen = false;
greeted = false;
clearWelcomeTimer();
lastStallBytes = null;
try {
socket?.close();
} catch {
@@ -271,12 +330,64 @@ export default function (pi: ExtensionAPI): void {
}
}
function clearWelcomeTimer(): void {
if (welcomeTimer === null) return;
try {
clearTimeout(welcomeTimer);
} catch {
// clearable timers never throw in practice
}
welcomeTimer = null;
}
/** A daemon that accepts TCP but never completes the handshake would hang
* the socket (and all mirroring) forever: undici has no handshake timeout
* of its own. Arm at connect time, clear at welcome/disconnect. */
function armWelcomeTimeout(socket: WebSocket): void {
clearWelcomeTimer();
welcomeTimer = setTimeout(() => {
welcomeTimer = null;
if (ws !== socket || greeted) return;
log("welcome timeout; abandoning socket");
handleDisconnect();
scheduleReconnect();
}, welcomeTimeoutMs());
(welcomeTimer as { unref?: () => void }).unref?.();
}
/** undici's WHATWG WebSocket has no ping(). Detect a daemon that died
* without a TCP close by watching bufferedAmount: if a backlog over
* STALL_BYTES makes no progress between two ticks, the peer is not
* reading — kill the socket and reconnect. Draining resets the state. */
function checkStall(): void {
const socket: WebSocket | null = ws;
if (socket === null || !wsOpen) {
lastStallBytes = null;
return;
}
const buffered: number = socket.bufferedAmount;
if (!Number.isFinite(buffered) || buffered < stallBytes()) {
lastStallBytes = null;
return;
}
if (lastStallBytes !== null && buffered >= lastStallBytes) {
log(`ws stalled: bufferedAmount=${buffered} not draining; reconnecting`);
lastStallBytes = null;
handleDisconnect();
scheduleReconnect();
return;
}
lastStallBytes = buffered;
}
function scheduleReconnect(): void {
if (stopped || reconnectTimer !== null || ws !== null) return;
const expMs: number = BACKOFF_BASE_MS * 2 ** backoffAttempt;
const waitMs: number =
Math.min(Number.isFinite(expMs) ? expMs : BACKOFF_MAX_MS, BACKOFF_MAX_MS) +
Math.floor(Math.random() * BACKOFF_JITTER_MS);
Math.min(
Number.isFinite(expMs) ? expMs : BACKOFF_MAX_MS,
BACKOFF_MAX_MS,
) + Math.floor(Math.random() * BACKOFF_JITTER_MS);
backoffAttempt++;
reconnectTimer = setTimeout(() => {
reconnectTimer = null;
@@ -289,13 +400,20 @@ export default function (pi: ExtensionAPI): void {
if (stopped || ws !== null || reconnectTimer !== null) return;
let socket: WebSocket;
try {
socket = new WebSocket(url, { headers: { Authorization: `Bearer ${token}` } });
socket = new WebSocket(url, {
headers: { Authorization: `Bearer ${token}` },
});
} catch (err) {
log(`ws construct error: ${err}`);
scheduleReconnect();
return;
}
ws = socket;
armWelcomeTimeout(socket);
if (stallTimer === null) {
stallTimer = setInterval(checkStall, stallCheckMs());
(stallTimer as { unref?: () => void }).unref?.();
}
// Stale-socket guards: after a session switch an old socket's callbacks
// can still fire; they must never touch the state of the new socket.
socket.onopen = () => {
@@ -347,13 +465,20 @@ export default function (pi: ExtensionAPI): void {
}
function onWelcome(frame: Record<string, unknown>): void {
if (typeof frame.sessionId === "string" && frame.sessionId !== currentSessionId) {
if (
typeof frame.sessionId === "string" &&
frame.sessionId !== currentSessionId
) {
log(`welcome for foreign session ${String(frame.sessionId)}, ignoring`);
return;
}
const lastSeq: number = typeof frame.lastSeq === "number" && Number.isFinite(frame.lastSeq) ? frame.lastSeq : 0;
const lastSeq: number =
typeof frame.lastSeq === "number" && Number.isFinite(frame.lastSeq)
? frame.lastSeq
: 0;
backoffAttempt = 0;
greeted = true;
clearWelcomeTimer();
// Post-restart safety: never reuse seq numbers the daemon already has.
if (currentSessionId !== null) {
const counter: number = seqCounters.get(currentSessionId) ?? 0;
@@ -364,12 +489,16 @@ export default function (pi: ExtensionAPI): void {
// here: they are already in replayBuf, so sending both would duplicate
// them (only transient message_update frames survive the merge).
const replay: Frame[] = replayBuf.filter((f) => f.seq > lastSeq);
sendQueue = replay.concat(sendQueue.filter((f) => TRANSIENT_TYPES.has(f.type)));
sendQueue = replay.concat(
sendQueue.filter((f) => TRANSIENT_TYPES.has(f.type)),
);
while (sendQueue.length > SEND_QUEUE_MAX) {
const dropped: Frame | undefined = sendQueue.shift();
if (dropped !== undefined) {
droppedEvents++;
log(`send queue overflow during replay, dropped ${dropped.type} seq=${dropped.seq}`);
log(
`send queue overflow during replay, dropped ${dropped.type} seq=${dropped.seq}`,
);
}
}
if (droppedEvents > 0) {
@@ -389,7 +518,12 @@ export default function (pi: ExtensionAPI): void {
return;
}
if (frame === null || typeof frame !== "object") return;
const f = frame as { v?: unknown; type?: unknown; message?: unknown; sessionId?: unknown };
const f = frame as {
v?: unknown;
type?: unknown;
message?: unknown;
sessionId?: unknown;
};
if (f.v !== PROTOCOL_VERSION) return;
if (f.type === "welcome") onWelcome(f as Record<string, unknown>);
else if (f.type === "prompt") deliverPrompt(f);
@@ -397,10 +531,17 @@ export default function (pi: ExtensionAPI): void {
// unknown types are ignored (forward compatibility)
}
function deliverPrompt(frame: { message?: unknown; sessionId?: unknown }): void {
function deliverPrompt(frame: {
message?: unknown;
sessionId?: unknown;
}): void {
const message = frame.message;
if (typeof message !== "string" || message.length === 0) return;
if (typeof frame.sessionId === "string" && frame.sessionId !== currentSessionId) return;
if (
typeof frame.sessionId === "string" &&
frame.sessionId !== currentSessionId
)
return;
try {
pi.sendUserMessage(message, { deliverAs: "steer" });
} catch (err) {
@@ -418,7 +559,10 @@ export default function (pi: ExtensionAPI): void {
}
}
function buildSnapshot(ctx: ExtensionContext, sessionId: string): SessionSnapshot {
function buildSnapshot(
ctx: ExtensionContext,
sessionId: string,
): SessionSnapshot {
const sm = ctx.sessionManager as {
getSessionName?: () => string | undefined;
getCwd?: () => string;
@@ -448,8 +592,12 @@ export default function (pi: ExtensionAPI): void {
id: sessionId,
name,
cwd,
model: model !== undefined && typeof model.id === "string" ? model.id : null,
provider: model !== undefined && typeof model.provider === "string" ? model.provider : null,
model:
model !== undefined && typeof model.id === "string" ? model.id : null,
provider:
model !== undefined && typeof model.provider === "string"
? model.provider
: null,
agent: process.env[ENV_AGENT] === "1",
repo: process.env[ENV_REPO] ?? null,
startedAt,
@@ -460,7 +608,9 @@ export default function (pi: ExtensionAPI): void {
stopped = false;
let sessionId: string | null = null;
try {
const id = (ctx.sessionManager as { getSessionId?: () => string }).getSessionId?.();
const id = (
ctx.sessionManager as { getSessionId?: () => string }
).getSessionId?.();
sessionId = typeof id === "string" && id.length > 0 ? id : null;
} catch (err) {
log(`getSessionId failed: ${err}`);
@@ -495,12 +645,24 @@ export default function (pi: ExtensionAPI): void {
// ignore
}
reconnectTimer = null;
clearWelcomeTimer();
try {
if (stallTimer !== null) clearInterval(stallTimer);
} catch {
// ignore
}
stallTimer = null;
lastStallBytes = null;
const socket: WebSocket | null = ws;
ws = null;
wsOpen = false;
greeted = false;
try {
if (socket !== null && socket.readyState === WebSocket.OPEN && currentSessionId !== null) {
if (
socket !== null &&
socket.readyState === WebSocket.OPEN &&
currentSessionId !== null
) {
socket.send(
JSON.stringify({
v: PROTOCOL_VERSION,
@@ -526,16 +688,23 @@ export default function (pi: ExtensionAPI): void {
emit("session_info", { session: snapshot });
});
sub("model_select", (event: { model?: { id?: unknown; provider?: unknown } }) => {
sub(
"model_select",
(event: { model?: { id?: unknown; provider?: unknown } }) => {
if (snapshot === null) return;
const model = event.model;
const id = model !== undefined && typeof model.id === "string" ? model.id : null;
const provider = model !== undefined && typeof model.provider === "string" ? model.provider : null;
const id =
model !== undefined && typeof model.id === "string" ? model.id : null;
const provider =
model !== undefined && typeof model.provider === "string"
? model.provider
: null;
if (id === snapshot.model && provider === snapshot.provider) return;
snapshot.model = id;
snapshot.provider = provider;
emit("session_info", { session: snapshot });
});
},
);
sub("message_start", (event: { message?: unknown }) => {
const mapped = mapMessage(event.message);
@@ -549,7 +718,9 @@ export default function (pi: ExtensionAPI): void {
if (mapped === null || mapped.role !== "assistant") return;
const text = typeof mapped.text === "string" ? mapped.text : "";
// Delta only: pi hands us accumulated text; diff against what we sent.
const delta: string = text.startsWith(lastStreamText) ? text.slice(lastStreamText.length) : text;
const delta: string = text.startsWith(lastStreamText)
? text.slice(lastStreamText.length)
: text;
lastStreamText = text;
if (delta.length > 0) emit("message_update", { delta });
});
@@ -561,32 +732,56 @@ export default function (pi: ExtensionAPI): void {
emit("message_end", { message: mapped });
});
sub("tool_execution_start", (event: { toolCallId?: unknown; toolName?: unknown; args?: unknown }) => {
sub(
"tool_execution_start",
(event: { toolCallId?: unknown; toolName?: unknown; args?: unknown }) => {
emit("tool_execution_start", {
toolCallId: String(event.toolCallId ?? ""),
toolName: String(event.toolName ?? ""),
args: event.args ?? null,
});
});
},
);
sub("tool_execution_update", (event: { toolCallId?: unknown; toolName?: unknown; partialResult?: { content?: unknown } }) => {
const partial: string = textOfContent(event.partialResult?.content).slice(0, RESULT_PREVIEW_MAX_CHARS);
sub(
"tool_execution_update",
(event: {
toolCallId?: unknown;
toolName?: unknown;
partialResult?: { content?: unknown };
}) => {
const partial: string = textOfContent(event.partialResult?.content).slice(
0,
RESULT_PREVIEW_MAX_CHARS,
);
emit("tool_execution_update", {
toolCallId: String(event.toolCallId ?? ""),
toolName: String(event.toolName ?? ""),
partial,
});
});
},
);
sub("tool_execution_end", (event: { toolCallId?: unknown; toolName?: unknown; result?: { content?: unknown }; isError?: unknown }) => {
const preview: string = textOfContent(event.result?.content).slice(0, RESULT_PREVIEW_MAX_CHARS);
sub(
"tool_execution_end",
(event: {
toolCallId?: unknown;
toolName?: unknown;
result?: { content?: unknown };
isError?: unknown;
}) => {
const preview: string = textOfContent(event.result?.content).slice(
0,
RESULT_PREVIEW_MAX_CHARS,
);
emit("tool_execution_end", {
toolCallId: String(event.toolCallId ?? ""),
toolName: String(event.toolName ?? ""),
isError: event.isError === true,
resultPreview: preview,
});
});
},
);
sub("agent_start", () => {
emit("agent_start", {});
@@ -599,16 +794,34 @@ export default function (pi: ExtensionAPI): void {
let seen = false;
if (Array.isArray(event.messages)) {
for (const m of event.messages) {
if (m === null || typeof m !== "object" || (m as { role?: unknown }).role !== "assistant") continue;
const u = (m as { usage?: { input?: unknown; output?: unknown; cost?: { total?: unknown } } }).usage;
if (
m === null ||
typeof m !== "object" ||
(m as { role?: unknown }).role !== "assistant"
)
continue;
const u = (
m as {
usage?: {
input?: unknown;
output?: unknown;
cost?: { total?: unknown };
};
}
).usage;
if (u === undefined) continue;
seen = true;
inputTokens += typeof u.input === "number" ? u.input : 0;
outputTokens += typeof u.output === "number" ? u.output : 0;
totalCost += u.cost !== undefined && typeof u.cost.total === "number" ? u.cost.total : 0;
totalCost +=
u.cost !== undefined && typeof u.cost.total === "number"
? u.cost.total
: 0;
}
}
emit("agent_end", { usage: seen ? { inputTokens, outputTokens, totalCost } : {} });
emit("agent_end", {
usage: seen ? { inputTokens, outputTokens, totalCost } : {},
});
});
sub("agent_settled", () => {
+26 -5
View File
@@ -27,6 +27,9 @@ export interface MiniDaemon {
connections(): number;
pushAll(text: string): void;
dropConnections(): void;
/** Pause reads on all live sockets: simulate a daemon wedged without TCP close. */
wedge(): void;
unwedge(): void;
close(): void;
}
@@ -39,7 +42,11 @@ export function sleep(ms: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, ms));
}
export async function waitFor(cond: () => boolean, timeoutMs: number, stepMs = 25): Promise<boolean> {
export async function waitFor(
cond: () => boolean,
timeoutMs: number,
stepMs = 25,
): Promise<boolean> {
const deadline = Date.now() + timeoutMs;
while (Date.now() < deadline) {
if (cond()) return true;
@@ -67,7 +74,10 @@ function encodeTextFrame(text: string): Buffer {
return Buffer.concat([header, payload]);
}
function decodeFrames(chunk: Buffer): { frames: Array<{ opcode: number; data: Buffer }>; consumed: number } {
function decodeFrames(chunk: Buffer): {
frames: Array<{ opcode: number; data: Buffer }>;
consumed: number;
} {
const frames: Array<{ opcode: number; data: Buffer }> = [];
let offset = 0;
while (offset + 2 <= chunk.length) {
@@ -103,7 +113,10 @@ function decodeFrames(chunk: Buffer): { frames: Array<{ opcode: number; data: Bu
return { frames, consumed: offset };
}
export function startMiniDaemon(getLastSeq: () => number): Promise<MiniDaemon> {
export function startMiniDaemon(
getLastSeq: () => number,
opts: { suppressWelcome?: boolean } = {},
): Promise<MiniDaemon> {
const server = http.createServer();
const sockets = new Set<Duplex>();
const daemon: MiniDaemon = {
@@ -119,6 +132,12 @@ export function startMiniDaemon(getLastSeq: () => number): Promise<MiniDaemon> {
for (const s of sockets) s.destroy();
sockets.clear();
},
wedge() {
for (const s of sockets) s.pause();
},
unwedge() {
for (const s of sockets) s.resume();
},
close() {
daemon.dropConnections();
server.close();
@@ -127,7 +146,9 @@ export function startMiniDaemon(getLastSeq: () => number): Promise<MiniDaemon> {
server.on("upgrade", (req: http.IncomingMessage, socket: Duplex) => {
daemon.authHeaders.push(String(req.headers.authorization ?? ""));
const key = String(req.headers["sec-websocket-key"] ?? "");
const accept = createHash("sha1").update(key + WS_GUID).digest("base64");
const accept = createHash("sha1")
.update(key + WS_GUID)
.digest("base64");
socket.write(
"HTTP/1.1 101 Switching Protocols\r\nUpgrade: websocket\r\nConnection: Upgrade\r\n" +
`Sec-WebSocket-Accept: ${accept}\r\n\r\n`,
@@ -151,7 +172,7 @@ export function startMiniDaemon(getLastSeq: () => number): Promise<MiniDaemon> {
try {
const frame = JSON.parse(f.data.toString("utf8")) as ClientFrame;
daemon.frames.push(frame);
if (frame.type === "hello") {
if (frame.type === "hello" && !opts.suppressWelcome) {
socket.write(
encodeTextFrame(
JSON.stringify({
+257 -39
View File
@@ -17,7 +17,12 @@
* 6. session_shutdown -> socket closed, no reconnect afterwards
*/
import { check as rawCheck, sleep, waitFor, startMiniDaemon } from "./mini-daemon.ts";
import {
check as rawCheck,
sleep,
waitFor,
startMiniDaemon,
} from "./mini-daemon.ts";
const SESSION_ID = "sess-1";
@@ -62,13 +67,18 @@ function makeFakeCtx(): unknown {
getSessionId: () => SESSION_ID,
getSessionName: () => undefined,
getCwd: () => "/work/repo",
getHeader: () => ({ timestamp: "2024-12-03T14:00:00.000Z", id: SESSION_ID }),
getHeader: () => ({
timestamp: "2024-12-03T14:00:00.000Z",
id: SESSION_ID,
}),
},
};
}
async function loadExtension(): Promise<(pi: unknown) => void> {
const mod = (await import("./lvmh-agent.ts")) as { default: (pi: unknown) => void };
const mod = (await import("./lvmh-agent.ts")) as {
default: (pi: unknown) => void;
};
return mod.default;
}
@@ -109,7 +119,10 @@ async function main(): Promise<void> {
"agent_end",
"agent_settled",
];
check("2 all 13 handlers registered", handlerNames.every((n) => pi.handlers.has(n)));
check(
"2 all 13 handlers registered",
handlerNames.every((n) => pi.handlers.has(n)),
);
let lastReturn: unknown = "sentinel";
const fire = (name: string, event: unknown): void => {
@@ -120,9 +133,16 @@ async function main(): Promise<void> {
};
fire("session_start", { reason: "startup" });
const helloSeen = await waitFor(() => daemon.frames.some((f) => f.type === "hello"), 5000);
const helloSeen = await waitFor(
() => daemon.frames.some((f) => f.type === "hello"),
5000,
);
check("2 connects and sends hello", helloSeen);
check("2 bearer auth on upgrade", daemon.authHeaders.at(-1) === "Bearer smoke-token", daemon.authHeaders.at(-1));
check(
"2 bearer auth on upgrade",
daemon.authHeaders.at(-1) === "Bearer smoke-token",
daemon.authHeaders.at(-1),
);
const hello = daemon.frames.find((f) => f.type === "hello");
const hs = (hello?.session ?? {}) as Record<string, unknown>;
@@ -139,34 +159,71 @@ async function main(): Promise<void> {
JSON.stringify(hs),
);
fire("message_start", { message: { role: "assistant", content: [], timestamp: 1000 } });
fire("message_update", { message: { role: "assistant", content: [{ type: "text", text: "Hel" }], timestamp: 1000 } });
fire("message_update", { message: { role: "assistant", content: [{ type: "text", text: "Hello world" }], timestamp: 1000 } });
fire("message_start", {
message: { role: "assistant", content: [], timestamp: 1000 },
});
fire("message_update", {
message: {
role: "assistant",
content: [{ type: "text", text: "Hel" }],
timestamp: 1000,
},
});
fire("message_update", {
message: {
role: "assistant",
content: [{ type: "text", text: "Hello world" }],
timestamp: 1000,
},
});
fire("message_end", {
message: {
role: "assistant",
content: [
{ type: "thinking", thinking: "hmm" },
{ type: "text", text: "Hello world" },
{ type: "toolCall", id: "tc1", name: "bash", arguments: { command: "ls" } },
{
type: "toolCall",
id: "tc1",
name: "bash",
arguments: { command: "ls" },
},
],
timestamp: 1000,
},
});
await waitFor(() => daemon.frames.some((f) => f.type === "message_end"), 2000);
await waitFor(
() => daemon.frames.some((f) => f.type === "message_end"),
2000,
);
const n0 = daemon.frames.findIndex((f) => f.type === "hello");
const mirrored = daemon.frames.slice(n0 + 1).map((f) => f.type);
check(
"2 mirror order",
JSON.stringify(mirrored) === JSON.stringify(["message_start", "message_update", "message_update", "message_end"]),
JSON.stringify(mirrored) ===
JSON.stringify([
"message_start",
"message_update",
"message_update",
"message_end",
]),
JSON.stringify(mirrored),
);
const deltas = daemon.frames.filter((f) => f.type === "message_update").map((f) => f.delta);
check("2 delta-only updates", JSON.stringify(deltas) === JSON.stringify(["Hel", "lo world"]), JSON.stringify(deltas));
const deltas = daemon.frames
.filter((f) => f.type === "message_update")
.map((f) => f.delta);
check(
"2 delta-only updates",
JSON.stringify(deltas) === JSON.stringify(["Hel", "lo world"]),
JSON.stringify(deltas),
);
const msg = daemon.frames.find((f) => f.type === "message_end")?.message as Record<string, unknown>;
const tc = (msg?.toolCalls as Array<Record<string, unknown>> | undefined)?.[0];
const msg = daemon.frames.find((f) => f.type === "message_end")
?.message as Record<string, unknown>;
const tc = (
msg?.toolCalls as Array<Record<string, unknown>> | undefined
)?.[0];
check(
"2 message_end mapping",
msg?.role === "assistant" &&
@@ -182,23 +239,52 @@ async function main(): Promise<void> {
const big = "x".repeat(3000);
fire("agent_start", {});
fire("tool_execution_start", { toolCallId: "tc1", toolName: "bash", args: { command: "ls" } });
fire("tool_execution_update", { toolCallId: "tc1", toolName: "bash", partialResult: { content: [{ type: "text", text: big }] } });
fire("tool_execution_end", { toolCallId: "tc1", toolName: "bash", isError: true, result: { content: [{ type: "text", text: big }] } });
fire("agent_end", { messages: [{ role: "assistant", usage: { input: 10, output: 5, cost: { total: 0.25 } } }] });
fire("tool_execution_start", {
toolCallId: "tc1",
toolName: "bash",
args: { command: "ls" },
});
fire("tool_execution_update", {
toolCallId: "tc1",
toolName: "bash",
partialResult: { content: [{ type: "text", text: big }] },
});
fire("tool_execution_end", {
toolCallId: "tc1",
toolName: "bash",
isError: true,
result: { content: [{ type: "text", text: big }] },
});
fire("agent_end", {
messages: [
{
role: "assistant",
usage: { input: 10, output: 5, cost: { total: 0.25 } },
},
],
});
fire("agent_settled", {});
await waitFor(() => daemon.frames.some((f) => f.type === "agent_settled"), 2000);
await waitFor(
() => daemon.frames.some((f) => f.type === "agent_settled"),
2000,
);
const partial = daemon.frames.find((f) => f.type === "tool_execution_update");
const end = daemon.frames.find((f) => f.type === "tool_execution_end");
const partialLen = (partial?.partial as string | undefined)?.length;
const previewLen = (end?.resultPreview as string | undefined)?.length;
check("2 partial truncated to 2000", partialLen === 2000, String(partialLen));
check("2 resultPreview truncated + isError", previewLen === 2000 && end?.isError === true);
const usage = daemon.frames.find((f) => f.type === "agent_end")?.usage as Record<string, number>;
check(
"2 resultPreview truncated + isError",
previewLen === 2000 && end?.isError === true,
);
const usage = daemon.frames.find((f) => f.type === "agent_end")
?.usage as Record<string, number>;
check(
"2 agent_end usage",
usage?.inputTokens === 10 && usage?.outputTokens === 5 && usage?.totalCost === 0.25,
usage?.inputTokens === 10 &&
usage?.outputTokens === 5 &&
usage?.totalCost === 0.25,
JSON.stringify(usage),
);
@@ -208,14 +294,33 @@ async function main(): Promise<void> {
seqs.length > 0 && seqs.every((s, i) => i === 0 || s > seqs[i - 1]),
JSON.stringify(seqs),
);
check("2 handler return values are undefined (never a promise)", lastReturn === undefined);
check(
"2 handler return values are undefined (never a promise)",
lastReturn === undefined,
);
// --- Scenario 3: prompt / abort ------------------------------------
daemon.pushAll(
JSON.stringify({ v: 1, type: "prompt", sessionId: "other-session", seq: 0, ts: Date.now(), promptId: "p0", message: "wrong session" }),
JSON.stringify({
v: 1,
type: "prompt",
sessionId: "other-session",
seq: 0,
ts: Date.now(),
promptId: "p0",
message: "wrong session",
}),
);
daemon.pushAll(
JSON.stringify({ v: 1, type: "prompt", sessionId: SESSION_ID, seq: 0, ts: Date.now(), promptId: "p1", message: "run the tests" }),
JSON.stringify({
v: 1,
type: "prompt",
sessionId: SESSION_ID,
seq: 0,
ts: Date.now(),
promptId: "p1",
message: "run the tests",
}),
);
await waitFor(() => pi.sentMessages.length > 0, 2000);
check(
@@ -225,7 +330,15 @@ async function main(): Promise<void> {
JSON.stringify(pi.sentMessages[0].options) === '{"deliverAs":"steer"}',
JSON.stringify(pi.sentMessages),
);
daemon.pushAll(JSON.stringify({ v: 1, type: "abort", sessionId: SESSION_ID, seq: 0, ts: Date.now() }));
daemon.pushAll(
JSON.stringify({
v: 1,
type: "abort",
sessionId: SESSION_ID,
seq: 0,
ts: Date.now(),
}),
);
await waitFor(() => pi.aborted > 0, 2000);
check("3 abort calls pi.abort", pi.aborted === 1);
@@ -234,8 +347,12 @@ async function main(): Promise<void> {
welcomeLastSeq = seqBeforeDrop; // daemon has everything up to the drop
daemon.dropConnections();
await sleep(150); // let the client notice
fire("message_end", { message: { role: "user", content: "offline-1", timestamp: 2000 } });
fire("message_end", { message: { role: "user", content: "offline-2", timestamp: 2001 } });
fire("message_end", {
message: { role: "user", content: "offline-1", timestamp: 2000 },
});
fire("message_end", {
message: { role: "user", content: "offline-2", timestamp: 2001 },
});
const reconnected = await waitFor(
() => daemon.frames.filter((f) => f.type === "hello").length >= 2,
10000,
@@ -251,11 +368,18 @@ async function main(): Promise<void> {
);
const allSeqs = daemon.frames.filter((f) => f.seq > 0).map((f) => f.seq);
const dupCount = allSeqs.length - new Set(allSeqs).size;
check("4 no duplicate seq delivery", dupCount === 0, `duplicates: ${dupCount}`);
const seqsAfter = daemon.frames.filter((f) => f.seq > seqBeforeDrop).map((f) => f.seq);
check(
"4 no duplicate seq delivery",
dupCount === 0,
`duplicates: ${dupCount}`,
);
const seqsAfter = daemon.frames
.filter((f) => f.seq > seqBeforeDrop)
.map((f) => f.seq);
check(
"4 seq continues monotonically across reconnect",
seqsAfter.length >= 2 && seqsAfter.every((s, i) => i === 0 || s > seqsAfter[i - 1]),
seqsAfter.length >= 2 &&
seqsAfter.every((s, i) => i === 0 || s > seqsAfter[i - 1]),
JSON.stringify(seqsAfter),
);
@@ -279,18 +403,107 @@ async function main(): Promise<void> {
const overflow = daemon.frames.find((f) => f.type === "buffer_overflow");
check(
"4 buffer_overflow notice after drops",
overflowNotice && typeof (overflow?.dropped as number) === "number" && (overflow?.dropped as number) > 0,
overflowNotice &&
typeof (overflow?.dropped as number) === "number" &&
(overflow?.dropped as number) > 0,
JSON.stringify(overflow ?? null),
);
// --- Scenario 9: daemon hangs mid-handshake (no welcome) ----------
// Fast watchdog knobs; read lazily by the plugin so late env wins.
process.env.LVMH_WELCOME_TIMEOUT_MS = "400";
process.env.LVMH_STALL_CHECK_MS = "120";
process.env.LVMH_STALL_BYTES = "1000000";
const hungDaemon = await startMiniDaemon(() => 0, { suppressWelcome: true });
const prevUrl = process.env.LVMH_URL;
process.env.LVMH_URL = hungDaemon.url;
const pi3 = makeFakePi();
factory(pi3);
pi3.handlers.get("session_start")?.({ reason: "startup" }, makeFakeCtx());
const hungHellos = await waitFor(
() => hungDaemon.frames.filter((f) => f.type === "hello").length >= 2,
12000,
);
check(
"9 welcome timeout: abandons hung socket and retries",
hungHellos,
String(hungDaemon.frames.length),
);
pi3.handlers.get("session_shutdown")?.({ reason: "quit" }, makeFakeCtx());
hungDaemon.close();
process.env.LVMH_URL = prevUrl;
// --- Scenario 10: daemon dies without TCP close (wedged socket) ----
// wedge() pauses daemon-side reads: TCP stays "open", undici keeps
// buffering sends. The plugin must detect bufferedAmount not draining
// and reconnect, or mirroring would be dead forever + memory unbounded.
// Fresh plugin instance: its stall interval inherits the fast knobs set
// in scenario 9 (interval period is fixed at first connect).
const stallDaemon = await startMiniDaemon(() => 0);
const urlBeforeStall: string | undefined = process.env.LVMH_URL;
process.env.LVMH_URL = stallDaemon.url;
const pi4 = makeFakePi();
factory(pi4);
const fire4 = (name: string, event: unknown): void => {
const h = pi4.handlers.get(name);
if (h === undefined) throw new Error(`missing handler ${name}`);
const r = h(event, makeFakeCtx());
if (r instanceof Promise) r.catch(() => undefined);
};
fire4("session_start", { reason: "startup" });
await waitFor(() => stallDaemon.frames.some((f) => f.type === "hello"), 5000);
stallDaemon.wedge();
const bigText: string = "w".repeat(1024 * 1024);
for (let i = 0; i < 60; i++) {
fire4("message_end", {
message: { role: "user", content: bigText, timestamp: 3000 + i },
});
}
const wedgedRecovered = await waitFor(
() => stallDaemon.frames.filter((f) => f.type === "hello").length >= 2,
20000,
);
check(
"10 stall watchdog: wedged socket detected, reconnected",
wedgedRecovered,
String(stallDaemon.frames.length),
);
stallDaemon.unwedge();
fire4("message_end", {
message: { role: "user", content: "after-stall", timestamp: 4000 },
});
const afterStallDelivered = await waitFor(
() =>
stallDaemon.frames.some(
(f) =>
f.type === "message_end" &&
(f.message as { text?: string })?.text === "after-stall",
),
15000,
);
check("10 mirroring works after stall recovery", afterStallDelivered);
fire4("session_shutdown", { reason: "quit" });
stallDaemon.close();
process.env.LVMH_URL = urlBeforeStall;
delete process.env.LVMH_WELCOME_TIMEOUT_MS;
delete process.env.LVMH_STALL_CHECK_MS;
delete process.env.LVMH_STALL_BYTES;
// --- Scenario 6: session_shutdown ----------------------------------
const helloCountAtShutdown = daemon.frames.filter((f) => f.type === "hello").length;
const helloCountAtShutdown = daemon.frames.filter(
(f) => f.type === "hello",
).length;
fire("session_shutdown", { reason: "quit" });
const closed = await waitFor(() => daemon.connections() === 0, 3000);
check("6 shutdown closes connection", closed);
await sleep(2600); // longer than one backoff attempt (~1-2s)
const helloCountAfter = daemon.frames.filter((f) => f.type === "hello").length;
check("6 no reconnect after shutdown", helloCountAfter === helloCountAtShutdown);
const helloCountAfter = daemon.frames.filter(
(f) => f.type === "hello",
).length;
check(
"6 no reconnect after shutdown",
helloCountAfter === helloCountAtShutdown,
);
daemon.close();
@@ -310,7 +523,10 @@ async function main(): Promise<void> {
check("5 dead host: pi loads, event loop responsive", loopResponsive);
let threw = false;
try {
pi2.handlers.get("message_end")?.({ message: { role: "user", content: "hi", timestamp: 1 } }, makeFakeCtx());
pi2.handlers.get("message_end")?.(
{ message: { role: "user", content: "hi", timestamp: 1 } },
makeFakeCtx(),
);
} catch {
threw = true;
}
@@ -318,7 +534,9 @@ async function main(): Promise<void> {
delete process.env.LVMH_URL;
delete process.env.LVMH_TOKEN;
console.log(failures === 0 ? "\nALL CHECKS PASSED" : `\n${failures} CHECK(S) FAILED`);
console.log(
failures === 0 ? "\nALL CHECKS PASSED" : `\n${failures} CHECK(S) FAILED`,
);
process.exit(failures === 0 ? 0 : 1);
}