fix(e2e): replace LLM-seeded question tests with route mocking

The question dock e2e tests used seedSessionQuestion which sends a
prompt to a real LLM and waits for it to call the question tool. This
is inherently flaky due to LLM latency and non-determinism.

Add withMockQuestion (mirroring the existing withMockPermission pattern)
that intercepts GET /question and POST /question/*/reply at the
Playwright route level, making the tests fully deterministic.
pull/17824/head
Kit Langton 2026-03-16 12:16:16 -04:00
parent 15b27e0d18
commit eb5c67de58
1 changed files with 109 additions and 43 deletions

View File

@ -5,7 +5,7 @@ import {
type ComposerProbeState, type ComposerProbeState,
type ComposerWindow, type ComposerWindow,
} from "../../src/testing/session-composer" } from "../../src/testing/session-composer"
import { cleanupSession, clearSessionDockSeed, seedSessionQuestion } from "../actions" import { cleanupSession } from "../actions"
import { import {
permissionDockSelector, permissionDockSelector,
promptSelector, promptSelector,
@ -13,8 +13,9 @@ import {
sessionComposerDockSelector, sessionComposerDockSelector,
sessionTodoToggleButtonSelector, sessionTodoToggleButtonSelector,
} from "../selectors" } from "../selectors"
import { createSdk } from "../utils"
type Sdk = Parameters<typeof clearSessionDockSeed>[0] type Sdk = ReturnType<typeof createSdk>
type PermissionRule = { permission: string; pattern: string; action: "allow" | "deny" | "ask" } type PermissionRule = { permission: string; pattern: string; action: "allow" | "deny" | "ask" }
async function withDockSession<T>( async function withDockSession<T>(
@ -36,14 +37,6 @@ async function withDockSession<T>(
test.setTimeout(120_000) test.setTimeout(120_000)
async function withDockSeed<T>(sdk: Sdk, sessionID: string, fn: () => Promise<T>) {
try {
return await fn()
} finally {
await clearSessionDockSeed(sdk, sessionID).catch(() => undefined)
}
}
async function clearPermissionDock(page: any, label: RegExp) { async function clearPermissionDock(page: any, label: RegExp) {
const dock = page.locator(permissionDockSelector) const dock = page.locator(permissionDockSelector)
await expect(dock).toBeVisible() await expect(dock).toBeVisible()
@ -79,6 +72,65 @@ async function expectPermissionOpen(page: any) {
await expect(page.locator(promptSelector)).toBeVisible() await expect(page.locator(promptSelector)).toBeVisible()
} }
async function withMockQuestion<T>(
page: any,
request: {
id: string
sessionID: string
questions: Array<{
header: string
question: string
options: Array<{ label: string; description: string }>
multiple?: boolean
custom?: boolean
}>
},
fn: (state: { resolved: () => Promise<void> }) => Promise<T>,
) {
let pending = [
{
id: request.id,
sessionID: request.sessionID,
questions: request.questions,
},
]
const list = async (route: any) => {
await route.fulfill({
status: 200,
contentType: "application/json",
body: JSON.stringify(pending),
})
}
const reply = async (route: any) => {
const url = new URL(route.request().url())
const id = url.pathname.split("/").at(-2)
pending = pending.filter((item) => item.id !== id)
await route.fulfill({
status: 200,
contentType: "application/json",
body: JSON.stringify(true),
})
}
await page.route("**/question", list)
await page.route("**/question/*/reply", reply)
const state = {
async resolved() {
await expect.poll(() => pending.length, { timeout: 10_000 }).toBe(0)
},
}
try {
return await fn(state)
} finally {
await page.unroute("**/question", list)
await page.unroute("**/question/*/reply", reply)
}
}
async function todoDock(page: any, sessionID: string) { async function todoDock(page: any, sessionID: string) {
await page.addInitScript(() => { await page.addInitScript(() => {
const win = window as ComposerWindow const win = window as ComposerWindow
@ -275,10 +327,12 @@ test("auto-accept toggle works before first submit", async ({ page, gotoSession
test("blocked question flow unblocks after submit", async ({ page, sdk, gotoSession }) => { test("blocked question flow unblocks after submit", async ({ page, sdk, gotoSession }) => {
await withDockSession(sdk, "e2e composer dock question", async (session) => { await withDockSession(sdk, "e2e composer dock question", async (session) => {
await withDockSeed(sdk, session.id, async () => { await gotoSession(session.id)
await gotoSession(session.id)
await seedSessionQuestion(sdk, { await withMockQuestion(
page,
{
id: "que_e2e_question",
sessionID: session.id, sessionID: session.id,
questions: [ questions: [
{ {
@ -290,16 +344,19 @@ test("blocked question flow unblocks after submit", async ({ page, sdk, gotoSess
], ],
}, },
], ],
}) },
async (state) => {
await page.goto(page.url())
await expectQuestionBlocked(page)
const dock = page.locator(questionDockSelector) const dock = page.locator(questionDockSelector)
await expectQuestionBlocked(page) await dock.locator('[data-slot="question-option"]').first().click()
await dock.getByRole("button", { name: /submit/i }).click()
await dock.locator('[data-slot="question-option"]').first().click() await state.resolved()
await dock.getByRole("button", { name: /submit/i }).click() await page.goto(page.url())
await expectQuestionOpen(page)
await expectQuestionOpen(page) },
}) )
}) })
}) })
@ -400,8 +457,10 @@ test("child session question request blocks parent dock and unblocks after submi
if (!child?.id) throw new Error("Child session create did not return an id") if (!child?.id) throw new Error("Child session create did not return an id")
try { try {
await withDockSeed(sdk, child.id, async () => { await withMockQuestion(
await seedSessionQuestion(sdk, { page,
{
id: "que_e2e_child_question",
sessionID: child.id, sessionID: child.id,
questions: [ questions: [
{ {
@ -413,16 +472,19 @@ test("child session question request blocks parent dock and unblocks after submi
], ],
}, },
], ],
}) },
async (state) => {
await page.goto(page.url())
await expectQuestionBlocked(page)
const dock = page.locator(questionDockSelector) const dock = page.locator(questionDockSelector)
await expectQuestionBlocked(page) await dock.locator('[data-slot="question-option"]').first().click()
await dock.getByRole("button", { name: /submit/i }).click()
await dock.locator('[data-slot="question-option"]').first().click() await state.resolved()
await dock.getByRole("button", { name: /submit/i }).click() await page.goto(page.url())
await expectQuestionOpen(page)
await expectQuestionOpen(page) },
}) )
} finally { } finally {
await cleanupSession({ sdk, sessionID: child.id }) await cleanupSession({ sdk, sessionID: child.id })
} }
@ -506,10 +568,12 @@ test("todo dock transitions and collapse behavior", async ({ page, sdk, gotoSess
test("keyboard focus stays off prompt while blocked", async ({ page, sdk, gotoSession }) => { test("keyboard focus stays off prompt while blocked", async ({ page, sdk, gotoSession }) => {
await withDockSession(sdk, "e2e composer dock keyboard", async (session) => { await withDockSession(sdk, "e2e composer dock keyboard", async (session) => {
await withDockSeed(sdk, session.id, async () => { await gotoSession(session.id)
await gotoSession(session.id)
await seedSessionQuestion(sdk, { await withMockQuestion(
page,
{
id: "que_e2e_keyboard",
sessionID: session.id, sessionID: session.id,
questions: [ questions: [
{ {
@ -518,13 +582,15 @@ test("keyboard focus stays off prompt while blocked", async ({ page, sdk, gotoSe
options: [{ label: "Continue", description: "Continue now" }], options: [{ label: "Continue", description: "Continue now" }],
}, },
], ],
}) },
async () => {
await page.goto(page.url())
await expectQuestionBlocked(page)
await expectQuestionBlocked(page) await page.locator("main").click({ position: { x: 5, y: 5 } })
await page.keyboard.type("abc")
await page.locator("main").click({ position: { x: 5, y: 5 } }) await expect(page.locator(promptSelector)).toHaveCount(0)
await page.keyboard.type("abc") },
await expect(page.locator(promptSelector)).toHaveCount(0) )
})
}) })
}) })