Compare commits

...

2 Commits

Author SHA1 Message Date
Kit Langton 7aeb0972aa test(e2e): share dock child session mock 2026-03-16 12:39:32 -04:00
Kit Langton eb5c67de58 fix(e2e): replace LLM-seeded question tests with route mocking
The question dock e2e tests used seedSessionQuestion which sends a
prompt to a real LLM and waits for it to call the question tool. This
is inherently flaky due to LLM latency and non-determinism.

Add withMockQuestion (mirroring the existing withMockPermission pattern)
that intercepts GET /question and POST /question/*/reply at the
Playwright route level, making the tests fully deterministic.
2026-03-16 12:16:16 -04:00
1 changed files with 142 additions and 64 deletions

View File

@ -5,7 +5,7 @@ import {
type ComposerProbeState, type ComposerProbeState,
type ComposerWindow, type ComposerWindow,
} from "../../src/testing/session-composer" } from "../../src/testing/session-composer"
import { cleanupSession, clearSessionDockSeed, seedSessionQuestion } from "../actions" import { cleanupSession } from "../actions"
import { import {
permissionDockSelector, permissionDockSelector,
promptSelector, promptSelector,
@ -13,9 +13,11 @@ import {
sessionComposerDockSelector, sessionComposerDockSelector,
sessionTodoToggleButtonSelector, sessionTodoToggleButtonSelector,
} from "../selectors" } from "../selectors"
import { createSdk } from "../utils"
type Sdk = Parameters<typeof clearSessionDockSeed>[0] type Sdk = ReturnType<typeof createSdk>
type PermissionRule = { permission: string; pattern: string; action: "allow" | "deny" | "ask" } type PermissionRule = { permission: string; pattern: string; action: "allow" | "deny" | "ask" }
type Child = { id: string }
async function withDockSession<T>( async function withDockSession<T>(
sdk: Sdk, sdk: Sdk,
@ -36,14 +38,6 @@ async function withDockSession<T>(
test.setTimeout(120_000) test.setTimeout(120_000)
async function withDockSeed<T>(sdk: Sdk, sessionID: string, fn: () => Promise<T>) {
try {
return await fn()
} finally {
await clearSessionDockSeed(sdk, sessionID).catch(() => undefined)
}
}
async function clearPermissionDock(page: any, label: RegExp) { async function clearPermissionDock(page: any, label: RegExp) {
const dock = page.locator(permissionDockSelector) const dock = page.locator(permissionDockSelector)
await expect(dock).toBeVisible() await expect(dock).toBeVisible()
@ -79,6 +73,91 @@ async function expectPermissionOpen(page: any) {
await expect(page.locator(promptSelector)).toBeVisible() await expect(page.locator(promptSelector)).toBeVisible()
} }
async function withMockSession<T>(page: any, child: Child | undefined, fn: () => Promise<T>) {
if (!child) return await fn()
const list = async (route: any) => {
const res = await route.fetch()
const json = await res.json()
const list = Array.isArray(json) ? json : Array.isArray(json?.data) ? json.data : undefined
if (Array.isArray(list) && !list.some((item) => item?.id === child.id)) list.push(child)
await route.fulfill({
status: res.status(),
headers: res.headers(),
contentType: "application/json",
body: JSON.stringify(json),
})
}
await page.route("**/session?*", list)
try {
return await fn()
} finally {
await page.unroute("**/session?*", list)
}
}
async function withMockQuestion<T>(
page: any,
request: {
id: string
sessionID: string
questions: Array<{
header: string
question: string
options: Array<{ label: string; description: string }>
multiple?: boolean
custom?: boolean
}>
},
child: Child | undefined,
fn: (state: { resolved: () => Promise<void> }) => Promise<T>,
) {
let pending = [
{
id: request.id,
sessionID: request.sessionID,
questions: request.questions,
},
]
const list = async (route: any) => {
await route.fulfill({
status: 200,
contentType: "application/json",
body: JSON.stringify(pending),
})
}
const reply = async (route: any) => {
const url = new URL(route.request().url())
const id = url.pathname.split("/").at(-2)
pending = pending.filter((item) => item.id !== id)
await route.fulfill({
status: 200,
contentType: "application/json",
body: JSON.stringify(true),
})
}
await page.route("**/question", list)
await page.route("**/question/*/reply", reply)
const state = {
async resolved() {
await expect.poll(() => pending.length, { timeout: 10_000 }).toBe(0)
},
}
try {
return await withMockSession(page, child, () => fn(state))
} finally {
await page.unroute("**/question", list)
await page.unroute("**/question/*/reply", reply)
}
}
async function todoDock(page: any, sessionID: string) { async function todoDock(page: any, sessionID: string) {
await page.addInitScript(() => { await page.addInitScript(() => {
const win = window as ComposerWindow const win = window as ComposerWindow
@ -183,7 +262,7 @@ async function withMockPermission<T>(
metadata?: Record<string, unknown> metadata?: Record<string, unknown>
always?: string[] always?: string[]
}, },
opts: { child?: any } | undefined, child: Child | undefined,
fn: (state: { resolved: () => Promise<void> }) => Promise<T>, fn: (state: { resolved: () => Promise<void> }) => Promise<T>,
) { ) {
let pending = [ let pending = [
@ -216,23 +295,6 @@ async function withMockPermission<T>(
await page.route("**/permission", list) await page.route("**/permission", list)
await page.route("**/session/*/permissions/*", reply) await page.route("**/session/*/permissions/*", reply)
const sessionList = opts?.child
? async (route: any) => {
const res = await route.fetch()
const json = await res.json()
const list = Array.isArray(json) ? json : Array.isArray(json?.data) ? json.data : undefined
if (Array.isArray(list) && !list.some((item) => item?.id === opts.child?.id)) list.push(opts.child)
await route.fulfill({
status: res.status(),
headers: res.headers(),
contentType: "application/json",
body: JSON.stringify(json),
})
}
: undefined
if (sessionList) await page.route("**/session?*", sessionList)
const state = { const state = {
async resolved() { async resolved() {
await expect.poll(() => pending.length, { timeout: 10_000 }).toBe(0) await expect.poll(() => pending.length, { timeout: 10_000 }).toBe(0)
@ -240,11 +302,10 @@ async function withMockPermission<T>(
} }
try { try {
return await fn(state) return await withMockSession(page, child, () => fn(state))
} finally { } finally {
await page.unroute("**/permission", list) await page.unroute("**/permission", list)
await page.unroute("**/session/*/permissions/*", reply) await page.unroute("**/session/*/permissions/*", reply)
if (sessionList) await page.unroute("**/session?*", sessionList)
} }
} }
@ -275,10 +336,12 @@ test("auto-accept toggle works before first submit", async ({ page, gotoSession
test("blocked question flow unblocks after submit", async ({ page, sdk, gotoSession }) => { test("blocked question flow unblocks after submit", async ({ page, sdk, gotoSession }) => {
await withDockSession(sdk, "e2e composer dock question", async (session) => { await withDockSession(sdk, "e2e composer dock question", async (session) => {
await withDockSeed(sdk, session.id, async () => {
await gotoSession(session.id) await gotoSession(session.id)
await seedSessionQuestion(sdk, { await withMockQuestion(
page,
{
id: "que_e2e_question",
sessionID: session.id, sessionID: session.id,
questions: [ questions: [
{ {
@ -290,16 +353,20 @@ test("blocked question flow unblocks after submit", async ({ page, sdk, gotoSess
], ],
}, },
], ],
}) },
undefined,
const dock = page.locator(questionDockSelector) async (state) => {
await page.goto(page.url())
await expectQuestionBlocked(page) await expectQuestionBlocked(page)
const dock = page.locator(questionDockSelector)
await dock.locator('[data-slot="question-option"]').first().click() await dock.locator('[data-slot="question-option"]').first().click()
await dock.getByRole("button", { name: /submit/i }).click() await dock.getByRole("button", { name: /submit/i }).click()
await state.resolved()
await page.goto(page.url())
await expectQuestionOpen(page) await expectQuestionOpen(page)
}) },
)
}) })
}) })
@ -400,8 +467,10 @@ test("child session question request blocks parent dock and unblocks after submi
if (!child?.id) throw new Error("Child session create did not return an id") if (!child?.id) throw new Error("Child session create did not return an id")
try { try {
await withDockSeed(sdk, child.id, async () => { await withMockQuestion(
await seedSessionQuestion(sdk, { page,
{
id: "que_e2e_child_question",
sessionID: child.id, sessionID: child.id,
questions: [ questions: [
{ {
@ -413,16 +482,20 @@ test("child session question request blocks parent dock and unblocks after submi
], ],
}, },
], ],
}) },
child,
const dock = page.locator(questionDockSelector) async (state) => {
await page.goto(page.url())
await expectQuestionBlocked(page) await expectQuestionBlocked(page)
const dock = page.locator(questionDockSelector)
await dock.locator('[data-slot="question-option"]').first().click() await dock.locator('[data-slot="question-option"]').first().click()
await dock.getByRole("button", { name: /submit/i }).click() await dock.getByRole("button", { name: /submit/i }).click()
await state.resolved()
await page.goto(page.url())
await expectQuestionOpen(page) await expectQuestionOpen(page)
}) },
)
} finally { } finally {
await cleanupSession({ sdk, sessionID: child.id }) await cleanupSession({ sdk, sessionID: child.id })
} }
@ -456,7 +529,7 @@ test("child session permission request blocks parent dock and supports allow onc
patterns: ["/tmp/opencode-e2e-perm-child"], patterns: ["/tmp/opencode-e2e-perm-child"],
metadata: { description: "Need child permission" }, metadata: { description: "Need child permission" },
}, },
{ child }, child,
async (state) => { async (state) => {
await page.goto(page.url()) await page.goto(page.url())
await expectPermissionBlocked(page) await expectPermissionBlocked(page)
@ -506,10 +579,12 @@ test("todo dock transitions and collapse behavior", async ({ page, sdk, gotoSess
test("keyboard focus stays off prompt while blocked", async ({ page, sdk, gotoSession }) => { test("keyboard focus stays off prompt while blocked", async ({ page, sdk, gotoSession }) => {
await withDockSession(sdk, "e2e composer dock keyboard", async (session) => { await withDockSession(sdk, "e2e composer dock keyboard", async (session) => {
await withDockSeed(sdk, session.id, async () => {
await gotoSession(session.id) await gotoSession(session.id)
await seedSessionQuestion(sdk, { await withMockQuestion(
page,
{
id: "que_e2e_keyboard",
sessionID: session.id, sessionID: session.id,
questions: [ questions: [
{ {
@ -518,13 +593,16 @@ test("keyboard focus stays off prompt while blocked", async ({ page, sdk, gotoSe
options: [{ label: "Continue", description: "Continue now" }], options: [{ label: "Continue", description: "Continue now" }],
}, },
], ],
}) },
undefined,
async () => {
await page.goto(page.url())
await expectQuestionBlocked(page) await expectQuestionBlocked(page)
await page.locator("main").click({ position: { x: 5, y: 5 } }) await page.locator("main").click({ position: { x: 5, y: 5 } })
await page.keyboard.type("abc") await page.keyboard.type("abc")
await expect(page.locator(promptSelector)).toHaveCount(0) await expect(page.locator(promptSelector)).toHaveCount(0)
}) },
)
}) })
}) })