fix(e2e): replace LLM-seeded question tests with route mocking
The question dock e2e tests used seedSessionQuestion which sends a prompt to a real LLM and waits for it to call the question tool. This is inherently flaky due to LLM latency and non-determinism. Add withMockQuestion (mirroring the existing withMockPermission pattern) that intercepts GET /question and POST /question/*/reply at the Playwright route level, making the tests fully deterministic.pull/17824/head
parent
15b27e0d18
commit
eb5c67de58
|
|
@ -5,7 +5,7 @@ import {
|
||||||
type ComposerProbeState,
|
type ComposerProbeState,
|
||||||
type ComposerWindow,
|
type ComposerWindow,
|
||||||
} from "../../src/testing/session-composer"
|
} from "../../src/testing/session-composer"
|
||||||
import { cleanupSession, clearSessionDockSeed, seedSessionQuestion } from "../actions"
|
import { cleanupSession } from "../actions"
|
||||||
import {
|
import {
|
||||||
permissionDockSelector,
|
permissionDockSelector,
|
||||||
promptSelector,
|
promptSelector,
|
||||||
|
|
@ -13,8 +13,9 @@ import {
|
||||||
sessionComposerDockSelector,
|
sessionComposerDockSelector,
|
||||||
sessionTodoToggleButtonSelector,
|
sessionTodoToggleButtonSelector,
|
||||||
} from "../selectors"
|
} from "../selectors"
|
||||||
|
import { createSdk } from "../utils"
|
||||||
|
|
||||||
type Sdk = Parameters<typeof clearSessionDockSeed>[0]
|
type Sdk = ReturnType<typeof createSdk>
|
||||||
type PermissionRule = { permission: string; pattern: string; action: "allow" | "deny" | "ask" }
|
type PermissionRule = { permission: string; pattern: string; action: "allow" | "deny" | "ask" }
|
||||||
|
|
||||||
async function withDockSession<T>(
|
async function withDockSession<T>(
|
||||||
|
|
@ -36,14 +37,6 @@ async function withDockSession<T>(
|
||||||
|
|
||||||
test.setTimeout(120_000)
|
test.setTimeout(120_000)
|
||||||
|
|
||||||
async function withDockSeed<T>(sdk: Sdk, sessionID: string, fn: () => Promise<T>) {
|
|
||||||
try {
|
|
||||||
return await fn()
|
|
||||||
} finally {
|
|
||||||
await clearSessionDockSeed(sdk, sessionID).catch(() => undefined)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async function clearPermissionDock(page: any, label: RegExp) {
|
async function clearPermissionDock(page: any, label: RegExp) {
|
||||||
const dock = page.locator(permissionDockSelector)
|
const dock = page.locator(permissionDockSelector)
|
||||||
await expect(dock).toBeVisible()
|
await expect(dock).toBeVisible()
|
||||||
|
|
@ -79,6 +72,65 @@ async function expectPermissionOpen(page: any) {
|
||||||
await expect(page.locator(promptSelector)).toBeVisible()
|
await expect(page.locator(promptSelector)).toBeVisible()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async function withMockQuestion<T>(
|
||||||
|
page: any,
|
||||||
|
request: {
|
||||||
|
id: string
|
||||||
|
sessionID: string
|
||||||
|
questions: Array<{
|
||||||
|
header: string
|
||||||
|
question: string
|
||||||
|
options: Array<{ label: string; description: string }>
|
||||||
|
multiple?: boolean
|
||||||
|
custom?: boolean
|
||||||
|
}>
|
||||||
|
},
|
||||||
|
fn: (state: { resolved: () => Promise<void> }) => Promise<T>,
|
||||||
|
) {
|
||||||
|
let pending = [
|
||||||
|
{
|
||||||
|
id: request.id,
|
||||||
|
sessionID: request.sessionID,
|
||||||
|
questions: request.questions,
|
||||||
|
},
|
||||||
|
]
|
||||||
|
|
||||||
|
const list = async (route: any) => {
|
||||||
|
await route.fulfill({
|
||||||
|
status: 200,
|
||||||
|
contentType: "application/json",
|
||||||
|
body: JSON.stringify(pending),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
const reply = async (route: any) => {
|
||||||
|
const url = new URL(route.request().url())
|
||||||
|
const id = url.pathname.split("/").at(-2)
|
||||||
|
pending = pending.filter((item) => item.id !== id)
|
||||||
|
await route.fulfill({
|
||||||
|
status: 200,
|
||||||
|
contentType: "application/json",
|
||||||
|
body: JSON.stringify(true),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
await page.route("**/question", list)
|
||||||
|
await page.route("**/question/*/reply", reply)
|
||||||
|
|
||||||
|
const state = {
|
||||||
|
async resolved() {
|
||||||
|
await expect.poll(() => pending.length, { timeout: 10_000 }).toBe(0)
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
return await fn(state)
|
||||||
|
} finally {
|
||||||
|
await page.unroute("**/question", list)
|
||||||
|
await page.unroute("**/question/*/reply", reply)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
async function todoDock(page: any, sessionID: string) {
|
async function todoDock(page: any, sessionID: string) {
|
||||||
await page.addInitScript(() => {
|
await page.addInitScript(() => {
|
||||||
const win = window as ComposerWindow
|
const win = window as ComposerWindow
|
||||||
|
|
@ -275,10 +327,12 @@ test("auto-accept toggle works before first submit", async ({ page, gotoSession
|
||||||
|
|
||||||
test("blocked question flow unblocks after submit", async ({ page, sdk, gotoSession }) => {
|
test("blocked question flow unblocks after submit", async ({ page, sdk, gotoSession }) => {
|
||||||
await withDockSession(sdk, "e2e composer dock question", async (session) => {
|
await withDockSession(sdk, "e2e composer dock question", async (session) => {
|
||||||
await withDockSeed(sdk, session.id, async () => {
|
await gotoSession(session.id)
|
||||||
await gotoSession(session.id)
|
|
||||||
|
|
||||||
await seedSessionQuestion(sdk, {
|
await withMockQuestion(
|
||||||
|
page,
|
||||||
|
{
|
||||||
|
id: "que_e2e_question",
|
||||||
sessionID: session.id,
|
sessionID: session.id,
|
||||||
questions: [
|
questions: [
|
||||||
{
|
{
|
||||||
|
|
@ -290,16 +344,19 @@ test("blocked question flow unblocks after submit", async ({ page, sdk, gotoSess
|
||||||
],
|
],
|
||||||
},
|
},
|
||||||
],
|
],
|
||||||
})
|
},
|
||||||
|
async (state) => {
|
||||||
|
await page.goto(page.url())
|
||||||
|
await expectQuestionBlocked(page)
|
||||||
|
|
||||||
const dock = page.locator(questionDockSelector)
|
const dock = page.locator(questionDockSelector)
|
||||||
await expectQuestionBlocked(page)
|
await dock.locator('[data-slot="question-option"]').first().click()
|
||||||
|
await dock.getByRole("button", { name: /submit/i }).click()
|
||||||
await dock.locator('[data-slot="question-option"]').first().click()
|
await state.resolved()
|
||||||
await dock.getByRole("button", { name: /submit/i }).click()
|
await page.goto(page.url())
|
||||||
|
await expectQuestionOpen(page)
|
||||||
await expectQuestionOpen(page)
|
},
|
||||||
})
|
)
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
|
||||||
|
|
@ -400,8 +457,10 @@ test("child session question request blocks parent dock and unblocks after submi
|
||||||
if (!child?.id) throw new Error("Child session create did not return an id")
|
if (!child?.id) throw new Error("Child session create did not return an id")
|
||||||
|
|
||||||
try {
|
try {
|
||||||
await withDockSeed(sdk, child.id, async () => {
|
await withMockQuestion(
|
||||||
await seedSessionQuestion(sdk, {
|
page,
|
||||||
|
{
|
||||||
|
id: "que_e2e_child_question",
|
||||||
sessionID: child.id,
|
sessionID: child.id,
|
||||||
questions: [
|
questions: [
|
||||||
{
|
{
|
||||||
|
|
@ -413,16 +472,19 @@ test("child session question request blocks parent dock and unblocks after submi
|
||||||
],
|
],
|
||||||
},
|
},
|
||||||
],
|
],
|
||||||
})
|
},
|
||||||
|
async (state) => {
|
||||||
|
await page.goto(page.url())
|
||||||
|
await expectQuestionBlocked(page)
|
||||||
|
|
||||||
const dock = page.locator(questionDockSelector)
|
const dock = page.locator(questionDockSelector)
|
||||||
await expectQuestionBlocked(page)
|
await dock.locator('[data-slot="question-option"]').first().click()
|
||||||
|
await dock.getByRole("button", { name: /submit/i }).click()
|
||||||
await dock.locator('[data-slot="question-option"]').first().click()
|
await state.resolved()
|
||||||
await dock.getByRole("button", { name: /submit/i }).click()
|
await page.goto(page.url())
|
||||||
|
await expectQuestionOpen(page)
|
||||||
await expectQuestionOpen(page)
|
},
|
||||||
})
|
)
|
||||||
} finally {
|
} finally {
|
||||||
await cleanupSession({ sdk, sessionID: child.id })
|
await cleanupSession({ sdk, sessionID: child.id })
|
||||||
}
|
}
|
||||||
|
|
@ -506,10 +568,12 @@ test("todo dock transitions and collapse behavior", async ({ page, sdk, gotoSess
|
||||||
|
|
||||||
test("keyboard focus stays off prompt while blocked", async ({ page, sdk, gotoSession }) => {
|
test("keyboard focus stays off prompt while blocked", async ({ page, sdk, gotoSession }) => {
|
||||||
await withDockSession(sdk, "e2e composer dock keyboard", async (session) => {
|
await withDockSession(sdk, "e2e composer dock keyboard", async (session) => {
|
||||||
await withDockSeed(sdk, session.id, async () => {
|
await gotoSession(session.id)
|
||||||
await gotoSession(session.id)
|
|
||||||
|
|
||||||
await seedSessionQuestion(sdk, {
|
await withMockQuestion(
|
||||||
|
page,
|
||||||
|
{
|
||||||
|
id: "que_e2e_keyboard",
|
||||||
sessionID: session.id,
|
sessionID: session.id,
|
||||||
questions: [
|
questions: [
|
||||||
{
|
{
|
||||||
|
|
@ -518,13 +582,15 @@ test("keyboard focus stays off prompt while blocked", async ({ page, sdk, gotoSe
|
||||||
options: [{ label: "Continue", description: "Continue now" }],
|
options: [{ label: "Continue", description: "Continue now" }],
|
||||||
},
|
},
|
||||||
],
|
],
|
||||||
})
|
},
|
||||||
|
async () => {
|
||||||
|
await page.goto(page.url())
|
||||||
|
await expectQuestionBlocked(page)
|
||||||
|
|
||||||
await expectQuestionBlocked(page)
|
await page.locator("main").click({ position: { x: 5, y: 5 } })
|
||||||
|
await page.keyboard.type("abc")
|
||||||
await page.locator("main").click({ position: { x: 5, y: 5 } })
|
await expect(page.locator(promptSelector)).toHaveCount(0)
|
||||||
await page.keyboard.type("abc")
|
},
|
||||||
await expect(page.locator(promptSelector)).toHaveCount(0)
|
)
|
||||||
})
|
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue