From eb5c67de58976c9d14edff183e7dca22fc298ecb Mon Sep 17 00:00:00 2001 From: Kit Langton Date: Mon, 16 Mar 2026 12:16:16 -0400 Subject: [PATCH] fix(e2e): replace LLM-seeded question tests with route mocking The question dock e2e tests used seedSessionQuestion which sends a prompt to a real LLM and waits for it to call the question tool. This is inherently flaky due to LLM latency and non-determinism. Add withMockQuestion (mirroring the existing withMockPermission pattern) that intercepts GET /question and POST /question/*/reply at the Playwright route level, making the tests fully deterministic. --- .../e2e/session/session-composer-dock.spec.ts | 152 +++++++++++++----- 1 file changed, 109 insertions(+), 43 deletions(-) diff --git a/packages/app/e2e/session/session-composer-dock.spec.ts b/packages/app/e2e/session/session-composer-dock.spec.ts index 5b2e8a8c6f..fad79aa9f2 100644 --- a/packages/app/e2e/session/session-composer-dock.spec.ts +++ b/packages/app/e2e/session/session-composer-dock.spec.ts @@ -5,7 +5,7 @@ import { type ComposerProbeState, type ComposerWindow, } from "../../src/testing/session-composer" -import { cleanupSession, clearSessionDockSeed, seedSessionQuestion } from "../actions" +import { cleanupSession } from "../actions" import { permissionDockSelector, promptSelector, @@ -13,8 +13,9 @@ import { sessionComposerDockSelector, sessionTodoToggleButtonSelector, } from "../selectors" +import { createSdk } from "../utils" -type Sdk = Parameters[0] +type Sdk = ReturnType type PermissionRule = { permission: string; pattern: string; action: "allow" | "deny" | "ask" } async function withDockSession( @@ -36,14 +37,6 @@ async function withDockSession( test.setTimeout(120_000) -async function withDockSeed(sdk: Sdk, sessionID: string, fn: () => Promise) { - try { - return await fn() - } finally { - await clearSessionDockSeed(sdk, sessionID).catch(() => undefined) - } -} - async function clearPermissionDock(page: any, label: RegExp) { const dock = page.locator(permissionDockSelector) await expect(dock).toBeVisible() @@ -79,6 +72,65 @@ async function expectPermissionOpen(page: any) { await expect(page.locator(promptSelector)).toBeVisible() } +async function withMockQuestion( + page: any, + request: { + id: string + sessionID: string + questions: Array<{ + header: string + question: string + options: Array<{ label: string; description: string }> + multiple?: boolean + custom?: boolean + }> + }, + fn: (state: { resolved: () => Promise }) => Promise, +) { + let pending = [ + { + id: request.id, + sessionID: request.sessionID, + questions: request.questions, + }, + ] + + const list = async (route: any) => { + await route.fulfill({ + status: 200, + contentType: "application/json", + body: JSON.stringify(pending), + }) + } + + const reply = async (route: any) => { + const url = new URL(route.request().url()) + const id = url.pathname.split("/").at(-2) + pending = pending.filter((item) => item.id !== id) + await route.fulfill({ + status: 200, + contentType: "application/json", + body: JSON.stringify(true), + }) + } + + await page.route("**/question", list) + await page.route("**/question/*/reply", reply) + + const state = { + async resolved() { + await expect.poll(() => pending.length, { timeout: 10_000 }).toBe(0) + }, + } + + try { + return await fn(state) + } finally { + await page.unroute("**/question", list) + await page.unroute("**/question/*/reply", reply) + } +} + async function todoDock(page: any, sessionID: string) { await page.addInitScript(() => { const win = window as ComposerWindow @@ -275,10 +327,12 @@ test("auto-accept toggle works before first submit", async ({ page, gotoSession test("blocked question flow unblocks after submit", async ({ page, sdk, gotoSession }) => { await withDockSession(sdk, "e2e composer dock question", async (session) => { - await withDockSeed(sdk, session.id, async () => { - await gotoSession(session.id) + await gotoSession(session.id) - await seedSessionQuestion(sdk, { + await withMockQuestion( + page, + { + id: "que_e2e_question", sessionID: session.id, questions: [ { @@ -290,16 +344,19 @@ test("blocked question flow unblocks after submit", async ({ page, sdk, gotoSess ], }, ], - }) + }, + async (state) => { + await page.goto(page.url()) + await expectQuestionBlocked(page) - const dock = page.locator(questionDockSelector) - await expectQuestionBlocked(page) - - await dock.locator('[data-slot="question-option"]').first().click() - await dock.getByRole("button", { name: /submit/i }).click() - - await expectQuestionOpen(page) - }) + const dock = page.locator(questionDockSelector) + await dock.locator('[data-slot="question-option"]').first().click() + await dock.getByRole("button", { name: /submit/i }).click() + await state.resolved() + await page.goto(page.url()) + await expectQuestionOpen(page) + }, + ) }) }) @@ -400,8 +457,10 @@ test("child session question request blocks parent dock and unblocks after submi if (!child?.id) throw new Error("Child session create did not return an id") try { - await withDockSeed(sdk, child.id, async () => { - await seedSessionQuestion(sdk, { + await withMockQuestion( + page, + { + id: "que_e2e_child_question", sessionID: child.id, questions: [ { @@ -413,16 +472,19 @@ test("child session question request blocks parent dock and unblocks after submi ], }, ], - }) + }, + async (state) => { + await page.goto(page.url()) + await expectQuestionBlocked(page) - const dock = page.locator(questionDockSelector) - await expectQuestionBlocked(page) - - await dock.locator('[data-slot="question-option"]').first().click() - await dock.getByRole("button", { name: /submit/i }).click() - - await expectQuestionOpen(page) - }) + const dock = page.locator(questionDockSelector) + await dock.locator('[data-slot="question-option"]').first().click() + await dock.getByRole("button", { name: /submit/i }).click() + await state.resolved() + await page.goto(page.url()) + await expectQuestionOpen(page) + }, + ) } finally { await cleanupSession({ sdk, sessionID: child.id }) } @@ -506,10 +568,12 @@ test("todo dock transitions and collapse behavior", async ({ page, sdk, gotoSess test("keyboard focus stays off prompt while blocked", async ({ page, sdk, gotoSession }) => { await withDockSession(sdk, "e2e composer dock keyboard", async (session) => { - await withDockSeed(sdk, session.id, async () => { - await gotoSession(session.id) + await gotoSession(session.id) - await seedSessionQuestion(sdk, { + await withMockQuestion( + page, + { + id: "que_e2e_keyboard", sessionID: session.id, questions: [ { @@ -518,13 +582,15 @@ test("keyboard focus stays off prompt while blocked", async ({ page, sdk, gotoSe options: [{ label: "Continue", description: "Continue now" }], }, ], - }) + }, + async () => { + await page.goto(page.url()) + await expectQuestionBlocked(page) - await expectQuestionBlocked(page) - - await page.locator("main").click({ position: { x: 5, y: 5 } }) - await page.keyboard.type("abc") - await expect(page.locator(promptSelector)).toHaveCount(0) - }) + await page.locator("main").click({ position: { x: 5, y: 5 } }) + await page.keyboard.type("abc") + await expect(page.locator(promptSelector)).toHaveCount(0) + }, + ) }) })