diff --git a/packages/app/e2e/actions.ts b/packages/app/e2e/actions.ts index dc023ddc0b..df8e0768ed 100644 --- a/packages/app/e2e/actions.ts +++ b/packages/app/e2e/actions.ts @@ -1,5 +1,5 @@ import { base64Decode, base64Encode } from "@opencode-ai/util/encode" -import { expect, type Locator, type Page } from "@playwright/test" +import { expect, type Locator, type Page, type Route } from "@playwright/test" import fs from "node:fs/promises" import os from "node:os" import path from "node:path" @@ -43,6 +43,27 @@ export async function defocus(page: Page) { .catch(() => undefined) } +export async function withNoReplyPrompt(page: Page, fn: () => Promise) { + const url = "**/session/*/prompt_async" + const route = async (input: Route) => { + const body = input.request().postDataJSON() + await input.continue({ + postData: JSON.stringify({ ...body, noReply: true }), + headers: { + ...input.request().headers(), + "content-type": "application/json", + }, + }) + } + + await page.route(url, route) + try { + return await fn() + } finally { + await page.unroute(url, route) + } +} + async function terminalID(term: Locator) { const id = await term.getAttribute(terminalAttr) if (id) return id diff --git a/packages/app/e2e/projects/projects-switch.spec.ts b/packages/app/e2e/projects/projects-switch.spec.ts index b46c1b407e..f87a47cf09 100644 --- a/packages/app/e2e/projects/projects-switch.spec.ts +++ b/packages/app/e2e/projects/projects-switch.spec.ts @@ -10,6 +10,7 @@ import { waitSession, waitSessionSaved, waitSlug, + withNoReplyPrompt, } from "../actions" import { projectSwitchSelector, promptSelector, workspaceItemSelector, workspaceNewSessionSelector } from "../selectors" import { dirSlug, resolveDirectory } from "../utils" @@ -81,8 +82,10 @@ test("switching back to a project opens the latest workspace session", async ({ // Create a session by sending a prompt const prompt = page.locator(promptSelector) await expect(prompt).toBeVisible() - await prompt.fill("test") - await page.keyboard.press("Enter") + await withNoReplyPrompt(page, async () => { + await prompt.fill("test") + await page.keyboard.press("Enter") + }) // Wait for the URL to update with the new session ID await expect.poll(() => sessionIDFromUrl(page.url()) ?? "", { timeout: 15_000 }).not.toBe("") diff --git a/packages/app/e2e/projects/workspace-new-session.spec.ts b/packages/app/e2e/projects/workspace-new-session.spec.ts index 3a7a6bbc22..835c8c99ed 100644 --- a/packages/app/e2e/projects/workspace-new-session.spec.ts +++ b/packages/app/e2e/projects/workspace-new-session.spec.ts @@ -9,6 +9,7 @@ import { waitSession, waitSessionSaved, waitSlug, + withNoReplyPrompt, } from "../actions" import { promptSelector, workspaceItemSelector, workspaceNewSessionSelector } from "../selectors" import { createSdk } from "../utils" @@ -58,8 +59,10 @@ async function createSessionFromWorkspace( const prompt = page.locator(promptSelector) await expect(prompt).toBeVisible() - await prompt.fill(text) - await page.keyboard.press("Enter") + await withNoReplyPrompt(page, async () => { + await prompt.fill(text) + await page.keyboard.press("Enter") + }) await expect.poll(() => sessionIDFromUrl(page.url()) ?? "", { timeout: 15_000 }).not.toBe("") const sessionID = sessionIDFromUrl(page.url()) diff --git a/packages/app/e2e/session/session-model-persistence.spec.ts b/packages/app/e2e/session/session-model-persistence.spec.ts index 36cbb0fbf1..66bc451bcf 100644 --- a/packages/app/e2e/session/session-model-persistence.spec.ts +++ b/packages/app/e2e/session/session-model-persistence.spec.ts @@ -8,11 +8,11 @@ import { waitSession, waitSessionIdle, waitSlug, + withNoReplyPrompt, } from "../actions" import { promptAgentSelector, promptModelSelector, - promptSelector, promptVariantSelector, workspaceItemSelector, workspaceNewSessionSelector, @@ -231,11 +231,14 @@ async function goto(page: Page, directory: string, sessionID?: string) { } async function submit(page: Page, value: string) { - const prompt = page.locator(promptSelector) + const prompt = page.locator('[data-component="prompt-input"]') await expect(prompt).toBeVisible() - await prompt.click() - await prompt.fill(value) - await prompt.press("Enter") + + await withNoReplyPrompt(page, async () => { + await prompt.click() + await prompt.fill(value) + await prompt.press("Enter") + }) await expect.poll(() => sessionIDFromUrl(page.url()) ?? "", { timeout: 30_000 }).not.toBe("") const id = sessionIDFromUrl(page.url()) diff --git a/packages/app/test/e2e/no-real-llm.test.ts b/packages/app/test/e2e/no-real-llm.test.ts new file mode 100644 index 0000000000..c801df56f8 --- /dev/null +++ b/packages/app/test/e2e/no-real-llm.test.ts @@ -0,0 +1,27 @@ +import { describe, expect, test } from "bun:test" +import path from "node:path" +import { fileURLToPath } from "node:url" + +const dir = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../../e2e") + +function hasPrompt(src: string) { + if (!src.includes("withProject(")) return false + if (src.includes("withNoReplyPrompt(")) return false + if (src.includes("session.promptAsync({") && !src.includes("noReply: true")) return true + if (!src.includes("promptSelector")) return false + return src.includes('keyboard.press("Enter")') || src.includes('prompt.press("Enter")') +} + +describe("e2e llm guard", () => { + test("withProject specs do not submit prompt replies", async () => { + const bad: string[] = [] + + for await (const file of new Bun.Glob("**/*.spec.ts").scan({ cwd: dir, absolute: true })) { + const src = await Bun.file(file).text() + if (!hasPrompt(src)) continue + bad.push(path.relative(dir, file)) + } + + expect(bad).toEqual([]) + }) +})