From c8ecd640220331ce7695d72ea8c618dd8909eab1 Mon Sep 17 00:00:00 2001 From: Kit Langton Date: Tue, 31 Mar 2026 21:24:39 -0400 Subject: [PATCH] test(app): add mock llm e2e fixture (#20375) --- packages/app/e2e/fixtures.ts | 67 ++++++++++++++- packages/app/e2e/prompt/prompt.spec.ts | 113 +++++++++++++++++-------- 2 files changed, 142 insertions(+), 38 deletions(-) diff --git a/packages/app/e2e/fixtures.ts b/packages/app/e2e/fixtures.ts index ca06858a45..7fc4cda057 100644 --- a/packages/app/e2e/fixtures.ts +++ b/packages/app/e2e/fixtures.ts @@ -1,5 +1,8 @@ import { test as base, expect, type Page } from "@playwright/test" +import { ManagedRuntime } from "effect" import type { E2EWindow } from "../src/testing/terminal" +import type { Item, Reply, Usage } from "../../opencode/test/lib/llm-server" +import { TestLLMServer } from "../../opencode/test/lib/llm-server" import { healthPhase, cleanupSession, @@ -13,6 +16,24 @@ import { } from "./actions" import { createSdk, dirSlug, getWorktree, sessionPath } from "./utils" +type LLMFixture = { + url: string + push: (...input: (Item | Reply)[]) => Promise + text: (value: string, opts?: { usage?: Usage }) => Promise + tool: (name: string, input: unknown) => Promise + toolHang: (name: string, input: unknown) => Promise + reason: (value: string, opts?: { text?: string; usage?: Usage }) => Promise + fail: (message?: unknown) => Promise + error: (status: number, body: unknown) => Promise + hang: () => Promise + hold: (value: string, wait: PromiseLike) => Promise + hits: () => Promise }>> + calls: () => Promise + wait: (count: number) => Promise + inputs: () => Promise[]> + pending: () => Promise +} + export const settingsKey = "settings.v3" const seedModel = (() => { @@ -26,6 +47,7 @@ const seedModel = (() => { })() type TestFixtures = { + llm: LLMFixture sdk: ReturnType gotoSession: (sessionID?: string) => Promise withProject: ( @@ -36,7 +58,11 @@ type TestFixtures = { trackSession: (sessionID: string, directory?: string) => void trackDirectory: (directory: string) => void }) => Promise, - options?: { extra?: string[] }, + options?: { + extra?: string[] + model?: { providerID: string; modelID: string } + setup?: (directory: string) => Promise + }, ) => Promise } @@ -46,6 +72,31 @@ type WorkerFixtures = { } export const test = base.extend({ + llm: async ({}, use) => { + const rt = ManagedRuntime.make(TestLLMServer.layer) + try { + const svc = await rt.runPromise(TestLLMServer.asEffect()) + await use({ + url: svc.url, + push: (...input) => rt.runPromise(svc.push(...input)), + text: (value, opts) => rt.runPromise(svc.text(value, opts)), + tool: (name, input) => rt.runPromise(svc.tool(name, input)), + toolHang: (name, input) => rt.runPromise(svc.toolHang(name, input)), + reason: (value, opts) => rt.runPromise(svc.reason(value, opts)), + fail: (message) => rt.runPromise(svc.fail(message)), + error: (status, body) => rt.runPromise(svc.error(status, body)), + hang: () => rt.runPromise(svc.hang), + hold: (value, wait) => rt.runPromise(svc.hold(value, wait)), + hits: () => rt.runPromise(svc.hits), + calls: () => rt.runPromise(svc.calls), + wait: (count) => rt.runPromise(svc.wait(count)), + inputs: () => rt.runPromise(svc.inputs), + pending: () => rt.runPromise(svc.pending), + }) + } finally { + await rt.dispose() + } + }, page: async ({ page }, use) => { let boundary: string | undefined setHealthPhase(page, "test") @@ -99,7 +150,8 @@ export const test = base.extend({ const root = await createTestProject() const sessions = new Map() const dirs = new Set() - await seedStorage(page, { directory: root, extra: options?.extra }) + await options?.setup?.(root) + await seedStorage(page, { directory: root, extra: options?.extra, model: options?.model }) const gotoSession = async (sessionID?: string) => { await page.goto(sessionPath(root, sessionID)) @@ -133,7 +185,14 @@ export const test = base.extend({ }, }) -async function seedStorage(page: Page, input: { directory: string; extra?: string[] }) { +async function seedStorage( + page: Page, + input: { + directory: string + extra?: string[] + model?: { providerID: string; modelID: string } + }, +) { await seedProjects(page, input) await page.addInitScript((model: { providerID: string; modelID: string }) => { const win = window as E2EWindow @@ -158,7 +217,7 @@ async function seedStorage(page: Page, input: { directory: string; extra?: strin variant: {}, }), ) - }, seedModel) + }, input.model ?? seedModel) } export { expect } diff --git a/packages/app/e2e/prompt/prompt.spec.ts b/packages/app/e2e/prompt/prompt.spec.ts index 0466d0988c..1acf17f5bf 100644 --- a/packages/app/e2e/prompt/prompt.spec.ts +++ b/packages/app/e2e/prompt/prompt.spec.ts @@ -1,8 +1,44 @@ +import fs from "node:fs/promises" +import path from "node:path" import { test, expect } from "../fixtures" import { promptSelector } from "../selectors" -import { cleanupSession, sessionIDFromUrl, withSession } from "../actions" +import { sessionIDFromUrl } from "../actions" +import { createSdk } from "../utils" -test("can send a prompt and receive a reply", async ({ page, sdk, gotoSession }) => { +async function config(dir: string, url: string) { + await fs.writeFile( + path.join(dir, "opencode.json"), + JSON.stringify({ + $schema: "https://opencode.ai/config.json", + enabled_providers: ["e2e-llm"], + provider: { + "e2e-llm": { + name: "E2E LLM", + npm: "@ai-sdk/openai-compatible", + env: [], + models: { + "test-model": { + name: "Test Model", + tool_call: true, + limit: { context: 128000, output: 32000 }, + }, + }, + options: { + apiKey: "test-key", + baseURL: url, + }, + }, + }, + agent: { + build: { + model: "e2e-llm/test-model", + }, + }, + }), + ) +} + +test("can send a prompt and receive a reply", async ({ page, llm, withProject }) => { test.setTimeout(120_000) const pageErrors: string[] = [] @@ -11,42 +47,51 @@ test("can send a prompt and receive a reply", async ({ page, sdk, gotoSession }) } page.on("pageerror", onPageError) - await gotoSession() - - const token = `E2E_OK_${Date.now()}` - - const prompt = page.locator(promptSelector) - await prompt.click() - await page.keyboard.type(`Reply with exactly: ${token}`) - await page.keyboard.press("Enter") - - await expect(page).toHaveURL(/\/session\/[^/?#]+/, { timeout: 30_000 }) - - const sessionID = (() => { - const id = sessionIDFromUrl(page.url()) - if (!id) throw new Error(`Failed to parse session id from url: ${page.url()}`) - return id - })() - try { - await expect - .poll( - async () => { - const messages = await sdk.session.messages({ sessionID, limit: 50 }).then((r) => r.data ?? []) - return messages - .filter((m) => m.info.role === "assistant") - .flatMap((m) => m.parts) - .filter((p) => p.type === "text") - .map((p) => p.text) - .join("\n") - }, - { timeout: 90_000 }, - ) + await withProject( + async (project) => { + const sdk = createSdk(project.directory) + const token = `E2E_OK_${Date.now()}` - .toContain(token) + await llm.text(token) + await project.gotoSession() + + const prompt = page.locator(promptSelector) + await prompt.click() + await page.keyboard.type(`Reply with exactly: ${token}`) + await page.keyboard.press("Enter") + + await expect(page).toHaveURL(/\/session\/[^/?#]+/, { timeout: 30_000 }) + + const sessionID = (() => { + const id = sessionIDFromUrl(page.url()) + if (!id) throw new Error(`Failed to parse session id from url: ${page.url()}`) + return id + })() + project.trackSession(sessionID) + + await expect + .poll( + async () => { + const messages = await sdk.session.messages({ sessionID, limit: 50 }).then((r) => r.data ?? []) + return messages + .filter((m) => m.info.role === "assistant") + .flatMap((m) => m.parts) + .filter((p) => p.type === "text") + .map((p) => p.text) + .join("\n") + }, + { timeout: 30_000 }, + ) + .toContain(token) + }, + { + model: { providerID: "e2e-llm", modelID: "test-model" }, + setup: (dir) => config(dir, llm.url), + }, + ) } finally { page.off("pageerror", onPageError) - await cleanupSession({ sdk, sessionID }) } if (pageErrors.length > 0) {