From 38d22765920ef5047db4d9f1bdc0bdf602e6906f Mon Sep 17 00:00:00 2001 From: Kit Langton Date: Wed, 1 Apr 2026 11:58:11 -0400 Subject: [PATCH] test(e2e): isolate prompt tests with per-worker backend (#20464) --- packages/app/e2e/actions.ts | 52 +++--- packages/app/e2e/backend.ts | 119 +++++++++++++ packages/app/e2e/fixtures.ts | 167 +++++++++++++------ packages/app/e2e/prompt/mock.ts | 46 +++++ packages/app/e2e/prompt/prompt-async.spec.ts | 58 ++++--- packages/app/e2e/prompt/prompt.spec.ts | 106 ++++-------- packages/app/e2e/utils.ts | 14 +- packages/opencode/test/lib/llm-server.ts | 53 ++++-- 8 files changed, 429 insertions(+), 186 deletions(-) create mode 100644 packages/app/e2e/backend.ts create mode 100644 packages/app/e2e/prompt/mock.ts diff --git a/packages/app/e2e/actions.ts b/packages/app/e2e/actions.ts index efd370d395..dc023ddc0b 100644 --- a/packages/app/e2e/actions.ts +++ b/packages/app/e2e/actions.ts @@ -312,10 +312,11 @@ export async function openSettings(page: Page) { return dialog } -export async function seedProjects(page: Page, input: { directory: string; extra?: string[] }) { +export async function seedProjects(page: Page, input: { directory: string; extra?: string[]; serverUrl?: string }) { await page.addInitScript( (args: { directory: string; serverUrl: string; extra: string[] }) => { const key = "opencode.global.dat:server" + const defaultKey = "opencode.settings.dat:defaultServerUrl" const raw = localStorage.getItem(key) const parsed = (() => { if (!raw) return undefined @@ -331,6 +332,7 @@ export async function seedProjects(page: Page, input: { directory: string; extra const lastProject = store.lastProject && typeof store.lastProject === "object" ? store.lastProject : {} const projects = store.projects && typeof store.projects === "object" ? store.projects : {} const nextProjects = { ...(projects as Record) } + const nextList = list.includes(args.serverUrl) ? list : [args.serverUrl, ...list] const add = (origin: string, directory: string) => { const current = nextProjects[origin] @@ -356,17 +358,18 @@ export async function seedProjects(page: Page, input: { directory: string; extra localStorage.setItem( key, JSON.stringify({ - list, + list: nextList, projects: nextProjects, lastProject, }), ) + localStorage.setItem(defaultKey, args.serverUrl) }, - { directory: input.directory, serverUrl, extra: input.extra ?? [] }, + { directory: input.directory, serverUrl: input.serverUrl ?? serverUrl, extra: input.extra ?? [] }, ) } -export async function createTestProject() { +export async function createTestProject(input?: { serverUrl?: string }) { const root = await fs.mkdtemp(path.join(os.tmpdir(), "opencode-e2e-project-")) const id = `e2e-${path.basename(root)}` @@ -381,7 +384,7 @@ export async function createTestProject() { stdio: "ignore", }) - return resolveDirectory(root) + return resolveDirectory(root, input?.serverUrl) } export async function cleanupTestProject(directory: string) { @@ -430,22 +433,22 @@ export async function waitSlug(page: Page, skip: string[] = []) { return next } -export async function resolveSlug(slug: string) { +export async function resolveSlug(slug: string, input?: { serverUrl?: string }) { const directory = base64Decode(slug) if (!directory) throw new Error(`Failed to decode workspace slug: ${slug}`) - const resolved = await resolveDirectory(directory) + const resolved = await resolveDirectory(directory, input?.serverUrl) return { directory: resolved, slug: base64Encode(resolved), raw: slug } } -export async function waitDir(page: Page, directory: string) { - const target = await resolveDirectory(directory) +export async function waitDir(page: Page, directory: string, input?: { serverUrl?: string }) { + const target = await resolveDirectory(directory, input?.serverUrl) await expect .poll( async () => { await assertHealthy(page, "waitDir") const slug = slugFromUrl(page.url()) if (!slug) return "" - return resolveSlug(slug) + return resolveSlug(slug, input) .then((item) => item.directory) .catch(() => "") }, @@ -455,15 +458,15 @@ export async function waitDir(page: Page, directory: string) { return { directory: target, slug: base64Encode(target) } } -export async function waitSession(page: Page, input: { directory: string; sessionID?: string }) { - const target = await resolveDirectory(input.directory) +export async function waitSession(page: Page, input: { directory: string; sessionID?: string; serverUrl?: string }) { + const target = await resolveDirectory(input.directory, input.serverUrl) await expect .poll( async () => { await assertHealthy(page, "waitSession") const slug = slugFromUrl(page.url()) if (!slug) return false - const resolved = await resolveSlug(slug).catch(() => undefined) + const resolved = await resolveSlug(slug, { serverUrl: input.serverUrl }).catch(() => undefined) if (!resolved || resolved.directory !== target) return false const current = sessionIDFromUrl(page.url()) if (input.sessionID && current !== input.sessionID) return false @@ -473,7 +476,7 @@ export async function waitSession(page: Page, input: { directory: string; sessio if (input.sessionID && (!state || state.sessionID !== input.sessionID)) return false if (!input.sessionID && state?.sessionID) return false if (state?.dir) { - const dir = await resolveDirectory(state.dir).catch(() => state.dir ?? "") + const dir = await resolveDirectory(state.dir, input.serverUrl).catch(() => state.dir ?? "") if (dir !== target) return false } @@ -489,9 +492,9 @@ export async function waitSession(page: Page, input: { directory: string; sessio return { directory: target, slug: base64Encode(target) } } -export async function waitSessionSaved(directory: string, sessionID: string, timeout = 30_000) { - const sdk = createSdk(directory) - const target = await resolveDirectory(directory) +export async function waitSessionSaved(directory: string, sessionID: string, timeout = 30_000, serverUrl?: string) { + const sdk = createSdk(directory, serverUrl) + const target = await resolveDirectory(directory, serverUrl) await expect .poll( @@ -501,7 +504,7 @@ export async function waitSessionSaved(directory: string, sessionID: string, tim .then((x) => x.data) .catch(() => undefined) if (!data?.directory) return "" - return resolveDirectory(data.directory).catch(() => data.directory) + return resolveDirectory(data.directory, serverUrl).catch(() => data.directory) }, { timeout }, ) @@ -666,8 +669,9 @@ export async function cleanupSession(input: { sessionID: string directory?: string sdk?: ReturnType + serverUrl?: string }) { - const sdk = input.sdk ?? (input.directory ? createSdk(input.directory) : undefined) + const sdk = input.sdk ?? (input.directory ? createSdk(input.directory, input.serverUrl) : undefined) if (!sdk) throw new Error("cleanupSession requires sdk or directory") await waitSessionIdle(sdk, input.sessionID, 5_000).catch(() => undefined) const current = await status(sdk, input.sessionID).catch(() => undefined) @@ -1019,3 +1023,13 @@ export async function openWorkspaceMenu(page: Page, workspaceSlug: string) { await expect(menu).toBeVisible() return menu } + +export async function assistantText(sdk: ReturnType, sessionID: string) { + const messages = await sdk.session.messages({ sessionID, limit: 50 }).then((r) => r.data ?? []) + return messages + .filter((m) => m.info.role === "assistant") + .flatMap((m) => m.parts) + .filter((p) => p.type === "text") + .map((p) => p.text) + .join("\n") +} diff --git a/packages/app/e2e/backend.ts b/packages/app/e2e/backend.ts new file mode 100644 index 0000000000..2acbe7179c --- /dev/null +++ b/packages/app/e2e/backend.ts @@ -0,0 +1,119 @@ +import { spawn } from "node:child_process" +import fs from "node:fs/promises" +import net from "node:net" +import os from "node:os" +import path from "node:path" +import { fileURLToPath } from "node:url" + +type Handle = { + url: string + stop: () => Promise +} + +function freePort() { + return new Promise((resolve, reject) => { + const server = net.createServer() + server.once("error", reject) + server.listen(0, () => { + const address = server.address() + if (!address || typeof address === "string") { + server.close(() => reject(new Error("Failed to acquire a free port"))) + return + } + server.close((err) => { + if (err) reject(err) + else resolve(address.port) + }) + }) + }) +} + +async function waitForHealth(url: string, probe = "/global/health") { + const end = Date.now() + 120_000 + let last = "" + while (Date.now() < end) { + try { + const res = await fetch(`${url}${probe}`) + if (res.ok) return + last = `status ${res.status}` + } catch (err) { + last = err instanceof Error ? err.message : String(err) + } + await new Promise((resolve) => setTimeout(resolve, 250)) + } + throw new Error(`Timed out waiting for backend health at ${url}${probe}${last ? ` (${last})` : ""}`) +} + +const LOG_CAP = 100 + +function cap(input: string[]) { + if (input.length > LOG_CAP) input.splice(0, input.length - LOG_CAP) +} + +function tail(input: string[]) { + return input.slice(-40).join("") +} + +export async function startBackend(label: string): Promise { + const port = await freePort() + const sandbox = await fs.mkdtemp(path.join(os.tmpdir(), `opencode-e2e-${label}-`)) + const appDir = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..") + const repoDir = path.resolve(appDir, "../..") + const opencodeDir = path.join(repoDir, "packages", "opencode") + const env = { + ...process.env, + OPENCODE_DISABLE_SHARE: process.env.OPENCODE_DISABLE_SHARE ?? "true", + OPENCODE_DISABLE_LSP_DOWNLOAD: "true", + OPENCODE_DISABLE_DEFAULT_PLUGINS: "true", + OPENCODE_EXPERIMENTAL_DISABLE_FILEWATCHER: "true", + OPENCODE_TEST_HOME: path.join(sandbox, "home"), + XDG_DATA_HOME: path.join(sandbox, "share"), + XDG_CACHE_HOME: path.join(sandbox, "cache"), + XDG_CONFIG_HOME: path.join(sandbox, "config"), + XDG_STATE_HOME: path.join(sandbox, "state"), + OPENCODE_CLIENT: "app", + OPENCODE_STRICT_CONFIG_DEPS: "true", + } satisfies Record + const out: string[] = [] + const err: string[] = [] + const proc = spawn( + "bun", + ["run", "--conditions=browser", "./src/index.ts", "serve", "--port", String(port), "--hostname", "127.0.0.1"], + { + cwd: opencodeDir, + env, + stdio: ["ignore", "pipe", "pipe"], + }, + ) + proc.stdout?.on("data", (chunk) => { out.push(String(chunk)); cap(out) }) + proc.stderr?.on("data", (chunk) => { err.push(String(chunk)); cap(err) }) + + const url = `http://127.0.0.1:${port}` + try { + await waitForHealth(url) + } catch (error) { + proc.kill("SIGTERM") + await fs.rm(sandbox, { recursive: true, force: true }).catch(() => undefined) + throw new Error( + [ + `Failed to start isolated e2e backend for ${label}`, + error instanceof Error ? error.message : String(error), + tail(out), + tail(err), + ] + .filter(Boolean) + .join("\n"), + ) + } + + return { + url, + async stop() { + if (proc.exitCode === null) { + proc.kill("SIGTERM") + await new Promise((resolve) => proc.once("exit", () => resolve(undefined))).catch(() => undefined) + } + await fs.rm(sandbox, { recursive: true, force: true }).catch(() => undefined) + }, + } +} diff --git a/packages/app/e2e/fixtures.ts b/packages/app/e2e/fixtures.ts index 7fc4cda057..c94c93992a 100644 --- a/packages/app/e2e/fixtures.ts +++ b/packages/app/e2e/fixtures.ts @@ -3,6 +3,7 @@ import { ManagedRuntime } from "effect" import type { E2EWindow } from "../src/testing/terminal" import type { Item, Reply, Usage } from "../../opencode/test/lib/llm-server" import { TestLLMServer } from "../../opencode/test/lib/llm-server" +import { startBackend } from "./backend" import { healthPhase, cleanupSession, @@ -19,6 +20,20 @@ import { createSdk, dirSlug, getWorktree, sessionPath } from "./utils" type LLMFixture = { url: string push: (...input: (Item | Reply)[]) => Promise + pushMatch: ( + match: (hit: { url: URL; body: Record }) => boolean, + ...input: (Item | Reply)[] + ) => Promise + textMatch: ( + match: (hit: { url: URL; body: Record }) => boolean, + value: string, + opts?: { usage?: Usage }, + ) => Promise + toolMatch: ( + match: (hit: { url: URL; body: Record }) => boolean, + name: string, + input: unknown, + ) => Promise text: (value: string, opts?: { usage?: Usage }) => Promise tool: (name: string, input: unknown) => Promise toolHang: (name: string, input: unknown) => Promise @@ -46,32 +61,54 @@ const seedModel = (() => { } })() +type ProjectHandle = { + directory: string + slug: string + gotoSession: (sessionID?: string) => Promise + trackSession: (sessionID: string, directory?: string) => void + trackDirectory: (directory: string) => void + sdk: ReturnType +} + +type ProjectOptions = { + extra?: string[] + model?: { providerID: string; modelID: string } + setup?: (directory: string) => Promise + beforeGoto?: (project: { directory: string; sdk: ReturnType }) => Promise +} + type TestFixtures = { llm: LLMFixture sdk: ReturnType gotoSession: (sessionID?: string) => Promise - withProject: ( - callback: (project: { - directory: string - slug: string - gotoSession: (sessionID?: string) => Promise - trackSession: (sessionID: string, directory?: string) => void - trackDirectory: (directory: string) => void - }) => Promise, - options?: { - extra?: string[] - model?: { providerID: string; modelID: string } - setup?: (directory: string) => Promise - }, - ) => Promise + withProject: (callback: (project: ProjectHandle) => Promise, options?: ProjectOptions) => Promise + withBackendProject: (callback: (project: ProjectHandle) => Promise, options?: ProjectOptions) => Promise } type WorkerFixtures = { + backend: { + url: string + sdk: (directory?: string) => ReturnType + } directory: string slug: string } export const test = base.extend({ + backend: [ + async ({}, use, workerInfo) => { + const handle = await startBackend(`w${workerInfo.workerIndex}`) + try { + await use({ + url: handle.url, + sdk: (directory?: string) => createSdk(directory, handle.url), + }) + } finally { + await handle.stop() + } + }, + { scope: "worker" }, + ], llm: async ({}, use) => { const rt = ManagedRuntime.make(TestLLMServer.layer) try { @@ -79,6 +116,9 @@ export const test = base.extend({ await use({ url: svc.url, push: (...input) => rt.runPromise(svc.push(...input)), + pushMatch: (match, ...input) => rt.runPromise(svc.pushMatch(match, ...input)), + textMatch: (match, value, opts) => rt.runPromise(svc.textMatch(match, value, opts)), + toolMatch: (match, name, input) => rt.runPromise(svc.toolMatch(match, name, input)), text: (value, opts) => rt.runPromise(svc.text(value, opts)), tool: (name, input) => rt.runPromise(svc.tool(name, input)), toolHang: (name, input) => rt.runPromise(svc.toolHang(name, input)), @@ -146,51 +186,78 @@ export const test = base.extend({ await use(gotoSession) }, withProject: async ({ page }, use) => { - await use(async (callback, options) => { - const root = await createTestProject() - const sessions = new Map() - const dirs = new Set() - await options?.setup?.(root) - await seedStorage(page, { directory: root, extra: options?.extra, model: options?.model }) - - const gotoSession = async (sessionID?: string) => { - await page.goto(sessionPath(root, sessionID)) - await waitSession(page, { directory: root, sessionID }) - const current = sessionIDFromUrl(page.url()) - if (current) trackSession(current) - } - - const trackSession = (sessionID: string, directory?: string) => { - sessions.set(sessionID, directory ?? root) - } - - const trackDirectory = (directory: string) => { - if (directory !== root) dirs.add(directory) - } - - try { - await gotoSession() - const slug = await waitSlug(page) - return await callback({ directory: root, slug, gotoSession, trackSession, trackDirectory }) - } finally { - setHealthPhase(page, "cleanup") - await Promise.allSettled( - Array.from(sessions, ([sessionID, directory]) => cleanupSession({ sessionID, directory })), - ) - await Promise.allSettled(Array.from(dirs, (directory) => cleanupTestProject(directory))) - await cleanupTestProject(root) - setHealthPhase(page, "test") - } - }) + await use((callback, options) => + runProject(page, callback, options), + ) + }, + withBackendProject: async ({ page, backend }, use) => { + await use((callback, options) => + runProject(page, callback, { ...options, serverUrl: backend.url, sdk: backend.sdk }), + ) }, }) +async function runProject( + page: Page, + callback: (project: ProjectHandle) => Promise, + options?: ProjectOptions & { + serverUrl?: string + sdk?: (directory?: string) => ReturnType + }, +) { + const url = options?.serverUrl + const root = await createTestProject(url ? { serverUrl: url } : undefined) + const sdk = options?.sdk?.(root) ?? createSdk(root, url) + const sessions = new Map() + const dirs = new Set() + await options?.setup?.(root) + await seedStorage(page, { + directory: root, + extra: options?.extra, + model: options?.model, + serverUrl: url, + }) + + const gotoSession = async (sessionID?: string) => { + await page.goto(sessionPath(root, sessionID)) + await waitSession(page, { directory: root, sessionID, serverUrl: url }) + const current = sessionIDFromUrl(page.url()) + if (current) trackSession(current) + } + + const trackSession = (sessionID: string, directory?: string) => { + sessions.set(sessionID, directory ?? root) + } + + const trackDirectory = (directory: string) => { + if (directory !== root) dirs.add(directory) + } + + try { + await options?.beforeGoto?.({ directory: root, sdk }) + await gotoSession() + const slug = await waitSlug(page) + return await callback({ directory: root, slug, gotoSession, trackSession, trackDirectory, sdk }) + } finally { + setHealthPhase(page, "cleanup") + await Promise.allSettled( + Array.from(sessions, ([sessionID, directory]) => + cleanupSession({ sessionID, directory, serverUrl: url }), + ), + ) + await Promise.allSettled(Array.from(dirs, (directory) => cleanupTestProject(directory))) + await cleanupTestProject(root) + setHealthPhase(page, "test") + } +} + async function seedStorage( page: Page, input: { directory: string extra?: string[] model?: { providerID: string; modelID: string } + serverUrl?: string }, ) { await seedProjects(page, input) diff --git a/packages/app/e2e/prompt/mock.ts b/packages/app/e2e/prompt/mock.ts new file mode 100644 index 0000000000..eb40a70cba --- /dev/null +++ b/packages/app/e2e/prompt/mock.ts @@ -0,0 +1,46 @@ +import { createSdk } from "../utils" + +export const openaiModel = { providerID: "openai", modelID: "gpt-5.3-chat-latest" } + +type Hit = { body: Record } + +export function bodyText(hit: Hit) { + return JSON.stringify(hit.body) +} + +export function titleMatch(hit: Hit) { + return bodyText(hit).includes("Generate a title for this conversation") +} + +export function promptMatch(token: string) { + return (hit: Hit) => bodyText(hit).includes(token) +} + +export async function withMockOpenAI(input: { serverUrl: string; llmUrl: string; fn: () => Promise }) { + const sdk = createSdk(undefined, input.serverUrl) + const prev = await sdk.global.config.get().then((res) => res.data ?? {}) + + try { + await sdk.global.config.update({ + config: { + ...prev, + model: `${openaiModel.providerID}/${openaiModel.modelID}`, + enabled_providers: ["openai"], + provider: { + ...prev.provider, + openai: { + ...prev.provider?.openai, + options: { + ...prev.provider?.openai?.options, + apiKey: "test-key", + baseURL: input.llmUrl, + }, + }, + }, + }, + }) + return await input.fn() + } finally { + await sdk.global.config.update({ config: prev }) + } +} diff --git a/packages/app/e2e/prompt/prompt-async.spec.ts b/packages/app/e2e/prompt/prompt-async.spec.ts index 51fbc3e4ae..99fa5f2d45 100644 --- a/packages/app/e2e/prompt/prompt-async.spec.ts +++ b/packages/app/e2e/prompt/prompt-async.spec.ts @@ -1,47 +1,53 @@ import { test, expect } from "../fixtures" import { promptSelector } from "../selectors" -import { cleanupSession, sessionIDFromUrl, withSession } from "../actions" +import { assistantText, sessionIDFromUrl, withSession } from "../actions" +import { openaiModel, promptMatch, withMockOpenAI } from "./mock" const text = (value: string | null) => (value ?? "").replace(/\u200B/g, "").trim() // Regression test for Issue #12453: the synchronous POST /message endpoint holds // the connection open while the agent works, causing "Failed to fetch" over // VPN/Tailscale. The fix switches to POST /prompt_async which returns immediately. -test("prompt succeeds when sync message endpoint is unreachable", async ({ page, sdk, gotoSession }) => { +test("prompt succeeds when sync message endpoint is unreachable", async ({ + page, + llm, + backend, + withBackendProject, +}) => { test.setTimeout(120_000) // Simulate Tailscale/VPN killing the long-lived sync connection await page.route("**/session/*/message", (route) => route.abort("connectionfailed")) - await gotoSession() + await withMockOpenAI({ + serverUrl: backend.url, + llmUrl: llm.url, + fn: async () => { + const token = `E2E_ASYNC_${Date.now()}` + await llm.textMatch(promptMatch(token), token) - const token = `E2E_ASYNC_${Date.now()}` - await page.locator(promptSelector).click() - await page.keyboard.type(`Reply with exactly: ${token}`) - await page.keyboard.press("Enter") + await withBackendProject( + async (project) => { + await page.locator(promptSelector).click() + await page.keyboard.type(`Reply with exactly: ${token}`) + await page.keyboard.press("Enter") - await expect(page).toHaveURL(/\/session\/[^/?#]+/, { timeout: 30_000 }) - const sessionID = sessionIDFromUrl(page.url())! + await expect(page).toHaveURL(/\/session\/[^/?#]+/, { timeout: 30_000 }) + const sessionID = sessionIDFromUrl(page.url())! + project.trackSession(sessionID) - try { - // Agent response arrives via SSE despite sync endpoint being dead - await expect - .poll( - async () => { - const messages = await sdk.session.messages({ sessionID, limit: 50 }).then((r) => r.data ?? []) - return messages - .filter((m) => m.info.role === "assistant") - .flatMap((m) => m.parts) - .filter((p) => p.type === "text") - .map((p) => p.text) - .join("\n") + await expect.poll(() => llm.calls()).toBeGreaterThanOrEqual(1) + + await expect + .poll(() => assistantText(project.sdk, sessionID), { timeout: 90_000 }) + .toContain(token) + }, + { + model: openaiModel, }, - { timeout: 90_000 }, ) - .toContain(token) - } finally { - await cleanupSession({ sdk, sessionID }) - } + }, + }) }) test("failed prompt send restores the composer input", async ({ page, sdk, gotoSession }) => { diff --git a/packages/app/e2e/prompt/prompt.spec.ts b/packages/app/e2e/prompt/prompt.spec.ts index 1acf17f5bf..e4545e97a9 100644 --- a/packages/app/e2e/prompt/prompt.spec.ts +++ b/packages/app/e2e/prompt/prompt.spec.ts @@ -1,44 +1,9 @@ -import fs from "node:fs/promises" -import path from "node:path" import { test, expect } from "../fixtures" import { promptSelector } from "../selectors" -import { sessionIDFromUrl } from "../actions" -import { createSdk } from "../utils" +import { assistantText, sessionIDFromUrl } from "../actions" +import { openaiModel, promptMatch, titleMatch, withMockOpenAI } from "./mock" -async function config(dir: string, url: string) { - await fs.writeFile( - path.join(dir, "opencode.json"), - JSON.stringify({ - $schema: "https://opencode.ai/config.json", - enabled_providers: ["e2e-llm"], - provider: { - "e2e-llm": { - name: "E2E LLM", - npm: "@ai-sdk/openai-compatible", - env: [], - models: { - "test-model": { - name: "Test Model", - tool_call: true, - limit: { context: 128000, output: 32000 }, - }, - }, - options: { - apiKey: "test-key", - baseURL: url, - }, - }, - }, - agent: { - build: { - model: "e2e-llm/test-model", - }, - }, - }), - ) -} - -test("can send a prompt and receive a reply", async ({ page, llm, withProject }) => { +test("can send a prompt and receive a reply", async ({ page, llm, backend, withBackendProject }) => { test.setTimeout(120_000) const pageErrors: string[] = [] @@ -48,48 +13,43 @@ test("can send a prompt and receive a reply", async ({ page, llm, withProject }) page.on("pageerror", onPageError) try { - await withProject( - async (project) => { - const sdk = createSdk(project.directory) + await withMockOpenAI({ + serverUrl: backend.url, + llmUrl: llm.url, + fn: async () => { const token = `E2E_OK_${Date.now()}` - await llm.text(token) - await project.gotoSession() + await llm.textMatch(titleMatch, "E2E Title") + await llm.textMatch(promptMatch(token), token) - const prompt = page.locator(promptSelector) - await prompt.click() - await page.keyboard.type(`Reply with exactly: ${token}`) - await page.keyboard.press("Enter") + await withBackendProject( + async (project) => { + const prompt = page.locator(promptSelector) + await prompt.click() + await page.keyboard.type(`Reply with exactly: ${token}`) + await page.keyboard.press("Enter") - await expect(page).toHaveURL(/\/session\/[^/?#]+/, { timeout: 30_000 }) + await expect(page).toHaveURL(/\/session\/[^/?#]+/, { timeout: 30_000 }) - const sessionID = (() => { - const id = sessionIDFromUrl(page.url()) - if (!id) throw new Error(`Failed to parse session id from url: ${page.url()}`) - return id - })() - project.trackSession(sessionID) + const sessionID = (() => { + const id = sessionIDFromUrl(page.url()) + if (!id) throw new Error(`Failed to parse session id from url: ${page.url()}`) + return id + })() + project.trackSession(sessionID) - await expect - .poll( - async () => { - const messages = await sdk.session.messages({ sessionID, limit: 50 }).then((r) => r.data ?? []) - return messages - .filter((m) => m.info.role === "assistant") - .flatMap((m) => m.parts) - .filter((p) => p.type === "text") - .map((p) => p.text) - .join("\n") - }, - { timeout: 30_000 }, - ) - .toContain(token) + await expect.poll(() => llm.calls()).toBeGreaterThanOrEqual(1) + + await expect + .poll(() => assistantText(project.sdk, sessionID), { timeout: 30_000 }) + .toContain(token) + }, + { + model: openaiModel, + }, + ) }, - { - model: { providerID: "e2e-llm", modelID: "test-model" }, - setup: (dir) => config(dir, llm.url), - }, - ) + }) } finally { page.off("pageerror", onPageError) } diff --git a/packages/app/e2e/utils.ts b/packages/app/e2e/utils.ts index f07a8d3f11..17a8785664 100644 --- a/packages/app/e2e/utils.ts +++ b/packages/app/e2e/utils.ts @@ -26,21 +26,21 @@ export const serverNamePattern = new RegExp(`(?:${serverNames.map(escape).join(" export const modKey = process.platform === "darwin" ? "Meta" : "Control" export const terminalToggleKey = "Control+Backquote" -export function createSdk(directory?: string) { - return createOpencodeClient({ baseUrl: serverUrl, directory, throwOnError: true }) +export function createSdk(directory?: string, baseUrl = serverUrl) { + return createOpencodeClient({ baseUrl, directory, throwOnError: true }) } -export async function resolveDirectory(directory: string) { - return createSdk(directory) +export async function resolveDirectory(directory: string, baseUrl = serverUrl) { + return createSdk(directory, baseUrl) .path.get() .then((x) => x.data?.directory ?? directory) } -export async function getWorktree() { - const sdk = createSdk() +export async function getWorktree(baseUrl = serverUrl) { + const sdk = createSdk(undefined, baseUrl) const result = await sdk.path.get() const data = result.data - if (!data?.worktree) throw new Error(`Failed to resolve a worktree from ${serverUrl}/path`) + if (!data?.worktree) throw new Error(`Failed to resolve a worktree from ${baseUrl}/path`) return data.worktree } diff --git a/packages/opencode/test/lib/llm-server.ts b/packages/opencode/test/lib/llm-server.ts index fb84f1175a..1c624cd0d5 100644 --- a/packages/opencode/test/lib/llm-server.ts +++ b/packages/opencode/test/lib/llm-server.ts @@ -20,6 +20,13 @@ type Hit = { body: Record } +type Match = (hit: Hit) => boolean + +type Queue = { + item: Item + match?: Match +} + type Wait = { count: number ready: Deferred.Deferred @@ -420,7 +427,7 @@ const reset = Effect.fn("TestLLMServer.reset")(function* (item: Sse) { for (const part of item.tail) res.write(line(part)) res.destroy(new Error("connection reset")) }) - yield* Effect.never + return yield* Effect.never }) function fail(item: HttpError) { @@ -581,6 +588,9 @@ namespace TestLLMServer { export interface Service { readonly url: string readonly push: (...input: (Item | Reply)[]) => Effect.Effect + readonly pushMatch: (match: Match, ...input: (Item | Reply)[]) => Effect.Effect + readonly textMatch: (match: Match, value: string, opts?: { usage?: Usage }) => Effect.Effect + readonly toolMatch: (match: Match, name: string, input: unknown) => Effect.Effect readonly text: (value: string, opts?: { usage?: Usage }) => Effect.Effect readonly tool: (name: string, input: unknown) => Effect.Effect readonly toolHang: (name: string, input: unknown) => Effect.Effect @@ -605,11 +615,15 @@ export class TestLLMServer extends ServiceMap.Service { - list = [...list, ...input.map(item)] + list = [...list, ...input.map((value) => ({ item: item(value) }))] + } + + const queueMatch = (match: Match, ...input: (Item | Reply)[]) => { + list = [...list, ...input.map((value) => ({ item: item(value), match }))] } const notify = Effect.fnUntraced(function* () { @@ -619,19 +633,21 @@ export class TestLLMServer extends ServiceMap.Service Deferred.succeed(item.ready, void 0)) }) - const pull = () => { - const first = list[0] - if (!first) return - list = list.slice(1) - return first + const pull = (hit: Hit) => { + const index = list.findIndex((entry) => !entry.match || entry.match(hit)) + if (index === -1) return + const first = list[index] + list = [...list.slice(0, index), ...list.slice(index + 1)] + return first.item } const handle = Effect.fn("TestLLMServer.handle")(function* (mode: "chat" | "responses") { const req = yield* HttpServerRequest.HttpServerRequest - const next = pull() - if (!next) return HttpServerResponse.text("unexpected request", { status: 500 }) const body = yield* req.json.pipe(Effect.orElseSucceed(() => ({}))) - hits = [...hits, hit(req.originalUrl, body)] + const current = hit(req.originalUrl, body) + const next = pull(current) + if (!next) return HttpServerResponse.text("unexpected request", { status: 500 }) + hits = [...hits, current] yield* notify() if (next.type !== "sse") return fail(next) if (mode === "responses") return send(responses(next, modelFrom(body))) @@ -655,6 +671,21 @@ export class TestLLMServer extends ServiceMap.Service