import { test as base, expect, type Page } from "@playwright/test" import { ManagedRuntime } from "effect" import type { E2EWindow } from "../src/testing/terminal" import type { Item, Reply, Usage } from "../../opencode/test/lib/llm-server" import { TestLLMServer } from "../../opencode/test/lib/llm-server" import { startBackend } from "./backend" import { healthPhase, cleanupSession, cleanupTestProject, createTestProject, setHealthPhase, sessionIDFromUrl, waitSession, waitSessionIdle, waitSessionSaved, waitSlug, } from "./actions" import { promptSelector } from "./selectors" import { createSdk, dirSlug, getWorktree, serverUrl, sessionPath } from "./utils" type LLMFixture = { url: string push: (...input: (Item | Reply)[]) => Promise pushMatch: ( match: (hit: { url: URL; body: Record }) => boolean, ...input: (Item | Reply)[] ) => Promise textMatch: ( match: (hit: { url: URL; body: Record }) => boolean, value: string, opts?: { usage?: Usage }, ) => Promise toolMatch: ( match: (hit: { url: URL; body: Record }) => boolean, name: string, input: unknown, ) => Promise text: (value: string, opts?: { usage?: Usage }) => Promise tool: (name: string, input: unknown) => Promise toolHang: (name: string, input: unknown) => Promise reason: (value: string, opts?: { text?: string; usage?: Usage }) => Promise fail: (message?: unknown) => Promise error: (status: number, body: unknown) => Promise hang: () => Promise hold: (value: string, wait: PromiseLike) => Promise hits: () => Promise }>> calls: () => Promise wait: (count: number) => Promise inputs: () => Promise[]> pending: () => Promise misses: () => Promise }>> } type LLMWorker = LLMFixture & { reset: () => Promise } type AssistantFixture = { reply: LLMFixture["text"] tool: LLMFixture["tool"] toolHang: LLMFixture["toolHang"] reason: LLMFixture["reason"] fail: LLMFixture["fail"] error: LLMFixture["error"] hang: LLMFixture["hang"] hold: LLMFixture["hold"] calls: LLMFixture["calls"] pending: LLMFixture["pending"] } export const settingsKey = "settings.v3" const seedModel = (() => { const [providerID = "opencode", modelID = "big-pickle"] = ( process.env.OPENCODE_E2E_MODEL ?? "opencode/big-pickle" ).split("/") return { providerID: providerID || "opencode", modelID: modelID || "big-pickle", } })() function clean(value: string | null) { return (value ?? "").replace(/\u200B/g, "").trim() } async function visit(page: Page, url: string) { let err: unknown for (const _ of [0, 1, 2]) { try { await page.goto(url) return } catch (cause) { err = cause if (!String(cause).includes("ERR_CONNECTION_REFUSED")) throw cause await new Promise((resolve) => setTimeout(resolve, 300)) } } throw err } async function promptSend(page: Page) { return page .evaluate(() => { const win = window as E2EWindow const sent = win.__opencode_e2e?.prompt?.sent return { started: sent?.started ?? 0, count: sent?.count ?? 0, sessionID: sent?.sessionID, directory: sent?.directory, } }) .catch(() => ({ started: 0, count: 0, sessionID: undefined, directory: undefined })) } type ProjectHandle = { directory: string slug: string gotoSession: (sessionID?: string) => Promise trackSession: (sessionID: string, directory?: string) => void trackDirectory: (directory: string) => void sdk: ReturnType } type ProjectOptions = { extra?: string[] model?: { providerID: string; modelID: string } setup?: (directory: string) => Promise beforeGoto?: (project: { directory: string; sdk: ReturnType }) => Promise } type ProjectFixture = ProjectHandle & { open: (options?: ProjectOptions) => Promise prompt: (text: string) => Promise user: (text: string) => Promise shell: (cmd: string) => Promise } type TestFixtures = { llm: LLMFixture assistant: AssistantFixture project: ProjectFixture sdk: ReturnType gotoSession: (sessionID?: string) => Promise } type WorkerFixtures = { _llm: LLMWorker backend: { url: string sdk: (directory?: string) => ReturnType } directory: string slug: string } export const test = base.extend({ _llm: [ async ({}, use) => { const rt = ManagedRuntime.make(TestLLMServer.layer) try { const svc = await rt.runPromise(TestLLMServer.asEffect()) await use({ url: svc.url, push: (...input) => rt.runPromise(svc.push(...input)), pushMatch: (match, ...input) => rt.runPromise(svc.pushMatch(match, ...input)), textMatch: (match, value, opts) => rt.runPromise(svc.textMatch(match, value, opts)), toolMatch: (match, name, input) => rt.runPromise(svc.toolMatch(match, name, input)), text: (value, opts) => rt.runPromise(svc.text(value, opts)), tool: (name, input) => rt.runPromise(svc.tool(name, input)), toolHang: (name, input) => rt.runPromise(svc.toolHang(name, input)), reason: (value, opts) => rt.runPromise(svc.reason(value, opts)), fail: (message) => rt.runPromise(svc.fail(message)), error: (status, body) => rt.runPromise(svc.error(status, body)), hang: () => rt.runPromise(svc.hang), hold: (value, wait) => rt.runPromise(svc.hold(value, wait)), reset: () => rt.runPromise(svc.reset), hits: () => rt.runPromise(svc.hits), calls: () => rt.runPromise(svc.calls), wait: (count) => rt.runPromise(svc.wait(count)), inputs: () => rt.runPromise(svc.inputs), pending: () => rt.runPromise(svc.pending), misses: () => rt.runPromise(svc.misses), }) } finally { await rt.dispose() } }, { scope: "worker" }, ], backend: [ async ({ _llm }, use, workerInfo) => { const handle = await startBackend(`w${workerInfo.workerIndex}`, { llmUrl: _llm.url }) try { await use({ url: handle.url, sdk: (directory?: string) => createSdk(directory, handle.url), }) } finally { await handle.stop() } }, { scope: "worker" }, ], llm: async ({ _llm }, use) => { await _llm.reset() await use({ url: _llm.url, push: _llm.push, pushMatch: _llm.pushMatch, textMatch: _llm.textMatch, toolMatch: _llm.toolMatch, text: _llm.text, tool: _llm.tool, toolHang: _llm.toolHang, reason: _llm.reason, fail: _llm.fail, error: _llm.error, hang: _llm.hang, hold: _llm.hold, hits: _llm.hits, calls: _llm.calls, wait: _llm.wait, inputs: _llm.inputs, pending: _llm.pending, misses: _llm.misses, }) const pending = await _llm.pending() if (pending > 0) { throw new Error(`TestLLMServer still has ${pending} queued response(s) after the test finished`) } }, assistant: async ({ llm }, use) => { await use({ reply: llm.text, tool: llm.tool, toolHang: llm.toolHang, reason: llm.reason, fail: llm.fail, error: llm.error, hang: llm.hang, hold: llm.hold, calls: llm.calls, pending: llm.pending, }) }, page: async ({ page }, use) => { let boundary: string | undefined setHealthPhase(page, "test") const consoleHandler = (msg: { text(): string }) => { const text = msg.text() if (!text.includes("[e2e:error-boundary]")) return if (healthPhase(page) === "cleanup") { console.warn(`[e2e:error-boundary][cleanup-warning]\n${text}`) return } boundary ||= text console.log(text) } const pageErrorHandler = (err: Error) => { console.log(`[e2e:pageerror] ${err.stack || err.message}`) } page.on("console", consoleHandler) page.on("pageerror", pageErrorHandler) await use(page) page.off("console", consoleHandler) page.off("pageerror", pageErrorHandler) if (boundary) throw new Error(boundary) }, directory: [ async ({ backend }, use) => { await use(await getWorktree(backend.url)) }, { scope: "worker" }, ], slug: [ async ({ directory }, use) => { await use(dirSlug(directory)) }, { scope: "worker" }, ], sdk: async ({ directory, backend }, use) => { await use(backend.sdk(directory)) }, gotoSession: async ({ page, directory, backend }, use) => { await seedStorage(page, { directory, serverUrl: backend.url }) const gotoSession = async (sessionID?: string) => { await visit(page, sessionPath(directory, sessionID)) await waitSession(page, { directory, sessionID, serverUrl: backend.url, allowAnySession: !sessionID, }) } await use(gotoSession) }, project: async ({ page, llm, backend }, use) => { const item = makeProject(page, llm, backend) try { await use(item.project) } finally { await item.cleanup() } }, }) function makeProject( page: Page, llm: LLMFixture, backend: { url: string; sdk: (directory?: string) => ReturnType }, ) { let state: | { directory: string slug: string sdk: ReturnType sessions: Map dirs: Set } | undefined const need = () => { if (state) return state throw new Error("project.open() must be called first") } const trackSession = (sessionID: string, directory?: string) => { const cur = need() cur.sessions.set(sessionID, directory ?? cur.directory) } const trackDirectory = (directory: string) => { const cur = need() if (directory !== cur.directory) cur.dirs.add(directory) } const gotoSession = async (sessionID?: string) => { const cur = need() await visit(page, sessionPath(cur.directory, sessionID)) await waitSession(page, { directory: cur.directory, sessionID, serverUrl: backend.url, allowAnySession: !sessionID, }) const current = sessionIDFromUrl(page.url()) if (current) trackSession(current) } const open = async (options?: ProjectOptions) => { if (state) return const directory = await createTestProject({ serverUrl: backend.url }) const sdk = backend.sdk(directory) await options?.setup?.(directory) await seedStorage(page, { directory, extra: options?.extra, model: options?.model, serverUrl: backend.url, }) state = { directory, slug: "", sdk, sessions: new Map(), dirs: new Set(), } await options?.beforeGoto?.({ directory, sdk }) await gotoSession() need().slug = await waitSlug(page) } const send = async (text: string, input: { noReply: boolean; shell: boolean }) => { if (input.noReply) { const cur = need() const state = await page.evaluate(() => { const model = (window as E2EWindow).__opencode_e2e?.model?.current if (!model) return null return { dir: model.dir, sessionID: model.sessionID, agent: model.agent, model: model.model ? { providerID: model.model.providerID, modelID: model.model.modelID } : undefined, variant: model.variant ?? undefined, } }) const dir = state?.dir ?? cur.directory const sdk = backend.sdk(dir) const sessionID = state?.sessionID ? state.sessionID : await sdk.session.create({ directory: dir, title: "E2E Session" }).then((res) => { if (!res.data?.id) throw new Error("Failed to create no-reply session") return res.data.id }) await sdk.session.prompt({ sessionID, agent: state?.agent, model: state?.model, variant: state?.variant, noReply: true, parts: [{ type: "text", text }], }) await visit(page, sessionPath(dir, sessionID)) const active = await waitSession(page, { directory: dir, sessionID, serverUrl: backend.url, }) trackSession(sessionID, active.directory) await waitSessionSaved(active.directory, sessionID, 90_000, backend.url) return sessionID } const prev = await promptSend(page) if (!input.noReply && !input.shell && (await llm.pending()) === 0) { await llm.text("ok") } const prompt = page.locator(promptSelector).first() const submit = async () => { await expect(prompt).toBeVisible() await prompt.click() if (input.shell) { await page.keyboard.type("!") await expect(prompt).toHaveAttribute("aria-label", /enter shell command/i) } await page.keyboard.type(text) await expect.poll(async () => clean(await prompt.textContent())).toBe(text) await page.keyboard.press("Enter") const started = await expect .poll(async () => (await promptSend(page)).started, { timeout: 5_000 }) .toBeGreaterThan(prev.started) .then(() => true) .catch(() => false) if (started) return const send = page.getByRole("button", { name: "Send" }).first() const enabled = await send .isEnabled() .then((x) => x) .catch(() => false) if (enabled) { await send.click() } else { await prompt.click() await page.keyboard.press("Enter") } await expect.poll(async () => (await promptSend(page)).started, { timeout: 5_000 }).toBeGreaterThan(prev.started) } await submit() let next: { sessionID: string; directory: string } | undefined await expect .poll( async () => { const sent = await promptSend(page) if (sent.count <= prev.count) return "" if (!sent.sessionID || !sent.directory) return "" next = { sessionID: sent.sessionID, directory: sent.directory } return sent.sessionID }, { timeout: 90_000 }, ) .not.toBe("") if (!next) throw new Error("Failed to observe prompt submission in e2e prompt probe") const active = await waitSession(page, { directory: next.directory, sessionID: next.sessionID, serverUrl: backend.url, }) trackSession(next.sessionID, active.directory) if (!input.shell) { await waitSessionSaved(active.directory, next.sessionID, 90_000, backend.url) } await waitSessionIdle(backend.sdk(active.directory), next.sessionID, 90_000).catch(() => undefined) return next.sessionID } const prompt = async (text: string) => { return send(text, { noReply: false, shell: false }) } const user = async (text: string) => { return send(text, { noReply: true, shell: false }) } const shell = async (cmd: string) => { return send(cmd, { noReply: false, shell: true }) } const cleanup = async () => { const cur = state if (!cur) return setHealthPhase(page, "cleanup") await Promise.allSettled( Array.from(cur.sessions, ([sessionID, directory]) => cleanupSession({ sessionID, directory, serverUrl: backend.url }), ), ) await Promise.allSettled(Array.from(cur.dirs, (directory) => cleanupTestProject(directory))) await cleanupTestProject(cur.directory) state = undefined setHealthPhase(page, "test") } return { project: { open, prompt, user, shell, gotoSession, trackSession, trackDirectory, get directory() { return need().directory }, get slug() { return need().slug }, get sdk() { return need().sdk }, }, cleanup, } } async function seedStorage( page: Page, input: { directory: string extra?: string[] model?: { providerID: string; modelID: string } serverUrl?: string }, ) { const origin = input.serverUrl ?? serverUrl await page.addInitScript( (args: { directory: string serverUrl: string extra: string[] model: { providerID: string; modelID: string } }) => { const key = "opencode.global.dat:server" const raw = localStorage.getItem(key) const parsed = (() => { if (!raw) return undefined try { return JSON.parse(raw) as unknown } catch { return undefined } })() const store = parsed && typeof parsed === "object" ? (parsed as Record) : {} const list = Array.isArray(store.list) ? store.list : [] const lastProject = store.lastProject && typeof store.lastProject === "object" ? store.lastProject : {} const projects = store.projects && typeof store.projects === "object" ? store.projects : {} const next = { ...(projects as Record) } const nextList = list.includes(args.serverUrl) ? list : [args.serverUrl, ...list] const add = (origin: string, directory: string) => { const current = next[origin] const items = Array.isArray(current) ? current : [] const existing = items.filter( (p): p is { worktree: string; expanded?: boolean } => !!p && typeof p === "object" && "worktree" in p && typeof (p as { worktree?: unknown }).worktree === "string", ) if (existing.some((p) => p.worktree === directory)) return next[origin] = [{ worktree: directory, expanded: true }, ...existing] } for (const directory of [args.directory, ...args.extra]) { add("local", directory) add(args.serverUrl, directory) } localStorage.setItem(key, JSON.stringify({ list: nextList, projects: next, lastProject })) localStorage.setItem("opencode.settings.dat:defaultServerUrl", args.serverUrl) const win = window as E2EWindow win.__opencode_e2e = { ...win.__opencode_e2e, model: { enabled: true }, prompt: { enabled: true }, terminal: { enabled: true, terminals: {} }, } localStorage.setItem("opencode.global.dat:model", JSON.stringify({ recent: [args.model], user: [], variant: {} })) }, { directory: input.directory, serverUrl: origin, extra: input.extra ?? [], model: input.model ?? seedModel }, ) } export { expect }