606 lines
18 KiB
TypeScript
606 lines
18 KiB
TypeScript
import { test as base, expect, type Page } from "@playwright/test"
|
|
import { ManagedRuntime } from "effect"
|
|
import type { E2EWindow } from "../src/testing/terminal"
|
|
import type { Item, Reply, Usage } from "../../opencode/test/lib/llm-server"
|
|
import { TestLLMServer } from "../../opencode/test/lib/llm-server"
|
|
import { startBackend } from "./backend"
|
|
import {
|
|
healthPhase,
|
|
cleanupSession,
|
|
cleanupTestProject,
|
|
createTestProject,
|
|
setHealthPhase,
|
|
seedProjects,
|
|
sessionIDFromUrl,
|
|
waitSession,
|
|
waitSessionIdle,
|
|
waitSessionSaved,
|
|
waitSlug,
|
|
withNoReplyPrompt,
|
|
} from "./actions"
|
|
import { openaiModel, withMockOpenAI } from "./prompt/mock"
|
|
import { promptSelector } from "./selectors"
|
|
import { createSdk, dirSlug, getWorktree, sessionPath } from "./utils"
|
|
|
|
type LLMFixture = {
|
|
url: string
|
|
push: (...input: (Item | Reply)[]) => Promise<void>
|
|
pushMatch: (
|
|
match: (hit: { url: URL; body: Record<string, unknown> }) => boolean,
|
|
...input: (Item | Reply)[]
|
|
) => Promise<void>
|
|
textMatch: (
|
|
match: (hit: { url: URL; body: Record<string, unknown> }) => boolean,
|
|
value: string,
|
|
opts?: { usage?: Usage },
|
|
) => Promise<void>
|
|
toolMatch: (
|
|
match: (hit: { url: URL; body: Record<string, unknown> }) => boolean,
|
|
name: string,
|
|
input: unknown,
|
|
) => Promise<void>
|
|
text: (value: string, opts?: { usage?: Usage }) => Promise<void>
|
|
tool: (name: string, input: unknown) => Promise<void>
|
|
toolHang: (name: string, input: unknown) => Promise<void>
|
|
reason: (value: string, opts?: { text?: string; usage?: Usage }) => Promise<void>
|
|
fail: (message?: unknown) => Promise<void>
|
|
error: (status: number, body: unknown) => Promise<void>
|
|
hang: () => Promise<void>
|
|
hold: (value: string, wait: PromiseLike<unknown>) => Promise<void>
|
|
hits: () => Promise<Array<{ url: URL; body: Record<string, unknown> }>>
|
|
calls: () => Promise<number>
|
|
wait: (count: number) => Promise<void>
|
|
inputs: () => Promise<Record<string, unknown>[]>
|
|
pending: () => Promise<number>
|
|
misses: () => Promise<Array<{ url: URL; body: Record<string, unknown> }>>
|
|
}
|
|
|
|
type LLMWorker = LLMFixture & {
|
|
reset: () => Promise<void>
|
|
}
|
|
|
|
type AssistantFixture = {
|
|
reply: (value: string, opts?: { usage?: Usage }) => Promise<void>
|
|
tool: (name: string, input: unknown) => Promise<void>
|
|
toolHang: (name: string, input: unknown) => Promise<void>
|
|
reason: (value: string, opts?: { text?: string; usage?: Usage }) => Promise<void>
|
|
fail: (message?: unknown) => Promise<void>
|
|
error: (status: number, body: unknown) => Promise<void>
|
|
hang: () => Promise<void>
|
|
hold: (value: string, wait: PromiseLike<unknown>) => Promise<void>
|
|
calls: () => Promise<number>
|
|
pending: () => Promise<number>
|
|
}
|
|
|
|
export const settingsKey = "settings.v3"
|
|
|
|
const seedModel = (() => {
|
|
const [providerID = "opencode", modelID = "big-pickle"] = (
|
|
process.env.OPENCODE_E2E_MODEL ?? "opencode/big-pickle"
|
|
).split("/")
|
|
return {
|
|
providerID: providerID || "opencode",
|
|
modelID: modelID || "big-pickle",
|
|
}
|
|
})()
|
|
|
|
function clean(value: string | null) {
|
|
return (value ?? "").replace(/\u200B/g, "").trim()
|
|
}
|
|
|
|
async function visit(page: Page, url: string) {
|
|
let err: unknown
|
|
for (const _ of [0, 1, 2]) {
|
|
try {
|
|
await page.goto(url)
|
|
return
|
|
} catch (cause) {
|
|
err = cause
|
|
if (!String(cause).includes("ERR_CONNECTION_REFUSED")) throw cause
|
|
await new Promise((resolve) => setTimeout(resolve, 300))
|
|
}
|
|
}
|
|
throw err
|
|
}
|
|
|
|
async function promptSend(page: Page) {
|
|
return page
|
|
.evaluate(() => {
|
|
const win = window as E2EWindow
|
|
const sent = win.__opencode_e2e?.prompt?.sent
|
|
return {
|
|
started: sent?.started ?? 0,
|
|
count: sent?.count ?? 0,
|
|
sessionID: sent?.sessionID,
|
|
directory: sent?.directory,
|
|
}
|
|
})
|
|
.catch(() => ({ started: 0, count: 0, sessionID: undefined, directory: undefined }))
|
|
}
|
|
|
|
type ProjectHandle = {
|
|
directory: string
|
|
slug: string
|
|
gotoSession: (sessionID?: string) => Promise<void>
|
|
trackSession: (sessionID: string, directory?: string) => void
|
|
trackDirectory: (directory: string) => void
|
|
sdk: ReturnType<typeof createSdk>
|
|
}
|
|
|
|
type ProjectOptions = {
|
|
extra?: string[]
|
|
model?: { providerID: string; modelID: string }
|
|
setup?: (directory: string) => Promise<void>
|
|
beforeGoto?: (project: { directory: string; sdk: ReturnType<typeof createSdk> }) => Promise<void>
|
|
}
|
|
|
|
type ProjectFixture = ProjectHandle & {
|
|
open: (options?: ProjectOptions) => Promise<void>
|
|
prompt: (text: string) => Promise<string>
|
|
user: (text: string) => Promise<string>
|
|
shell: (cmd: string) => Promise<string>
|
|
}
|
|
|
|
type TestFixtures = {
|
|
llm: LLMFixture
|
|
assistant: AssistantFixture
|
|
project: ProjectFixture
|
|
sdk: ReturnType<typeof createSdk>
|
|
gotoSession: (sessionID?: string) => Promise<void>
|
|
withProject: <T>(callback: (project: ProjectHandle) => Promise<T>, options?: ProjectOptions) => Promise<T>
|
|
withBackendProject: <T>(callback: (project: ProjectHandle) => Promise<T>, options?: ProjectOptions) => Promise<T>
|
|
withMockProject: <T>(callback: (project: ProjectHandle) => Promise<T>, options?: ProjectOptions) => Promise<T>
|
|
}
|
|
|
|
type WorkerFixtures = {
|
|
_llm: LLMWorker
|
|
backend: {
|
|
url: string
|
|
sdk: (directory?: string) => ReturnType<typeof createSdk>
|
|
}
|
|
directory: string
|
|
slug: string
|
|
}
|
|
|
|
export const test = base.extend<TestFixtures, WorkerFixtures>({
|
|
_llm: [
|
|
async ({}, use) => {
|
|
const rt = ManagedRuntime.make(TestLLMServer.layer)
|
|
try {
|
|
const svc = await rt.runPromise(TestLLMServer.asEffect())
|
|
await use({
|
|
url: svc.url,
|
|
push: (...input) => rt.runPromise(svc.push(...input)),
|
|
pushMatch: (match, ...input) => rt.runPromise(svc.pushMatch(match, ...input)),
|
|
textMatch: (match, value, opts) => rt.runPromise(svc.textMatch(match, value, opts)),
|
|
toolMatch: (match, name, input) => rt.runPromise(svc.toolMatch(match, name, input)),
|
|
text: (value, opts) => rt.runPromise(svc.text(value, opts)),
|
|
tool: (name, input) => rt.runPromise(svc.tool(name, input)),
|
|
toolHang: (name, input) => rt.runPromise(svc.toolHang(name, input)),
|
|
reason: (value, opts) => rt.runPromise(svc.reason(value, opts)),
|
|
fail: (message) => rt.runPromise(svc.fail(message)),
|
|
error: (status, body) => rt.runPromise(svc.error(status, body)),
|
|
hang: () => rt.runPromise(svc.hang),
|
|
hold: (value, wait) => rt.runPromise(svc.hold(value, wait)),
|
|
reset: () => rt.runPromise(svc.reset),
|
|
hits: () => rt.runPromise(svc.hits),
|
|
calls: () => rt.runPromise(svc.calls),
|
|
wait: (count) => rt.runPromise(svc.wait(count)),
|
|
inputs: () => rt.runPromise(svc.inputs),
|
|
pending: () => rt.runPromise(svc.pending),
|
|
misses: () => rt.runPromise(svc.misses),
|
|
})
|
|
} finally {
|
|
await rt.dispose()
|
|
}
|
|
},
|
|
{ scope: "worker" },
|
|
],
|
|
backend: [
|
|
async ({ _llm }, use, workerInfo) => {
|
|
const handle = await startBackend(`w${workerInfo.workerIndex}`, { llmUrl: _llm.url })
|
|
try {
|
|
await use({
|
|
url: handle.url,
|
|
sdk: (directory?: string) => createSdk(directory, handle.url),
|
|
})
|
|
} finally {
|
|
await handle.stop()
|
|
}
|
|
},
|
|
{ scope: "worker" },
|
|
],
|
|
llm: async ({ _llm }, use) => {
|
|
await _llm.reset()
|
|
await use({
|
|
url: _llm.url,
|
|
push: _llm.push,
|
|
pushMatch: _llm.pushMatch,
|
|
textMatch: _llm.textMatch,
|
|
toolMatch: _llm.toolMatch,
|
|
text: _llm.text,
|
|
tool: _llm.tool,
|
|
toolHang: _llm.toolHang,
|
|
reason: _llm.reason,
|
|
fail: _llm.fail,
|
|
error: _llm.error,
|
|
hang: _llm.hang,
|
|
hold: _llm.hold,
|
|
hits: _llm.hits,
|
|
calls: _llm.calls,
|
|
wait: _llm.wait,
|
|
inputs: _llm.inputs,
|
|
pending: _llm.pending,
|
|
misses: _llm.misses,
|
|
})
|
|
const pending = await _llm.pending()
|
|
if (pending > 0) {
|
|
throw new Error(`TestLLMServer still has ${pending} queued response(s) after the test finished`)
|
|
}
|
|
},
|
|
assistant: async ({ llm }, use) => {
|
|
await use({
|
|
reply: llm.text,
|
|
tool: llm.tool,
|
|
toolHang: llm.toolHang,
|
|
reason: llm.reason,
|
|
fail: llm.fail,
|
|
error: llm.error,
|
|
hang: llm.hang,
|
|
hold: llm.hold,
|
|
calls: llm.calls,
|
|
pending: llm.pending,
|
|
})
|
|
},
|
|
page: async ({ page }, use) => {
|
|
let boundary: string | undefined
|
|
setHealthPhase(page, "test")
|
|
const consoleHandler = (msg: { text(): string }) => {
|
|
const text = msg.text()
|
|
if (!text.includes("[e2e:error-boundary]")) return
|
|
if (healthPhase(page) === "cleanup") {
|
|
console.warn(`[e2e:error-boundary][cleanup-warning]\n${text}`)
|
|
return
|
|
}
|
|
boundary ||= text
|
|
console.log(text)
|
|
}
|
|
const pageErrorHandler = (err: Error) => {
|
|
console.log(`[e2e:pageerror] ${err.stack || err.message}`)
|
|
}
|
|
page.on("console", consoleHandler)
|
|
page.on("pageerror", pageErrorHandler)
|
|
await use(page)
|
|
page.off("console", consoleHandler)
|
|
page.off("pageerror", pageErrorHandler)
|
|
if (boundary) throw new Error(boundary)
|
|
},
|
|
directory: [
|
|
async ({}, use) => {
|
|
await use(await getWorktree())
|
|
},
|
|
{ scope: "worker" },
|
|
],
|
|
slug: [
|
|
async ({ directory }, use) => {
|
|
await use(dirSlug(directory))
|
|
},
|
|
{ scope: "worker" },
|
|
],
|
|
sdk: async ({ directory }, use) => {
|
|
await use(createSdk(directory))
|
|
},
|
|
gotoSession: async ({ page, directory }, use) => {
|
|
await seedStorage(page, { directory })
|
|
|
|
const gotoSession = async (sessionID?: string) => {
|
|
await visit(page, sessionPath(directory, sessionID))
|
|
await waitSession(page, { directory, sessionID })
|
|
}
|
|
await use(gotoSession)
|
|
},
|
|
project: async ({ page, llm, backend }, use) => {
|
|
const item = makeProject(page, llm, backend)
|
|
try {
|
|
await use(item.project)
|
|
} finally {
|
|
await item.cleanup()
|
|
}
|
|
},
|
|
withProject: async ({ page }, use) => {
|
|
await use((callback, options) => runProject(page, callback, options))
|
|
},
|
|
withBackendProject: async ({ page, backend }, use) => {
|
|
await use((callback, options) =>
|
|
runProject(page, callback, { ...options, serverUrl: backend.url, sdk: backend.sdk }),
|
|
)
|
|
},
|
|
withMockProject: async ({ page, llm, backend }, use) => {
|
|
await use((callback, options) =>
|
|
withMockOpenAI({
|
|
serverUrl: backend.url,
|
|
llmUrl: llm.url,
|
|
fn: () =>
|
|
runProject(page, callback, {
|
|
...options,
|
|
model: options?.model ?? openaiModel,
|
|
serverUrl: backend.url,
|
|
sdk: backend.sdk,
|
|
}),
|
|
}),
|
|
)
|
|
},
|
|
})
|
|
|
|
function makeProject(
|
|
page: Page,
|
|
llm: LLMFixture,
|
|
backend: { url: string; sdk: (directory?: string) => ReturnType<typeof createSdk> },
|
|
) {
|
|
let state:
|
|
| {
|
|
directory: string
|
|
slug: string
|
|
sdk: ReturnType<typeof createSdk>
|
|
sessions: Map<string, string>
|
|
dirs: Set<string>
|
|
}
|
|
| undefined
|
|
|
|
const need = () => {
|
|
if (state) return state
|
|
throw new Error("project.open() must be called first")
|
|
}
|
|
|
|
const trackSession = (sessionID: string, directory?: string) => {
|
|
const cur = need()
|
|
cur.sessions.set(sessionID, directory ?? cur.directory)
|
|
}
|
|
|
|
const trackDirectory = (directory: string) => {
|
|
const cur = need()
|
|
if (directory !== cur.directory) cur.dirs.add(directory)
|
|
}
|
|
|
|
const gotoSession = async (sessionID?: string) => {
|
|
const cur = need()
|
|
await visit(page, sessionPath(cur.directory, sessionID))
|
|
await waitSession(page, { directory: cur.directory, sessionID, serverUrl: backend.url })
|
|
const current = sessionIDFromUrl(page.url())
|
|
if (current) trackSession(current)
|
|
}
|
|
|
|
const open = async (options?: ProjectOptions) => {
|
|
if (state) return
|
|
const directory = await createTestProject({ serverUrl: backend.url })
|
|
const sdk = backend.sdk(directory)
|
|
await options?.setup?.(directory)
|
|
await seedStorage(page, {
|
|
directory,
|
|
extra: options?.extra,
|
|
model: options?.model,
|
|
serverUrl: backend.url,
|
|
})
|
|
state = {
|
|
directory,
|
|
slug: "",
|
|
sdk,
|
|
sessions: new Map(),
|
|
dirs: new Set(),
|
|
}
|
|
await options?.beforeGoto?.({ directory, sdk })
|
|
await gotoSession()
|
|
need().slug = await waitSlug(page)
|
|
}
|
|
|
|
const send = async (text: string, input: { noReply: boolean; shell: boolean }) => {
|
|
const prev = await promptSend(page)
|
|
if (!input.noReply && !input.shell && (await llm.pending()) === 0) {
|
|
await llm.text("ok")
|
|
}
|
|
|
|
const prompt = page.locator(promptSelector).first()
|
|
const submit = async () => {
|
|
await expect(prompt).toBeVisible()
|
|
await prompt.click()
|
|
if (input.shell) {
|
|
await page.keyboard.type("!")
|
|
await expect(prompt).toHaveAttribute("aria-label", /enter shell command/i)
|
|
}
|
|
await page.keyboard.type(text)
|
|
await expect.poll(async () => clean(await prompt.textContent())).toBe(text)
|
|
await page.keyboard.press("Enter")
|
|
const started = await expect
|
|
.poll(async () => (await promptSend(page)).started, { timeout: 5_000 })
|
|
.toBeGreaterThan(prev.started)
|
|
.then(() => true)
|
|
.catch(() => false)
|
|
if (started) return
|
|
const send = page.getByRole("button", { name: "Send" }).first()
|
|
const enabled = await send
|
|
.isEnabled()
|
|
.then((x) => x)
|
|
.catch(() => false)
|
|
if (enabled) {
|
|
await send.click()
|
|
} else {
|
|
await prompt.click()
|
|
await page.keyboard.press("Enter")
|
|
}
|
|
await expect.poll(async () => (await promptSend(page)).started, { timeout: 5_000 }).toBeGreaterThan(prev.started)
|
|
}
|
|
|
|
if (input.noReply) {
|
|
await withNoReplyPrompt(page, submit)
|
|
} else {
|
|
await submit()
|
|
}
|
|
|
|
let next: { sessionID: string; directory: string } | undefined
|
|
await expect
|
|
.poll(
|
|
async () => {
|
|
const sent = await promptSend(page)
|
|
if (sent.count <= prev.count) return ""
|
|
if (!sent.sessionID || !sent.directory) return ""
|
|
next = { sessionID: sent.sessionID, directory: sent.directory }
|
|
return sent.sessionID
|
|
},
|
|
{ timeout: 90_000 },
|
|
)
|
|
.not.toBe("")
|
|
|
|
if (!next) throw new Error("Failed to observe prompt submission in e2e prompt probe")
|
|
const active = await waitSession(page, {
|
|
directory: next.directory,
|
|
sessionID: next.sessionID,
|
|
serverUrl: backend.url,
|
|
})
|
|
trackSession(next.sessionID, active.directory)
|
|
if (!input.shell) {
|
|
await waitSessionSaved(active.directory, next.sessionID, 90_000, backend.url)
|
|
}
|
|
await waitSessionIdle(backend.sdk(active.directory), next.sessionID, 90_000).catch(() => undefined)
|
|
return next.sessionID
|
|
}
|
|
|
|
const prompt = async (text: string) => {
|
|
return send(text, { noReply: false, shell: false })
|
|
}
|
|
|
|
const user = async (text: string) => {
|
|
return send(text, { noReply: true, shell: false })
|
|
}
|
|
|
|
const shell = async (cmd: string) => {
|
|
return send(cmd, { noReply: false, shell: true })
|
|
}
|
|
|
|
const cleanup = async () => {
|
|
const cur = state
|
|
if (!cur) return
|
|
setHealthPhase(page, "cleanup")
|
|
await Promise.allSettled(
|
|
Array.from(cur.sessions, ([sessionID, directory]) =>
|
|
cleanupSession({ sessionID, directory, serverUrl: backend.url }),
|
|
),
|
|
)
|
|
await Promise.allSettled(Array.from(cur.dirs, (directory) => cleanupTestProject(directory)))
|
|
await cleanupTestProject(cur.directory)
|
|
state = undefined
|
|
setHealthPhase(page, "test")
|
|
}
|
|
|
|
return {
|
|
project: {
|
|
open,
|
|
prompt,
|
|
user,
|
|
shell,
|
|
gotoSession,
|
|
trackSession,
|
|
trackDirectory,
|
|
get directory() {
|
|
return need().directory
|
|
},
|
|
get slug() {
|
|
return need().slug
|
|
},
|
|
get sdk() {
|
|
return need().sdk
|
|
},
|
|
},
|
|
cleanup,
|
|
}
|
|
}
|
|
|
|
async function runProject<T>(
|
|
page: Page,
|
|
callback: (project: ProjectHandle) => Promise<T>,
|
|
options?: ProjectOptions & {
|
|
serverUrl?: string
|
|
sdk?: (directory?: string) => ReturnType<typeof createSdk>
|
|
},
|
|
) {
|
|
const url = options?.serverUrl
|
|
const root = await createTestProject(url ? { serverUrl: url } : undefined)
|
|
const sdk = options?.sdk?.(root) ?? createSdk(root, url)
|
|
const sessions = new Map<string, string>()
|
|
const dirs = new Set<string>()
|
|
await options?.setup?.(root)
|
|
await seedStorage(page, {
|
|
directory: root,
|
|
extra: options?.extra,
|
|
model: options?.model,
|
|
serverUrl: url,
|
|
})
|
|
|
|
const gotoSession = async (sessionID?: string) => {
|
|
await visit(page, sessionPath(root, sessionID))
|
|
await waitSession(page, { directory: root, sessionID, serverUrl: url })
|
|
const current = sessionIDFromUrl(page.url())
|
|
if (current) trackSession(current)
|
|
}
|
|
|
|
const trackSession = (sessionID: string, directory?: string) => {
|
|
sessions.set(sessionID, directory ?? root)
|
|
}
|
|
|
|
const trackDirectory = (directory: string) => {
|
|
if (directory !== root) dirs.add(directory)
|
|
}
|
|
|
|
try {
|
|
await options?.beforeGoto?.({ directory: root, sdk })
|
|
await gotoSession()
|
|
const slug = await waitSlug(page)
|
|
return await callback({ directory: root, slug, gotoSession, trackSession, trackDirectory, sdk })
|
|
} finally {
|
|
setHealthPhase(page, "cleanup")
|
|
await Promise.allSettled(
|
|
Array.from(sessions, ([sessionID, directory]) => cleanupSession({ sessionID, directory, serverUrl: url })),
|
|
)
|
|
await Promise.allSettled(Array.from(dirs, (directory) => cleanupTestProject(directory)))
|
|
await cleanupTestProject(root)
|
|
setHealthPhase(page, "test")
|
|
}
|
|
}
|
|
|
|
async function seedStorage(
|
|
page: Page,
|
|
input: {
|
|
directory: string
|
|
extra?: string[]
|
|
model?: { providerID: string; modelID: string }
|
|
serverUrl?: string
|
|
},
|
|
) {
|
|
await seedProjects(page, input)
|
|
await page.addInitScript((model: { providerID: string; modelID: string }) => {
|
|
const win = window as E2EWindow
|
|
win.__opencode_e2e = {
|
|
...win.__opencode_e2e,
|
|
model: {
|
|
enabled: true,
|
|
},
|
|
prompt: {
|
|
enabled: true,
|
|
},
|
|
terminal: {
|
|
enabled: true,
|
|
terminals: {},
|
|
},
|
|
}
|
|
localStorage.setItem(
|
|
"opencode.global.dat:model",
|
|
JSON.stringify({
|
|
recent: [model],
|
|
user: [],
|
|
variant: {},
|
|
}),
|
|
)
|
|
}, input.model ?? seedModel)
|
|
}
|
|
|
|
export { expect }
|