sandvich
/
opencode
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

test(e2e): isolate prompt tests with per-worker backend (#20464)

pull/20494/head^2
Kit Langton 2026-04-01 11:58:11 -04:00 committed by GitHub
parent d58004a864
commit 38d2276592
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
8 changed files with 429 additions and 186 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

52
packages/app/e2e/actions.ts
View File

@ -312,10 +312,11 @@ export async function openSettings(page: Page) {
return dialog
}
export async function seedProjects(page: Page, input: { directory: string; extra?: string[] }) {
export async function seedProjects(page: Page, input: { directory: string; extra?: string[]; serverUrl?: string }) {
await page.addInitScript(
(args: { directory: string; serverUrl: string; extra: string[] }) => {
const key = "opencode.global.dat:server"
const defaultKey = "opencode.settings.dat:defaultServerUrl"
const raw = localStorage.getItem(key)
const parsed = (() => {
if (!raw) return undefined
@ -331,6 +332,7 @@ export async function seedProjects(page: Page, input: { directory: string; extra
const lastProject = store.lastProject && typeof store.lastProject === "object" ? store.lastProject : {}
const projects = store.projects && typeof store.projects === "object" ? store.projects : {}
const nextProjects = { ...(projects as Record<string, unknown>) }
const nextList = list.includes(args.serverUrl) ? list : [args.serverUrl, ...list]
const add = (origin: string, directory: string) => {
const current = nextProjects[origin]
@ -356,17 +358,18 @@ export async function seedProjects(page: Page, input: { directory: string; extra
localStorage.setItem(
key,
JSON.stringify({
list,
list: nextList,
projects: nextProjects,
lastProject,
}),
)
localStorage.setItem(defaultKey, args.serverUrl)
},
{ directory: input.directory, serverUrl, extra: input.extra ?? [] },
{ directory: input.directory, serverUrl: input.serverUrl ?? serverUrl, extra: input.extra ?? [] },
)
}
export async function createTestProject() {
export async function createTestProject(input?: { serverUrl?: string }) {
const root = await fs.mkdtemp(path.join(os.tmpdir(), "opencode-e2e-project-"))
const id = `e2e-${path.basename(root)}`
@ -381,7 +384,7 @@ export async function createTestProject() {
stdio: "ignore",
})
return resolveDirectory(root)
return resolveDirectory(root, input?.serverUrl)
}
export async function cleanupTestProject(directory: string) {
@ -430,22 +433,22 @@ export async function waitSlug(page: Page, skip: string[] = []) {
return next
}
export async function resolveSlug(slug: string) {
export async function resolveSlug(slug: string, input?: { serverUrl?: string }) {
const directory = base64Decode(slug)
if (!directory) throw new Error(`Failed to decode workspace slug: ${slug}`)
const resolved = await resolveDirectory(directory)
const resolved = await resolveDirectory(directory, input?.serverUrl)
return { directory: resolved, slug: base64Encode(resolved), raw: slug }
}
export async function waitDir(page: Page, directory: string) {
const target = await resolveDirectory(directory)
export async function waitDir(page: Page, directory: string, input?: { serverUrl?: string }) {
const target = await resolveDirectory(directory, input?.serverUrl)
await expect
.poll(
async () => {
await assertHealthy(page, "waitDir")
const slug = slugFromUrl(page.url())
if (!slug) return ""
return resolveSlug(slug)
return resolveSlug(slug, input)
.then((item) => item.directory)
.catch(() => "")
},
@ -455,15 +458,15 @@ export async function waitDir(page: Page, directory: string) {
return { directory: target, slug: base64Encode(target) }
}
export async function waitSession(page: Page, input: { directory: string; sessionID?: string }) {
const target = await resolveDirectory(input.directory)
export async function waitSession(page: Page, input: { directory: string; sessionID?: string; serverUrl?: string }) {
const target = await resolveDirectory(input.directory, input.serverUrl)
await expect
.poll(
async () => {
await assertHealthy(page, "waitSession")
const slug = slugFromUrl(page.url())
if (!slug) return false
const resolved = await resolveSlug(slug).catch(() => undefined)
const resolved = await resolveSlug(slug, { serverUrl: input.serverUrl }).catch(() => undefined)
if (!resolved || resolved.directory !== target) return false
const current = sessionIDFromUrl(page.url())
if (input.sessionID && current !== input.sessionID) return false
@ -473,7 +476,7 @@ export async function waitSession(page: Page, input: { directory: string; sessio
if (input.sessionID && (!state || state.sessionID !== input.sessionID)) return false
if (!input.sessionID && state?.sessionID) return false
if (state?.dir) {
const dir = await resolveDirectory(state.dir).catch(() => state.dir ?? "")
const dir = await resolveDirectory(state.dir, input.serverUrl).catch(() => state.dir ?? "")
if (dir !== target) return false
}
@ -489,9 +492,9 @@ export async function waitSession(page: Page, input: { directory: string; sessio
return { directory: target, slug: base64Encode(target) }
}
export async function waitSessionSaved(directory: string, sessionID: string, timeout = 30_000) {
const sdk = createSdk(directory)
const target = await resolveDirectory(directory)
export async function waitSessionSaved(directory: string, sessionID: string, timeout = 30_000, serverUrl?: string) {
const sdk = createSdk(directory, serverUrl)
const target = await resolveDirectory(directory, serverUrl)
await expect
.poll(
@ -501,7 +504,7 @@ export async function waitSessionSaved(directory: string, sessionID: string, tim
.then((x) => x.data)
.catch(() => undefined)
if (!data?.directory) return ""
return resolveDirectory(data.directory).catch(() => data.directory)
return resolveDirectory(data.directory, serverUrl).catch(() => data.directory)
},
{ timeout },
)
@ -666,8 +669,9 @@ export async function cleanupSession(input: {
sessionID: string
directory?: string
sdk?: ReturnType<typeof createSdk>
serverUrl?: string
}) {
const sdk = input.sdk ?? (input.directory ? createSdk(input.directory) : undefined)
const sdk = input.sdk ?? (input.directory ? createSdk(input.directory, input.serverUrl) : undefined)
if (!sdk) throw new Error("cleanupSession requires sdk or directory")
await waitSessionIdle(sdk, input.sessionID, 5_000).catch(() => undefined)
const current = await status(sdk, input.sessionID).catch(() => undefined)
@ -1019,3 +1023,13 @@ export async function openWorkspaceMenu(page: Page, workspaceSlug: string) {
await expect(menu).toBeVisible()
return menu
}
export async function assistantText(sdk: ReturnType<typeof createSdk>, sessionID: string) {
const messages = await sdk.session.messages({ sessionID, limit: 50 }).then((r) => r.data ?? [])
return messages
.filter((m) => m.info.role === "assistant")
.flatMap((m) => m.parts)
.filter((p) => p.type === "text")
.map((p) => p.text)
.join("\n")
}

119
packages/app/e2e/backend.ts 100644
View File

@ -0,0 +1,119 @@
import { spawn } from "node:child_process"
import fs from "node:fs/promises"
import net from "node:net"
import os from "node:os"
import path from "node:path"
import { fileURLToPath } from "node:url"
type Handle = {
url: string
stop: () => Promise<void>
}
function freePort() {
return new Promise<number>((resolve, reject) => {
const server = net.createServer()
server.once("error", reject)
server.listen(0, () => {
const address = server.address()
if (!address || typeof address === "string") {
server.close(() => reject(new Error("Failed to acquire a free port")))
return
}
server.close((err) => {
if (err) reject(err)
else resolve(address.port)
})
})
})
}
async function waitForHealth(url: string, probe = "/global/health") {
const end = Date.now() + 120_000
let last = ""
while (Date.now() < end) {
try {
const res = await fetch(`${url}${probe}`)
if (res.ok) return
last = `status ${res.status}`
} catch (err) {
last = err instanceof Error ? err.message : String(err)
}
await new Promise((resolve) => setTimeout(resolve, 250))
}
throw new Error(`Timed out waiting for backend health at ${url}${probe}${last ? ` (${last})` : ""}`)
}
const LOG_CAP = 100
function cap(input: string[]) {
if (input.length > LOG_CAP) input.splice(0, input.length - LOG_CAP)
}
function tail(input: string[]) {
return input.slice(-40).join("")
}
export async function startBackend(label: string): Promise<Handle> {
const port = await freePort()
const sandbox = await fs.mkdtemp(path.join(os.tmpdir(), `opencode-e2e-${label}-`))
const appDir = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..")
const repoDir = path.resolve(appDir, "../..")
const opencodeDir = path.join(repoDir, "packages", "opencode")
const env = {
...process.env,
OPENCODE_DISABLE_SHARE: process.env.OPENCODE_DISABLE_SHARE ?? "true",
OPENCODE_DISABLE_LSP_DOWNLOAD: "true",
OPENCODE_DISABLE_DEFAULT_PLUGINS: "true",
OPENCODE_EXPERIMENTAL_DISABLE_FILEWATCHER: "true",
OPENCODE_TEST_HOME: path.join(sandbox, "home"),
XDG_DATA_HOME: path.join(sandbox, "share"),
XDG_CACHE_HOME: path.join(sandbox, "cache"),
XDG_CONFIG_HOME: path.join(sandbox, "config"),
XDG_STATE_HOME: path.join(sandbox, "state"),
OPENCODE_CLIENT: "app",
OPENCODE_STRICT_CONFIG_DEPS: "true",
} satisfies Record<string, string | undefined>
const out: string[] = []
const err: string[] = []
const proc = spawn(
"bun",
["run", "--conditions=browser", "./src/index.ts", "serve", "--port", String(port), "--hostname", "127.0.0.1"],
{
cwd: opencodeDir,
env,
stdio: ["ignore", "pipe", "pipe"],
},
)
proc.stdout?.on("data", (chunk) => { out.push(String(chunk)); cap(out) })
proc.stderr?.on("data", (chunk) => { err.push(String(chunk)); cap(err) })
const url = `http://127.0.0.1:${port}`
try {
await waitForHealth(url)
} catch (error) {
proc.kill("SIGTERM")
await fs.rm(sandbox, { recursive: true, force: true }).catch(() => undefined)
throw new Error(
[
`Failed to start isolated e2e backend for ${label}`,
error instanceof Error ? error.message : String(error),
tail(out),
tail(err),
]
.filter(Boolean)
.join("\n"),
)
}
return {
url,
async stop() {
if (proc.exitCode === null) {
proc.kill("SIGTERM")
await new Promise((resolve) => proc.once("exit", () => resolve(undefined))).catch(() => undefined)
}
await fs.rm(sandbox, { recursive: true, force: true }).catch(() => undefined)
},
}
}

167
packages/app/e2e/fixtures.ts
View File

@ -3,6 +3,7 @@ import { ManagedRuntime } from "effect"
import type { E2EWindow } from "../src/testing/terminal"
import type { Item, Reply, Usage } from "../../opencode/test/lib/llm-server"
import { TestLLMServer } from "../../opencode/test/lib/llm-server"
import { startBackend } from "./backend"
import {
healthPhase,
cleanupSession,
@ -19,6 +20,20 @@ import { createSdk, dirSlug, getWorktree, sessionPath } from "./utils"
type LLMFixture = {
url: string
push: (...input: (Item | Reply)[]) => Promise<void>
pushMatch: (
match: (hit: { url: URL; body: Record<string, unknown> }) => boolean,
...input: (Item | Reply)[]
) => Promise<void>
textMatch: (
match: (hit: { url: URL; body: Record<string, unknown> }) => boolean,
value: string,
opts?: { usage?: Usage },
) => Promise<void>
toolMatch: (
match: (hit: { url: URL; body: Record<string, unknown> }) => boolean,
name: string,
input: unknown,
) => Promise<void>
text: (value: string, opts?: { usage?: Usage }) => Promise<void>
tool: (name: string, input: unknown) => Promise<void>
toolHang: (name: string, input: unknown) => Promise<void>
@ -46,32 +61,54 @@ const seedModel = (() => {
}
})()
type ProjectHandle = {
directory: string
slug: string
gotoSession: (sessionID?: string) => Promise<void>
trackSession: (sessionID: string, directory?: string) => void
trackDirectory: (directory: string) => void
sdk: ReturnType<typeof createSdk>
}
type ProjectOptions = {
extra?: string[]
model?: { providerID: string; modelID: string }
setup?: (directory: string) => Promise<void>
beforeGoto?: (project: { directory: string; sdk: ReturnType<typeof createSdk> }) => Promise<void>
}
type TestFixtures = {
llm: LLMFixture
sdk: ReturnType<typeof createSdk>
gotoSession: (sessionID?: string) => Promise<void>
withProject: <T>(
callback: (project: {
directory: string
slug: string
gotoSession: (sessionID?: string) => Promise<void>
trackSession: (sessionID: string, directory?: string) => void
trackDirectory: (directory: string) => void
}) => Promise<T>,
options?: {
extra?: string[]
model?: { providerID: string; modelID: string }
setup?: (directory: string) => Promise<void>
},
) => Promise<T>
withProject: <T>(callback: (project: ProjectHandle) => Promise<T>, options?: ProjectOptions) => Promise<T>
withBackendProject: <T>(callback: (project: ProjectHandle) => Promise<T>, options?: ProjectOptions) => Promise<T>
}
type WorkerFixtures = {
backend: {
url: string
sdk: (directory?: string) => ReturnType<typeof createSdk>
}
directory: string
slug: string
}
export const test = base.extend<TestFixtures, WorkerFixtures>({
backend: [
async ({}, use, workerInfo) => {
const handle = await startBackend(`w${workerInfo.workerIndex}`)
try {
await use({
url: handle.url,
sdk: (directory?: string) => createSdk(directory, handle.url),
})
} finally {
await handle.stop()
}
},
{ scope: "worker" },
],
llm: async ({}, use) => {
const rt = ManagedRuntime.make(TestLLMServer.layer)
try {
@ -79,6 +116,9 @@ export const test = base.extend<TestFixtures, WorkerFixtures>({
await use({
url: svc.url,
push: (...input) => rt.runPromise(svc.push(...input)),
pushMatch: (match, ...input) => rt.runPromise(svc.pushMatch(match, ...input)),
textMatch: (match, value, opts) => rt.runPromise(svc.textMatch(match, value, opts)),
toolMatch: (match, name, input) => rt.runPromise(svc.toolMatch(match, name, input)),
text: (value, opts) => rt.runPromise(svc.text(value, opts)),
tool: (name, input) => rt.runPromise(svc.tool(name, input)),
toolHang: (name, input) => rt.runPromise(svc.toolHang(name, input)),
@ -146,51 +186,78 @@ export const test = base.extend<TestFixtures, WorkerFixtures>({
await use(gotoSession)
},
withProject: async ({ page }, use) => {
await use(async (callback, options) => {
const root = await createTestProject()
const sessions = new Map<string, string>()
const dirs = new Set<string>()
await options?.setup?.(root)
await seedStorage(page, { directory: root, extra: options?.extra, model: options?.model })
const gotoSession = async (sessionID?: string) => {
await page.goto(sessionPath(root, sessionID))
await waitSession(page, { directory: root, sessionID })
const current = sessionIDFromUrl(page.url())
if (current) trackSession(current)
}
const trackSession = (sessionID: string, directory?: string) => {
sessions.set(sessionID, directory ?? root)
}
const trackDirectory = (directory: string) => {
if (directory !== root) dirs.add(directory)
}
try {
await gotoSession()
const slug = await waitSlug(page)
return await callback({ directory: root, slug, gotoSession, trackSession, trackDirectory })
} finally {
setHealthPhase(page, "cleanup")
await Promise.allSettled(
Array.from(sessions, ([sessionID, directory]) => cleanupSession({ sessionID, directory })),
)
await Promise.allSettled(Array.from(dirs, (directory) => cleanupTestProject(directory)))
await cleanupTestProject(root)
setHealthPhase(page, "test")
}
})
await use((callback, options) =>
runProject(page, callback, options),
)
},
withBackendProject: async ({ page, backend }, use) => {
await use((callback, options) =>
runProject(page, callback, { ...options, serverUrl: backend.url, sdk: backend.sdk }),
)
},
})
async function runProject<T>(
page: Page,
callback: (project: ProjectHandle) => Promise<T>,
options?: ProjectOptions & {
serverUrl?: string
sdk?: (directory?: string) => ReturnType<typeof createSdk>
},
) {
const url = options?.serverUrl
const root = await createTestProject(url ? { serverUrl: url } : undefined)
const sdk = options?.sdk?.(root) ?? createSdk(root, url)
const sessions = new Map<string, string>()
const dirs = new Set<string>()
await options?.setup?.(root)
await seedStorage(page, {
directory: root,
extra: options?.extra,
model: options?.model,
serverUrl: url,
})
const gotoSession = async (sessionID?: string) => {
await page.goto(sessionPath(root, sessionID))
await waitSession(page, { directory: root, sessionID, serverUrl: url })
const current = sessionIDFromUrl(page.url())
if (current) trackSession(current)
}
const trackSession = (sessionID: string, directory?: string) => {
sessions.set(sessionID, directory ?? root)
}
const trackDirectory = (directory: string) => {
if (directory !== root) dirs.add(directory)
}
try {
await options?.beforeGoto?.({ directory: root, sdk })
await gotoSession()
const slug = await waitSlug(page)
return await callback({ directory: root, slug, gotoSession, trackSession, trackDirectory, sdk })
} finally {
setHealthPhase(page, "cleanup")
await Promise.allSettled(
Array.from(sessions, ([sessionID, directory]) =>
cleanupSession({ sessionID, directory, serverUrl: url }),
),
)
await Promise.allSettled(Array.from(dirs, (directory) => cleanupTestProject(directory)))
await cleanupTestProject(root)
setHealthPhase(page, "test")
}
}
async function seedStorage(
page: Page,
input: {
directory: string
extra?: string[]
model?: { providerID: string; modelID: string }
serverUrl?: string
},
) {
await seedProjects(page, input)

46
packages/app/e2e/prompt/mock.ts 100644
View File

@ -0,0 +1,46 @@
import { createSdk } from "../utils"
export const openaiModel = { providerID: "openai", modelID: "gpt-5.3-chat-latest" }
type Hit = { body: Record<string, unknown> }
export function bodyText(hit: Hit) {
return JSON.stringify(hit.body)
}
export function titleMatch(hit: Hit) {
return bodyText(hit).includes("Generate a title for this conversation")
}
export function promptMatch(token: string) {
return (hit: Hit) => bodyText(hit).includes(token)
}
export async function withMockOpenAI<T>(input: { serverUrl: string; llmUrl: string; fn: () => Promise<T> }) {
const sdk = createSdk(undefined, input.serverUrl)
const prev = await sdk.global.config.get().then((res) => res.data ?? {})
try {
await sdk.global.config.update({
config: {
...prev,
model: `${openaiModel.providerID}/${openaiModel.modelID}`,
enabled_providers: ["openai"],
provider: {
...prev.provider,
openai: {
...prev.provider?.openai,
options: {
...prev.provider?.openai?.options,
apiKey: "test-key",
baseURL: input.llmUrl,
},
},
},
},
})
return await input.fn()
} finally {
await sdk.global.config.update({ config: prev })
}
}

58
packages/app/e2e/prompt/prompt-async.spec.ts
View File

@ -1,47 +1,53 @@
import { test, expect } from "../fixtures"
import { promptSelector } from "../selectors"
import { cleanupSession, sessionIDFromUrl, withSession } from "../actions"
import { assistantText, sessionIDFromUrl, withSession } from "../actions"
import { openaiModel, promptMatch, withMockOpenAI } from "./mock"
const text = (value: string | null) => (value ?? "").replace(/\u200B/g, "").trim()
// Regression test for Issue #12453: the synchronous POST /message endpoint holds
// the connection open while the agent works, causing "Failed to fetch" over
// VPN/Tailscale. The fix switches to POST /prompt_async which returns immediately.
test("prompt succeeds when sync message endpoint is unreachable", async ({ page, sdk, gotoSession }) => {
test("prompt succeeds when sync message endpoint is unreachable", async ({
page,
llm,
backend,
withBackendProject,
}) => {
test.setTimeout(120_000)
// Simulate Tailscale/VPN killing the long-lived sync connection
await page.route("**/session/*/message", (route) => route.abort("connectionfailed"))
await gotoSession()
await withMockOpenAI({
serverUrl: backend.url,
llmUrl: llm.url,
fn: async () => {
const token = `E2E_ASYNC_${Date.now()}`
await llm.textMatch(promptMatch(token), token)
const token = `E2E_ASYNC_${Date.now()}`
await page.locator(promptSelector).click()
await page.keyboard.type(`Reply with exactly: ${token}`)
await page.keyboard.press("Enter")
await withBackendProject(
async (project) => {
await page.locator(promptSelector).click()
await page.keyboard.type(`Reply with exactly: ${token}`)
await page.keyboard.press("Enter")
await expect(page).toHaveURL(/\/session\/[^/?#]+/, { timeout: 30_000 })
const sessionID = sessionIDFromUrl(page.url())!
await expect(page).toHaveURL(/\/session\/[^/?#]+/, { timeout: 30_000 })
const sessionID = sessionIDFromUrl(page.url())!
project.trackSession(sessionID)
try {
// Agent response arrives via SSE despite sync endpoint being dead
await expect
.poll(
async () => {
const messages = await sdk.session.messages({ sessionID, limit: 50 }).then((r) => r.data ?? [])
return messages
.filter((m) => m.info.role === "assistant")
.flatMap((m) => m.parts)
.filter((p) => p.type === "text")
.map((p) => p.text)
.join("\n")
await expect.poll(() => llm.calls()).toBeGreaterThanOrEqual(1)
await expect
.poll(() => assistantText(project.sdk, sessionID), { timeout: 90_000 })
.toContain(token)
},
{
model: openaiModel,
},
{ timeout: 90_000 },
)
.toContain(token)
} finally {
await cleanupSession({ sdk, sessionID })
}
},
})
})
test("failed prompt send restores the composer input", async ({ page, sdk, gotoSession }) => {

106
packages/app/e2e/prompt/prompt.spec.ts
View File

@ -1,44 +1,9 @@
import fs from "node:fs/promises"
import path from "node:path"
import { test, expect } from "../fixtures"
import { promptSelector } from "../selectors"
import { sessionIDFromUrl } from "../actions"
import { createSdk } from "../utils"
import { assistantText, sessionIDFromUrl } from "../actions"
import { openaiModel, promptMatch, titleMatch, withMockOpenAI } from "./mock"
async function config(dir: string, url: string) {
await fs.writeFile(
path.join(dir, "opencode.json"),
JSON.stringify({
$schema: "https://opencode.ai/config.json",
enabled_providers: ["e2e-llm"],
provider: {
"e2e-llm": {
name: "E2E LLM",
npm: "@ai-sdk/openai-compatible",
env: [],
models: {
"test-model": {
name: "Test Model",
tool_call: true,
limit: { context: 128000, output: 32000 },
},
},
options: {
apiKey: "test-key",
baseURL: url,
},
},
},
agent: {
build: {
model: "e2e-llm/test-model",
},
},
}),
)
}
test("can send a prompt and receive a reply", async ({ page, llm, withProject }) => {
test("can send a prompt and receive a reply", async ({ page, llm, backend, withBackendProject }) => {
test.setTimeout(120_000)
const pageErrors: string[] = []
@ -48,48 +13,43 @@ test("can send a prompt and receive a reply", async ({ page, llm, withProject })
page.on("pageerror", onPageError)
try {
await withProject(
async (project) => {
const sdk = createSdk(project.directory)
await withMockOpenAI({
serverUrl: backend.url,
llmUrl: llm.url,
fn: async () => {
const token = `E2E_OK_${Date.now()}`
await llm.text(token)
await project.gotoSession()
await llm.textMatch(titleMatch, "E2E Title")
await llm.textMatch(promptMatch(token), token)
const prompt = page.locator(promptSelector)
await prompt.click()
await page.keyboard.type(`Reply with exactly: ${token}`)
await page.keyboard.press("Enter")
await withBackendProject(
async (project) => {
const prompt = page.locator(promptSelector)
await prompt.click()
await page.keyboard.type(`Reply with exactly: ${token}`)
await page.keyboard.press("Enter")
await expect(page).toHaveURL(/\/session\/[^/?#]+/, { timeout: 30_000 })
await expect(page).toHaveURL(/\/session\/[^/?#]+/, { timeout: 30_000 })
const sessionID = (() => {
const id = sessionIDFromUrl(page.url())
if (!id) throw new Error(`Failed to parse session id from url: ${page.url()}`)
return id
})()
project.trackSession(sessionID)
const sessionID = (() => {
const id = sessionIDFromUrl(page.url())
if (!id) throw new Error(`Failed to parse session id from url: ${page.url()}`)
return id
})()
project.trackSession(sessionID)
await expect
.poll(
async () => {
const messages = await sdk.session.messages({ sessionID, limit: 50 }).then((r) => r.data ?? [])
return messages
.filter((m) => m.info.role === "assistant")
.flatMap((m) => m.parts)
.filter((p) => p.type === "text")
.map((p) => p.text)
.join("\n")
},
{ timeout: 30_000 },
)
.toContain(token)
await expect.poll(() => llm.calls()).toBeGreaterThanOrEqual(1)
await expect
.poll(() => assistantText(project.sdk, sessionID), { timeout: 30_000 })
.toContain(token)
},
{
model: openaiModel,
},
)
},
{
model: { providerID: "e2e-llm", modelID: "test-model" },
setup: (dir) => config(dir, llm.url),
},
)
})
} finally {
page.off("pageerror", onPageError)
}

14
packages/app/e2e/utils.ts
View File

@ -26,21 +26,21 @@ export const serverNamePattern = new RegExp(`(?:${serverNames.map(escape).join("
export const modKey = process.platform === "darwin" ? "Meta" : "Control"
export const terminalToggleKey = "Control+Backquote"
export function createSdk(directory?: string) {
return createOpencodeClient({ baseUrl: serverUrl, directory, throwOnError: true })
export function createSdk(directory?: string, baseUrl = serverUrl) {
return createOpencodeClient({ baseUrl, directory, throwOnError: true })
}
export async function resolveDirectory(directory: string) {
return createSdk(directory)
export async function resolveDirectory(directory: string, baseUrl = serverUrl) {
return createSdk(directory, baseUrl)
.path.get()
.then((x) => x.data?.directory ?? directory)
}
export async function getWorktree() {
const sdk = createSdk()
export async function getWorktree(baseUrl = serverUrl) {
const sdk = createSdk(undefined, baseUrl)
const result = await sdk.path.get()
const data = result.data
if (!data?.worktree) throw new Error(`Failed to resolve a worktree from ${serverUrl}/path`)
if (!data?.worktree) throw new Error(`Failed to resolve a worktree from ${baseUrl}/path`)
return data.worktree
}

53
packages/opencode/test/lib/llm-server.ts
View File

@ -20,6 +20,13 @@ type Hit = {
body: Record<string, unknown>
}
type Match = (hit: Hit) => boolean
type Queue = {
item: Item
match?: Match
}
type Wait = {
count: number
ready: Deferred.Deferred<void>
@ -420,7 +427,7 @@ const reset = Effect.fn("TestLLMServer.reset")(function* (item: Sse) {
for (const part of item.tail) res.write(line(part))
res.destroy(new Error("connection reset"))
})
yield* Effect.never
return yield* Effect.never
})
function fail(item: HttpError) {
@ -581,6 +588,9 @@ namespace TestLLMServer {
export interface Service {
readonly url: string
readonly push: (...input: (Item | Reply)[]) => Effect.Effect<void>
readonly pushMatch: (match: Match, ...input: (Item | Reply)[]) => Effect.Effect<void>
readonly textMatch: (match: Match, value: string, opts?: { usage?: Usage }) => Effect.Effect<void>
readonly toolMatch: (match: Match, name: string, input: unknown) => Effect.Effect<void>
readonly text: (value: string, opts?: { usage?: Usage }) => Effect.Effect<void>
readonly tool: (name: string, input: unknown) => Effect.Effect<void>
readonly toolHang: (name: string, input: unknown) => Effect.Effect<void>
@ -605,11 +615,15 @@ export class TestLLMServer extends ServiceMap.Service<TestLLMServer, TestLLMServ
const router = yield* HttpRouter.HttpRouter
let hits: Hit[] = []
let list: Item[] = []
let list: Queue[] = []
let waits: Wait[] = []
const queue = (...input: (Item | Reply)[]) => {
list = [...list, ...input.map(item)]
list = [...list, ...input.map((value) => ({ item: item(value) }))]
}
const queueMatch = (match: Match, ...input: (Item | Reply)[]) => {
list = [...list, ...input.map((value) => ({ item: item(value), match }))]
}
const notify = Effect.fnUntraced(function* () {
@ -619,19 +633,21 @@ export class TestLLMServer extends ServiceMap.Service<TestLLMServer, TestLLMServ
yield* Effect.forEach(ready, (item) => Deferred.succeed(item.ready, void 0))
})
const pull = () => {
const first = list[0]
if (!first) return
list = list.slice(1)
return first
const pull = (hit: Hit) => {
const index = list.findIndex((entry) => !entry.match || entry.match(hit))
if (index === -1) return
const first = list[index]
list = [...list.slice(0, index), ...list.slice(index + 1)]
return first.item
}
const handle = Effect.fn("TestLLMServer.handle")(function* (mode: "chat" | "responses") {
const req = yield* HttpServerRequest.HttpServerRequest
const next = pull()
if (!next) return HttpServerResponse.text("unexpected request", { status: 500 })
const body = yield* req.json.pipe(Effect.orElseSucceed(() => ({})))
hits = [...hits, hit(req.originalUrl, body)]
const current = hit(req.originalUrl, body)
const next = pull(current)
if (!next) return HttpServerResponse.text("unexpected request", { status: 500 })
hits = [...hits, current]
yield* notify()
if (next.type !== "sse") return fail(next)
if (mode === "responses") return send(responses(next, modelFrom(body)))
@ -655,6 +671,21 @@ export class TestLLMServer extends ServiceMap.Service<TestLLMServer, TestLLMServ
push: Effect.fn("TestLLMServer.push")(function* (...input: (Item | Reply)[]) {
queue(...input)
}),
pushMatch: Effect.fn("TestLLMServer.pushMatch")(function* (match: Match, ...input: (Item | Reply)[]) {
queueMatch(match, ...input)
}),
textMatch: Effect.fn("TestLLMServer.textMatch")(function* (
match: Match,
value: string,
opts?: { usage?: Usage },
) {
const out = reply().text(value)
if (opts?.usage) out.usage(opts.usage)
queueMatch(match, out.stop().item())
}),
toolMatch: Effect.fn("TestLLMServer.toolMatch")(function* (match: Match, name: string, input: unknown) {
queueMatch(match, reply().tool(name, input).item())
}),
text: Effect.fn("TestLLMServer.text")(function* (value: string, opts?: { usage?: Usage }) {
const out = reply().text(value)
if (opts?.usage) out.usage(opts.usage)