test: stabilize patch seeding across e2e backends

pull/20593/head
Kit Langton 2026-04-02 11:29:47 -04:00
parent 1c0812fe01
commit 0022cba7c5
4 changed files with 248 additions and 0 deletions

View File

@ -47,6 +47,11 @@ jobs:
- name: Run unit tests
run: bun turbo test
env:
# Bun 1.3.11 intermittently crashes on Windows during test teardown
# inside the native @parcel/watcher binding. Unit CI does not rely on
# the live watcher backend there, so disable it for that platform.
OPENCODE_EXPERIMENTAL_DISABLE_FILEWATCHER: ${{ runner.os == 'Windows' && 'true' || 'false' }}
e2e:
name: e2e (${{ matrix.settings.name }})

View File

@ -2,6 +2,8 @@ import { waitSessionIdle, withSession } from "../actions"
import { test, expect } from "../fixtures"
import { bodyText } from "../prompt/mock"
const patchModel = { providerID: "openai", modelID: "gpt-5.4" } as const
const count = 14
function body(mark: string) {
@ -55,6 +57,7 @@ async function patchWithMock(
await sdk.session.prompt({
sessionID,
agent: "build",
model: patchModel,
system: [
"You are seeding deterministic e2e UI state.",
"Your only valid response is one apply_patch tool call.",

View File

@ -254,6 +254,16 @@ function responseToolArgs(id: string, text: string, seq: number) {
}
}
function responseToolArgsDone(id: string, args: string, seq: number) {
return {
type: "response.function_call_arguments.done",
sequence_number: seq,
output_index: 0,
item_id: id,
arguments: args,
}
}
function responseToolDone(tool: { id: string; item: string; name: string; args: string }, seq: number) {
return {
type: "response.output_item.done",
@ -390,6 +400,8 @@ function responses(item: Sse, model: string) {
lines.push(responseReasonDone(reason, seq))
}
if (call && !item.hang && !item.error) {
seq += 1
lines.push(responseToolArgsDone(call.item, call.args, seq))
seq += 1
lines.push(responseToolDone(call, seq))
}

View File

@ -0,0 +1,228 @@
/**
* Reproduction test for e2e LLM URL routing.
*
* Tests whether OPENCODE_E2E_LLM_URL correctly routes LLM calls
* to the mock server when no explicit provider config is set.
* This mimics the e2e `project` fixture path (vs. withMockOpenAI).
*/
import { expect } from "bun:test"
import { Effect, Layer } from "effect"
import { Session } from "../../src/session"
import { SessionPrompt } from "../../src/session/prompt"
import { SessionSummary } from "../../src/session/summary"
import { Log } from "../../src/util/log"
import { provideTmpdirServer } from "../fixture/fixture"
import { testEffect } from "../lib/effect"
import { TestLLMServer } from "../lib/llm-server"
import { NodeFileSystem } from "@effect/platform-node"
import { Agent as AgentSvc } from "../../src/agent/agent"
import { Bus } from "../../src/bus"
import { Command } from "../../src/command"
import { Config } from "../../src/config/config"
import { FileTime } from "../../src/file/time"
import { LSP } from "../../src/lsp"
import { MCP } from "../../src/mcp"
import { Permission } from "../../src/permission"
import { Plugin } from "../../src/plugin"
import { Provider as ProviderSvc } from "../../src/provider/provider"
import { SessionCompaction } from "../../src/session/compaction"
import { Instruction } from "../../src/session/instruction"
import { SessionProcessor } from "../../src/session/processor"
import { SessionStatus } from "../../src/session/status"
import { LLM } from "../../src/session/llm"
import { Shell } from "../../src/shell/shell"
import { Snapshot } from "../../src/snapshot"
import { ToolRegistry } from "../../src/tool/registry"
import { Truncate } from "../../src/tool/truncate"
import { AppFileSystem } from "../../src/filesystem"
import * as CrossSpawnSpawner from "../../src/effect/cross-spawn-spawner"
Log.init({ print: false })
const mcp = Layer.succeed(
MCP.Service,
MCP.Service.of({
status: () => Effect.succeed({}),
clients: () => Effect.succeed({}),
tools: () => Effect.succeed({}),
prompts: () => Effect.succeed({}),
resources: () => Effect.succeed({}),
add: () => Effect.succeed({ status: { status: "disabled" as const } }),
connect: () => Effect.void,
disconnect: () => Effect.void,
getPrompt: () => Effect.succeed(undefined),
readResource: () => Effect.succeed(undefined),
startAuth: () => Effect.die("unexpected MCP auth"),
authenticate: () => Effect.die("unexpected MCP auth"),
finishAuth: () => Effect.die("unexpected MCP auth"),
removeAuth: () => Effect.void,
supportsOAuth: () => Effect.succeed(false),
hasStoredTokens: () => Effect.succeed(false),
getAuthStatus: () => Effect.succeed("not_authenticated" as const),
}),
)
const lsp = Layer.succeed(
LSP.Service,
LSP.Service.of({
init: () => Effect.void,
status: () => Effect.succeed([]),
hasClients: () => Effect.succeed(false),
touchFile: () => Effect.void,
diagnostics: () => Effect.succeed({}),
hover: () => Effect.succeed(undefined),
definition: () => Effect.succeed([]),
references: () => Effect.succeed([]),
implementation: () => Effect.succeed([]),
documentSymbol: () => Effect.succeed([]),
workspaceSymbol: () => Effect.succeed([]),
prepareCallHierarchy: () => Effect.succeed([]),
incomingCalls: () => Effect.succeed([]),
outgoingCalls: () => Effect.succeed([]),
}),
)
const filetime = Layer.succeed(
FileTime.Service,
FileTime.Service.of({
read: () => Effect.void,
get: () => Effect.succeed(undefined),
assert: () => Effect.void,
withLock: (_filepath, fn) => Effect.promise(fn),
}),
)
const status = SessionStatus.layer.pipe(Layer.provideMerge(Bus.layer))
const infra = Layer.mergeAll(NodeFileSystem.layer, CrossSpawnSpawner.defaultLayer)
const patchModel = { providerID: "openai", modelID: "gpt-5.4" } as const
function makeHttp() {
const deps = Layer.mergeAll(
Session.defaultLayer,
Snapshot.defaultLayer,
LLM.defaultLayer,
AgentSvc.defaultLayer,
Command.defaultLayer,
Permission.layer,
Plugin.defaultLayer,
Config.defaultLayer,
ProviderSvc.defaultLayer,
filetime,
lsp,
mcp,
AppFileSystem.defaultLayer,
status,
).pipe(Layer.provideMerge(infra))
const registry = ToolRegistry.layer.pipe(Layer.provideMerge(deps))
const trunc = Truncate.layer.pipe(Layer.provideMerge(deps))
const proc = SessionProcessor.layer.pipe(Layer.provideMerge(deps))
const compact = SessionCompaction.layer.pipe(Layer.provideMerge(proc), Layer.provideMerge(deps))
return Layer.mergeAll(
TestLLMServer.layer,
SessionPrompt.layer.pipe(
Layer.provideMerge(compact),
Layer.provideMerge(proc),
Layer.provideMerge(registry),
Layer.provideMerge(trunc),
Layer.provide(Instruction.defaultLayer),
Layer.provideMerge(deps),
),
)
}
const it = testEffect(makeHttp())
it.live("e2eURL routes apply_patch through mock server", () =>
provideTmpdirServer(
Effect.fnUntraced(function* ({ dir, llm }) {
// Set the env var to route all LLM calls through the mock
const prev = process.env.OPENCODE_E2E_LLM_URL
process.env.OPENCODE_E2E_LLM_URL = llm.url
yield* Effect.addFinalizer(() =>
Effect.sync(() => {
if (prev === undefined) delete process.env.OPENCODE_E2E_LLM_URL
else process.env.OPENCODE_E2E_LLM_URL = prev
}),
)
const prompt = yield* SessionPrompt.Service
const sessions = yield* Session.Service
const session = yield* sessions.create({
title: "e2e url test",
permission: [{ permission: "*", pattern: "*", action: "allow" }],
})
const patch = ["*** Begin Patch", "*** Add File: e2e-test.txt", "+line 1", "+line 2", "*** End Patch"].join("\n")
// Queue mock response: match on system prompt, return apply_patch tool call
yield* llm.toolMatch(
(hit) => JSON.stringify(hit.body).includes("Your only valid response is one apply_patch tool call"),
"apply_patch",
{ patchText: patch },
)
// After tool execution, LLM gets called again with tool result — return "done"
yield* llm.text("done")
// Seed user message
yield* prompt.prompt({
sessionID: session.id,
agent: "build",
model: patchModel,
noReply: true,
system: [
"You are seeding deterministic e2e UI state.",
"Your only valid response is one apply_patch tool call.",
`Use this JSON input: ${JSON.stringify({ patchText: patch })}`,
"Do not call any other tools.",
"Do not output plain text.",
].join("\n"),
parts: [{ type: "text", text: "Apply the provided patch exactly once." }],
})
// Run the agent loop
const result = yield* prompt.loop({ sessionID: session.id })
expect(result.info.role).toBe("assistant")
const calls = yield* llm.calls
expect(calls).toBe(2)
const missed = yield* llm.misses
expect(missed.length).toBe(0)
const content = yield* Effect.promise(() =>
Bun.file(`${dir}/e2e-test.txt`)
.text()
.catch(() => "NOT FOUND"),
)
expect(content).toContain("line 1")
let diff: Awaited<ReturnType<typeof SessionSummary.diff>> = []
for (let i = 0; i < 20; i++) {
diff = yield* Effect.promise(() => SessionSummary.diff({ sessionID: session.id }))
if (diff.length > 0) break
yield* Effect.sleep("100 millis")
}
expect(diff.length).toBeGreaterThan(0)
}),
{
git: true,
config: () => ({
model: "openai/gpt-5.4",
agent: {
build: {
model: "openai/gpt-5.4",
},
},
provider: {
openai: {
options: {
apiKey: "test-openai-key",
},
},
},
}),
},
),
)