test(app): route prompt e2e through mock llm (#20383)
Kit Langton
GitHub
parent
a3a6cf1c07
commit
9a1c9ae15a
|
|
@ -8,6 +8,13 @@ export type Usage = { input: number; output: number }
|
|||
|
||||
type Line = Record<string, unknown>
|
||||
|
||||
type Flow =
|
||||
| { type: "text"; text: string }
|
||||
| { type: "reason"; text: string }
|
||||
| { type: "tool-start"; id: string; name: string }
|
||||
| { type: "tool-args"; text: string }
|
||||
| { type: "usage"; usage: Usage }
|
||||
|
||||
type Hit = {
|
||||
url: URL
|
||||
body: Record<string, unknown>
|
||||
|
|
@ -119,6 +126,276 @@ function bytes(input: Iterable<unknown>) {
|
|||
return Stream.fromIterable([...input].map(line)).pipe(Stream.encodeText)
|
||||
}
|
||||
|
||||
function responseCreated(model: string) {
|
||||
return {
|
||||
type: "response.created",
|
||||
sequence_number: 1,
|
||||
response: {
|
||||
id: "resp_test",
|
||||
created_at: Math.floor(Date.now() / 1000),
|
||||
model,
|
||||
service_tier: null,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
function responseCompleted(input: { seq: number; usage?: Usage }) {
|
||||
return {
|
||||
type: "response.completed",
|
||||
sequence_number: input.seq,
|
||||
response: {
|
||||
incomplete_details: null,
|
||||
service_tier: null,
|
||||
usage: {
|
||||
input_tokens: input.usage?.input ?? 0,
|
||||
input_tokens_details: { cached_tokens: null },
|
||||
output_tokens: input.usage?.output ?? 0,
|
||||
output_tokens_details: { reasoning_tokens: null },
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
function responseMessage(id: string, seq: number) {
|
||||
return {
|
||||
type: "response.output_item.added",
|
||||
sequence_number: seq,
|
||||
output_index: 0,
|
||||
item: { type: "message", id },
|
||||
}
|
||||
}
|
||||
|
||||
function responseText(id: string, text: string, seq: number) {
|
||||
return {
|
||||
type: "response.output_text.delta",
|
||||
sequence_number: seq,
|
||||
item_id: id,
|
||||
delta: text,
|
||||
logprobs: null,
|
||||
}
|
||||
}
|
||||
|
||||
function responseMessageDone(id: string, seq: number) {
|
||||
return {
|
||||
type: "response.output_item.done",
|
||||
sequence_number: seq,
|
||||
output_index: 0,
|
||||
item: { type: "message", id },
|
||||
}
|
||||
}
|
||||
|
||||
function responseReason(id: string, seq: number) {
|
||||
return {
|
||||
type: "response.output_item.added",
|
||||
sequence_number: seq,
|
||||
output_index: 0,
|
||||
item: { type: "reasoning", id, encrypted_content: null },
|
||||
}
|
||||
}
|
||||
|
||||
function responseReasonPart(id: string, seq: number) {
|
||||
return {
|
||||
type: "response.reasoning_summary_part.added",
|
||||
sequence_number: seq,
|
||||
item_id: id,
|
||||
summary_index: 0,
|
||||
}
|
||||
}
|
||||
|
||||
function responseReasonText(id: string, text: string, seq: number) {
|
||||
return {
|
||||
type: "response.reasoning_summary_text.delta",
|
||||
sequence_number: seq,
|
||||
item_id: id,
|
||||
summary_index: 0,
|
||||
delta: text,
|
||||
}
|
||||
}
|
||||
|
||||
function responseReasonDone(id: string, seq: number) {
|
||||
return {
|
||||
type: "response.output_item.done",
|
||||
sequence_number: seq,
|
||||
output_index: 0,
|
||||
item: { type: "reasoning", id, encrypted_content: null },
|
||||
}
|
||||
}
|
||||
|
||||
function responseTool(id: string, item: string, name: string, seq: number) {
|
||||
return {
|
||||
type: "response.output_item.added",
|
||||
sequence_number: seq,
|
||||
output_index: 0,
|
||||
item: {
|
||||
type: "function_call",
|
||||
id: item,
|
||||
call_id: id,
|
||||
name,
|
||||
arguments: "",
|
||||
status: "in_progress",
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
function responseToolArgs(id: string, text: string, seq: number) {
|
||||
return {
|
||||
type: "response.function_call_arguments.delta",
|
||||
sequence_number: seq,
|
||||
output_index: 0,
|
||||
item_id: id,
|
||||
delta: text,
|
||||
}
|
||||
}
|
||||
|
||||
function responseToolDone(tool: { id: string; item: string; name: string; args: string }, seq: number) {
|
||||
return {
|
||||
type: "response.output_item.done",
|
||||
sequence_number: seq,
|
||||
output_index: 0,
|
||||
item: {
|
||||
type: "function_call",
|
||||
id: tool.item,
|
||||
call_id: tool.id,
|
||||
name: tool.name,
|
||||
arguments: tool.args,
|
||||
status: "completed",
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
function choices(part: unknown) {
|
||||
if (!part || typeof part !== "object") return
|
||||
if (!("choices" in part) || !Array.isArray(part.choices)) return
|
||||
const choice = part.choices[0]
|
||||
if (!choice || typeof choice !== "object") return
|
||||
return choice
|
||||
}
|
||||
|
||||
function flow(item: Sse) {
|
||||
const out: Flow[] = []
|
||||
for (const part of [...item.head, ...item.tail]) {
|
||||
const choice = choices(part)
|
||||
const delta =
|
||||
choice && "delta" in choice && choice.delta && typeof choice.delta === "object" ? choice.delta : undefined
|
||||
|
||||
if (delta && "content" in delta && typeof delta.content === "string") {
|
||||
out.push({ type: "text", text: delta.content })
|
||||
}
|
||||
|
||||
if (delta && "reasoning_content" in delta && typeof delta.reasoning_content === "string") {
|
||||
out.push({ type: "reason", text: delta.reasoning_content })
|
||||
}
|
||||
|
||||
if (delta && "tool_calls" in delta && Array.isArray(delta.tool_calls)) {
|
||||
for (const tool of delta.tool_calls) {
|
||||
if (!tool || typeof tool !== "object") continue
|
||||
const fn = "function" in tool && tool.function && typeof tool.function === "object" ? tool.function : undefined
|
||||
if ("id" in tool && typeof tool.id === "string" && fn && "name" in fn && typeof fn.name === "string") {
|
||||
out.push({ type: "tool-start", id: tool.id, name: fn.name })
|
||||
}
|
||||
if (fn && "arguments" in fn && typeof fn.arguments === "string" && fn.arguments) {
|
||||
out.push({ type: "tool-args", text: fn.arguments })
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (part && typeof part === "object" && "usage" in part && part.usage && typeof part.usage === "object") {
|
||||
const raw = part.usage as Record<string, unknown>
|
||||
if (typeof raw.prompt_tokens === "number" && typeof raw.completion_tokens === "number") {
|
||||
out.push({
|
||||
type: "usage",
|
||||
usage: { input: raw.prompt_tokens, output: raw.completion_tokens },
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
function responses(item: Sse, model: string) {
|
||||
let seq = 1
|
||||
let msg: string | undefined
|
||||
let reason: string | undefined
|
||||
let hasMsg = false
|
||||
let hasReason = false
|
||||
let call:
|
||||
| {
|
||||
id: string
|
||||
item: string
|
||||
name: string
|
||||
args: string
|
||||
}
|
||||
| undefined
|
||||
let usage: Usage | undefined
|
||||
const lines: unknown[] = [responseCreated(model)]
|
||||
|
||||
for (const part of flow(item)) {
|
||||
if (part.type === "text") {
|
||||
msg ??= "msg_1"
|
||||
if (!hasMsg) {
|
||||
hasMsg = true
|
||||
seq += 1
|
||||
lines.push(responseMessage(msg, seq))
|
||||
}
|
||||
seq += 1
|
||||
lines.push(responseText(msg, part.text, seq))
|
||||
continue
|
||||
}
|
||||
|
||||
if (part.type === "reason") {
|
||||
reason ||= "rs_1"
|
||||
if (!hasReason) {
|
||||
hasReason = true
|
||||
seq += 1
|
||||
lines.push(responseReason(reason, seq))
|
||||
seq += 1
|
||||
lines.push(responseReasonPart(reason, seq))
|
||||
}
|
||||
seq += 1
|
||||
lines.push(responseReasonText(reason, part.text, seq))
|
||||
continue
|
||||
}
|
||||
|
||||
if (part.type === "tool-start") {
|
||||
call ||= { id: part.id, item: "fc_1", name: part.name, args: "" }
|
||||
seq += 1
|
||||
lines.push(responseTool(call.id, call.item, call.name, seq))
|
||||
continue
|
||||
}
|
||||
|
||||
if (part.type === "tool-args") {
|
||||
if (!call) continue
|
||||
call.args += part.text
|
||||
seq += 1
|
||||
lines.push(responseToolArgs(call.item, part.text, seq))
|
||||
continue
|
||||
}
|
||||
|
||||
usage = part.usage
|
||||
}
|
||||
|
||||
if (msg) {
|
||||
seq += 1
|
||||
lines.push(responseMessageDone(msg, seq))
|
||||
}
|
||||
if (reason) {
|
||||
seq += 1
|
||||
lines.push(responseReasonDone(reason, seq))
|
||||
}
|
||||
if (call && !item.hang && !item.error) {
|
||||
seq += 1
|
||||
lines.push(responseToolDone(call, seq))
|
||||
}
|
||||
if (!item.hang && !item.error) lines.push(responseCompleted({ seq: seq + 1, usage }))
|
||||
return { ...item, head: lines, tail: [] } satisfies Sse
|
||||
}
|
||||
|
||||
function modelFrom(body: unknown) {
|
||||
if (!body || typeof body !== "object") return "test-model"
|
||||
if (!("model" in body) || typeof body.model !== "string") return "test-model"
|
||||
return body.model
|
||||
}
|
||||
|
||||
function send(item: Sse) {
|
||||
const head = bytes(item.head)
|
||||
const tail = bytes([...item.tail, ...(item.hang || item.error ? [] : [done])])
|
||||
|
|
@ -293,6 +570,13 @@ function item(input: Item | Reply) {
|
|||
return input instanceof Reply ? input.item() : input
|
||||
}
|
||||
|
||||
function hit(url: string, body: unknown) {
|
||||
return {
|
||||
url: new URL(url, "http://localhost"),
|
||||
body: body && typeof body === "object" ? (body as Record<string, unknown>) : {},
|
||||
} satisfies Hit
|
||||
}
|
||||
|
||||
namespace TestLLMServer {
|
||||
export interface Service {
|
||||
readonly url: string
|
||||
|
|
@ -342,30 +626,24 @@ export class TestLLMServer extends ServiceMap.Service<TestLLMServer, TestLLMServ
|
|||
return first
|
||||
}
|
||||
|
||||
yield* router.add(
|
||||
"POST",
|
||||
"/v1/chat/completions",
|
||||
Effect.gen(function* () {
|
||||
const handle = Effect.fn("TestLLMServer.handle")(function* (mode: "chat" | "responses") {
|
||||
const req = yield* HttpServerRequest.HttpServerRequest
|
||||
const next = pull()
|
||||
if (!next) return HttpServerResponse.text("unexpected request", { status: 500 })
|
||||
const body = yield* req.json.pipe(Effect.orElseSucceed(() => ({})))
|
||||
hits = [
|
||||
...hits,
|
||||
{
|
||||
url: new URL(req.originalUrl, "http://localhost"),
|
||||
body: body && typeof body === "object" ? (body as Record<string, unknown>) : {},
|
||||
},
|
||||
]
|
||||
hits = [...hits, hit(req.originalUrl, body)]
|
||||
yield* notify()
|
||||
if (next.type === "sse" && next.reset) {
|
||||
if (next.type !== "sse") return fail(next)
|
||||
if (mode === "responses") return send(responses(next, modelFrom(body)))
|
||||
if (next.reset) {
|
||||
yield* reset(next)
|
||||
return HttpServerResponse.empty()
|
||||
}
|
||||
if (next.type === "sse") return send(next)
|
||||
return fail(next)
|
||||
}),
|
||||
)
|
||||
return send(next)
|
||||
})
|
||||
|
||||
yield* router.add("POST", "/v1/chat/completions", handle("chat"))
|
||||
yield* router.add("POST", "/v1/responses", handle("responses"))
|
||||
|
||||
yield* server.serve(router.asHttpEffect())
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue