Files
shopify-ai-backup/opencode/packages/opencode/test/session/llm.test.ts
2026-02-11 15:03:23 +00:00

817 lines
23 KiB
TypeScript

import { afterAll, beforeAll, beforeEach, describe, expect, test } from "bun:test"
import path from "path"
import { type ModelMessage, jsonSchema, tool } from "ai"
import { LLM } from "../../src/session/llm"
import { Global } from "../../src/global"
import { Instance } from "../../src/project/instance"
import { Provider } from "../../src/provider/provider"
import { ProviderTransform } from "../../src/provider/transform"
import { ModelsDev } from "../../src/provider/models"
import { tmpdir } from "../fixture/fixture"
import type { Agent } from "../../src/agent/agent"
import type { MessageV2 } from "../../src/session/message-v2"
describe("session.llm.hasToolCalls", () => {
test("returns false for empty messages array", () => {
expect(LLM.hasToolCalls([])).toBe(false)
})
test("returns false for messages with only text content", () => {
const messages: ModelMessage[] = [
{
role: "user",
content: [{ type: "text", text: "Hello" }],
},
{
role: "assistant",
content: [{ type: "text", text: "Hi there" }],
},
]
expect(LLM.hasToolCalls(messages)).toBe(false)
})
test("returns true when messages contain tool-call", () => {
const messages = [
{
role: "user",
content: [{ type: "text", text: "Run a command" }],
},
{
role: "assistant",
content: [
{
type: "tool-call",
toolCallId: "call-123",
toolName: "bash",
},
],
},
] as ModelMessage[]
expect(LLM.hasToolCalls(messages)).toBe(true)
})
test("returns true when messages contain tool-result", () => {
const messages = [
{
role: "tool",
content: [
{
type: "tool-result",
toolCallId: "call-123",
toolName: "bash",
},
],
},
] as ModelMessage[]
expect(LLM.hasToolCalls(messages)).toBe(true)
})
test("returns false for messages with string content", () => {
const messages: ModelMessage[] = [
{
role: "user",
content: "Hello world",
},
{
role: "assistant",
content: "Hi there",
},
]
expect(LLM.hasToolCalls(messages)).toBe(false)
})
test("returns true when tool-call is mixed with text content", () => {
const messages = [
{
role: "assistant",
content: [
{ type: "text", text: "Let me run that command" },
{
type: "tool-call",
toolCallId: "call-456",
toolName: "read",
},
],
},
] as ModelMessage[]
expect(LLM.hasToolCalls(messages)).toBe(true)
})
})
type Capture = {
url: URL
headers: Headers
body: Record<string, unknown>
}
const state = {
server: null as ReturnType<typeof Bun.serve> | null,
queue: [] as Array<{ path: string; response: Response; resolve: (value: Capture) => void }>,
}
function deferred<T>() {
const result = {} as { promise: Promise<T>; resolve: (value: T) => void }
result.promise = new Promise((resolve) => {
result.resolve = resolve
})
return result
}
function waitRequest(pathname: string, response: Response) {
const pending = deferred<Capture>()
state.queue.push({ path: pathname, response, resolve: pending.resolve })
return pending.promise
}
beforeAll(() => {
state.server = Bun.serve({
port: 0,
async fetch(req) {
const next = state.queue.shift()
if (!next) {
return new Response("unexpected request", { status: 500 })
}
const url = new URL(req.url)
const body = (await req.json()) as Record<string, unknown>
next.resolve({ url, headers: req.headers, body })
if (!url.pathname.endsWith(next.path)) {
return new Response("not found", { status: 404 })
}
return next.response
},
})
})
beforeEach(() => {
state.queue.length = 0
})
afterAll(() => {
state.server?.stop()
})
function createChatStream(text: string) {
const payload =
[
`data: ${JSON.stringify({
id: "chatcmpl-1",
object: "chat.completion.chunk",
choices: [{ delta: { role: "assistant" } }],
})}`,
`data: ${JSON.stringify({
id: "chatcmpl-1",
object: "chat.completion.chunk",
choices: [{ delta: { content: text } }],
})}`,
`data: ${JSON.stringify({
id: "chatcmpl-1",
object: "chat.completion.chunk",
choices: [{ delta: {}, finish_reason: "stop" }],
})}`,
"data: [DONE]",
].join("\n\n") + "\n\n"
const encoder = new TextEncoder()
return new ReadableStream<Uint8Array>({
start(controller) {
controller.enqueue(encoder.encode(payload))
controller.close()
},
})
}
async function loadFixture(providerID: string, modelID: string) {
const fixturePath = path.join(import.meta.dir, "../tool/fixtures/models-api.json")
const data = (await Bun.file(fixturePath).json()) as Record<string, ModelsDev.Provider>
const provider = data[providerID]
if (!provider) {
throw new Error(`Missing provider in fixture: ${providerID}`)
}
const model = provider.models[modelID]
if (!model) {
throw new Error(`Missing model in fixture: ${modelID}`)
}
return { provider, model }
}
function createEventStream(chunks: unknown[], includeDone = false) {
const lines = chunks.map((chunk) => `data: ${typeof chunk === "string" ? chunk : JSON.stringify(chunk)}`)
if (includeDone) {
lines.push("data: [DONE]")
}
const payload = lines.join("\n\n") + "\n\n"
const encoder = new TextEncoder()
return new ReadableStream<Uint8Array>({
start(controller) {
controller.enqueue(encoder.encode(payload))
controller.close()
},
})
}
function createEventResponse(chunks: unknown[], includeDone = false) {
return new Response(createEventStream(chunks, includeDone), {
status: 200,
headers: { "Content-Type": "text/event-stream" },
})
}
describe("session.llm.stream", () => {
test("sends temperature, tokens, and reasoning options for openai-compatible models", async () => {
const server = state.server
if (!server) {
throw new Error("Server not initialized")
}
const providerID = "alibaba"
const modelID = "qwen-plus"
const fixture = await loadFixture(providerID, modelID)
const provider = fixture.provider
const model = fixture.model
const request = waitRequest(
"/chat/completions",
new Response(createChatStream("Hello"), {
status: 200,
headers: { "Content-Type": "text/event-stream" },
}),
)
await using tmp = await tmpdir({
init: async (dir) => {
await Bun.write(
path.join(dir, "opencode.json"),
JSON.stringify({
$schema: "https://opencode.ai/config.json",
enabled_providers: [providerID],
provider: {
[providerID]: {
options: {
apiKey: "test-key",
baseURL: `${server.url.origin}/v1`,
},
},
},
}),
)
},
})
await Instance.provide({
directory: tmp.path,
fn: async () => {
const resolved = await Provider.getModel(providerID, model.id)
const sessionID = "session-test-1"
const agent = {
name: "test",
mode: "primary",
options: {},
permission: [{ permission: "*", pattern: "*", action: "allow" }],
temperature: 0.4,
topP: 0.8,
} satisfies Agent.Info
const user = {
id: "user-1",
sessionID,
role: "user",
time: { created: Date.now() },
agent: agent.name,
model: { providerID, modelID: resolved.id },
variant: "high",
} satisfies MessageV2.User
const stream = await LLM.stream({
user,
sessionID,
model: resolved,
agent,
system: ["You are a helpful assistant."],
abort: new AbortController().signal,
messages: [{ role: "user", content: "Hello" }],
tools: {},
})
for await (const _ of stream.fullStream) {
}
const capture = await request
const body = capture.body
const headers = capture.headers
const url = capture.url
expect(url.pathname.startsWith("/v1/")).toBe(true)
expect(url.pathname.endsWith("/chat/completions")).toBe(true)
expect(headers.get("Authorization")).toBe("Bearer test-key")
expect(headers.get("User-Agent") ?? "").toMatch(/^opencode\//)
expect(body.model).toBe(resolved.api.id)
expect(body.temperature).toBe(0.4)
expect(body.top_p).toBe(0.8)
expect(body.stream).toBe(true)
const maxTokens = (body.max_tokens as number | undefined) ?? (body.max_output_tokens as number | undefined)
const expectedMaxTokens = ProviderTransform.maxOutputTokens(
resolved.api.npm,
ProviderTransform.options({ model: resolved, sessionID }),
resolved.limit.output,
LLM.OUTPUT_TOKEN_MAX,
)
expect(maxTokens).toBe(expectedMaxTokens)
const reasoning = (body.reasoningEffort as string | undefined) ?? (body.reasoning_effort as string | undefined)
expect(reasoning).toBe("high")
},
})
})
test("sends responses API payload for OpenAI models", async () => {
const server = state.server
if (!server) {
throw new Error("Server not initialized")
}
const source = await loadFixture("openai", "gpt-5.2")
const model = source.model
const responseChunks = [
{
type: "response.created",
response: {
id: "resp-1",
created_at: Math.floor(Date.now() / 1000),
model: model.id,
service_tier: null,
},
},
{
type: "response.output_text.delta",
item_id: "item-1",
delta: "Hello",
logprobs: null,
},
{
type: "response.completed",
response: {
incomplete_details: null,
usage: {
input_tokens: 1,
input_tokens_details: null,
output_tokens: 1,
output_tokens_details: null,
},
service_tier: null,
},
},
]
const request = waitRequest("/responses", createEventResponse(responseChunks, true))
await using tmp = await tmpdir({
init: async (dir) => {
await Bun.write(
path.join(dir, "opencode.json"),
JSON.stringify({
$schema: "https://opencode.ai/config.json",
enabled_providers: ["openai"],
provider: {
openai: {
name: "OpenAI",
env: ["OPENAI_API_KEY"],
npm: "@ai-sdk/openai",
api: "https://api.openai.com/v1",
models: {
[model.id]: model,
},
options: {
apiKey: "test-openai-key",
baseURL: `${server.url.origin}/v1`,
},
},
},
}),
)
},
})
await Instance.provide({
directory: tmp.path,
fn: async () => {
const resolved = await Provider.getModel("openai", model.id)
const sessionID = "session-test-2"
const agent = {
name: "test",
mode: "primary",
options: {},
permission: [{ permission: "*", pattern: "*", action: "allow" }],
temperature: 0.2,
} satisfies Agent.Info
const user = {
id: "user-2",
sessionID,
role: "user",
time: { created: Date.now() },
agent: agent.name,
model: { providerID: "openai", modelID: resolved.id },
variant: "high",
} satisfies MessageV2.User
const stream = await LLM.stream({
user,
sessionID,
model: resolved,
agent,
system: ["You are a helpful assistant."],
abort: new AbortController().signal,
messages: [{ role: "user", content: "Hello" }],
tools: {},
})
for await (const _ of stream.fullStream) {
}
const capture = await request
const body = capture.body
expect(capture.url.pathname.endsWith("/responses")).toBe(true)
expect(body.model).toBe(resolved.api.id)
expect(body.stream).toBe(true)
expect((body.reasoning as { effort?: string } | undefined)?.effort).toBe("high")
const maxTokens = body.max_output_tokens as number | undefined
const expectedMaxTokens = ProviderTransform.maxOutputTokens(
resolved.api.npm,
ProviderTransform.options({ model: resolved, sessionID }),
resolved.limit.output,
LLM.OUTPUT_TOKEN_MAX,
)
expect(maxTokens).toBe(expectedMaxTokens)
},
})
})
test("sends messages API payload for Anthropic models", async () => {
const server = state.server
if (!server) {
throw new Error("Server not initialized")
}
const providerID = "anthropic"
const modelID = "claude-3-5-sonnet-20241022"
const fixture = await loadFixture(providerID, modelID)
const provider = fixture.provider
const model = fixture.model
const chunks = [
{
type: "message_start",
message: {
id: "msg-1",
model: model.id,
usage: {
input_tokens: 3,
cache_creation_input_tokens: null,
cache_read_input_tokens: null,
},
},
},
{
type: "content_block_start",
index: 0,
content_block: { type: "text", text: "" },
},
{
type: "content_block_delta",
index: 0,
delta: { type: "text_delta", text: "Hello" },
},
{ type: "content_block_stop", index: 0 },
{
type: "message_delta",
delta: { stop_reason: "end_turn", stop_sequence: null, container: null },
usage: {
input_tokens: 3,
output_tokens: 2,
cache_creation_input_tokens: null,
cache_read_input_tokens: null,
},
},
{ type: "message_stop" },
]
const request = waitRequest("/messages", createEventResponse(chunks))
await using tmp = await tmpdir({
init: async (dir) => {
await Bun.write(
path.join(dir, "opencode.json"),
JSON.stringify({
$schema: "https://opencode.ai/config.json",
enabled_providers: [providerID],
provider: {
[providerID]: {
options: {
apiKey: "test-anthropic-key",
baseURL: `${server.url.origin}/v1`,
},
},
},
}),
)
},
})
await Instance.provide({
directory: tmp.path,
fn: async () => {
const resolved = await Provider.getModel(providerID, model.id)
const sessionID = "session-test-3"
const agent = {
name: "test",
mode: "primary",
options: {},
permission: [{ permission: "*", pattern: "*", action: "allow" }],
temperature: 0.4,
topP: 0.9,
} satisfies Agent.Info
const user = {
id: "user-3",
sessionID,
role: "user",
time: { created: Date.now() },
agent: agent.name,
model: { providerID, modelID: resolved.id },
} satisfies MessageV2.User
const stream = await LLM.stream({
user,
sessionID,
model: resolved,
agent,
system: ["You are a helpful assistant."],
abort: new AbortController().signal,
messages: [{ role: "user", content: "Hello" }],
tools: {},
})
for await (const _ of stream.fullStream) {
}
const capture = await request
const body = capture.body
expect(capture.url.pathname.endsWith("/messages")).toBe(true)
expect(body.model).toBe(resolved.api.id)
expect(body.max_tokens).toBe(
ProviderTransform.maxOutputTokens(
resolved.api.npm,
ProviderTransform.options({ model: resolved, sessionID }),
resolved.limit.output,
LLM.OUTPUT_TOKEN_MAX,
),
)
expect(body.temperature).toBe(0.4)
expect(body.top_p).toBe(0.9)
},
})
})
test("limits chutes tool runs to a single SDK step", async () => {
const server = state.server
if (!server) throw new Error("Server not initialized")
const providerID = "chutes"
const modelID = "NousResearch/Hermes-4.3-36B"
const fixture = await loadFixture(providerID, modelID)
const model = fixture.model
const request = waitRequest(
"/chat/completions",
createEventResponse(
[
{
id: "chatcmpl-1",
object: "chat.completion.chunk",
choices: [{ delta: { role: "assistant" } }],
},
{
id: "chatcmpl-1",
object: "chat.completion.chunk",
choices: [
{
delta: {
tool_calls: [
{
index: 0,
id: "call_1",
type: "function",
function: {
name: "echo",
arguments: '{"value":"hello"}',
},
},
],
},
},
],
},
{
id: "chatcmpl-1",
object: "chat.completion.chunk",
choices: [{ delta: {}, finish_reason: "tool_calls" }],
},
],
true,
),
)
await using tmp = await tmpdir({
init: async (dir) => {
await Bun.write(
path.join(dir, "opencode.json"),
JSON.stringify({
$schema: "https://opencode.ai/config.json",
enabled_providers: [providerID],
provider: {
[providerID]: {
options: {
apiKey: "test-chutes-key",
baseURL: `${server.url.origin}/v1`,
},
},
},
}),
)
},
})
await Instance.provide({
directory: tmp.path,
fn: async () => {
const resolved = await Provider.getModel(providerID, model.id)
const sessionID = "session-test-5"
const agent = {
name: "test",
mode: "primary",
options: {},
permission: [{ permission: "*", pattern: "*", action: "allow" }],
temperature: 0.4,
} satisfies Agent.Info
const user = {
id: "user-5",
sessionID,
role: "user",
time: { created: Date.now() },
agent: agent.name,
model: { providerID, modelID: resolved.id },
} satisfies MessageV2.User
const stream = await LLM.stream({
user,
sessionID,
model: resolved,
agent,
system: ["You are a helpful assistant."],
abort: new AbortController().signal,
messages: [{ role: "user", content: "Use echo" }],
tools: {
echo: tool({
inputSchema: jsonSchema({
type: "object",
properties: {
value: { type: "string" },
},
required: ["value"],
additionalProperties: false,
}),
execute: async () => "ok",
}),
},
})
for await (const _ of stream.fullStream) {
}
const capture = await request
expect(capture.url.pathname.endsWith("/chat/completions")).toBe(true)
expect(state.queue.length).toBe(0)
},
})
})
test("sends Google API payload for Gemini models", async () => {
const server = state.server
if (!server) {
throw new Error("Server not initialized")
}
const providerID = "google"
const modelID = "gemini-2.5-flash"
const fixture = await loadFixture(providerID, modelID)
const provider = fixture.provider
const model = fixture.model
const pathSuffix = `/v1beta/models/${model.id}:streamGenerateContent`
const chunks = [
{
candidates: [
{
content: {
parts: [{ text: "Hello" }],
},
finishReason: "STOP",
},
],
usageMetadata: {
promptTokenCount: 1,
candidatesTokenCount: 1,
totalTokenCount: 2,
},
},
]
const request = waitRequest(pathSuffix, createEventResponse(chunks))
await using tmp = await tmpdir({
init: async (dir) => {
await Bun.write(
path.join(dir, "opencode.json"),
JSON.stringify({
$schema: "https://opencode.ai/config.json",
enabled_providers: [providerID],
provider: {
[providerID]: {
options: {
apiKey: "test-google-key",
baseURL: `${server.url.origin}/v1beta`,
},
},
},
}),
)
},
})
await Instance.provide({
directory: tmp.path,
fn: async () => {
const resolved = await Provider.getModel(providerID, model.id)
const sessionID = "session-test-4"
const agent = {
name: "test",
mode: "primary",
options: {},
permission: [{ permission: "*", pattern: "*", action: "allow" }],
temperature: 0.3,
topP: 0.8,
} satisfies Agent.Info
const user = {
id: "user-4",
sessionID,
role: "user",
time: { created: Date.now() },
agent: agent.name,
model: { providerID, modelID: resolved.id },
} satisfies MessageV2.User
const stream = await LLM.stream({
user,
sessionID,
model: resolved,
agent,
system: ["You are a helpful assistant."],
abort: new AbortController().signal,
messages: [{ role: "user", content: "Hello" }],
tools: {},
})
for await (const _ of stream.fullStream) {
}
const capture = await request
const body = capture.body
const config = body.generationConfig as
| { temperature?: number; topP?: number; maxOutputTokens?: number }
| undefined
expect(capture.url.pathname).toBe(pathSuffix)
expect(config?.temperature).toBe(0.3)
expect(config?.topP).toBe(0.8)
expect(config?.maxOutputTokens).toBe(
ProviderTransform.maxOutputTokens(
resolved.api.npm,
ProviderTransform.options({ model: resolved, sessionID }),
resolved.limit.output,
LLM.OUTPUT_TOKEN_MAX,
),
)
},
})
})
})