fix: prevent multiple HTTP requests for Chutes AI provider
Chutes AI counts each HTTP API request separately. The existing fix using stepCountIs(1) only limited the Vercel AI SDK's internal loop, but the outer while(true) loops in processor.ts and prompt.ts continued to make additional HTTP requests after tool execution. This fix: - Returns singleStepTools flag from LLM.stream() to signal single-step mode - Breaks out of processor.ts inner loop after one iteration for Chutes - Breaks out of prompt.ts outer loop after one iteration for Chutes This ensures only one HTTP request is made per user message for providers like Chutes that bill per request.
This commit is contained in:
@@ -164,7 +164,7 @@ export namespace SessionCompaction {
|
|||||||
model,
|
model,
|
||||||
})
|
})
|
||||||
|
|
||||||
if (result === "continue" && input.auto) {
|
if (result.status === "continue" && input.auto) {
|
||||||
const continueMsg = await Session.updateMessage({
|
const continueMsg = await Session.updateMessage({
|
||||||
id: Identifier.ascending("message"),
|
id: Identifier.ascending("message"),
|
||||||
role: "user",
|
role: "user",
|
||||||
|
|||||||
@@ -45,7 +45,10 @@ export namespace LLM {
|
|||||||
allProviders?: Record<string, Provider.Info>
|
allProviders?: Record<string, Provider.Info>
|
||||||
}
|
}
|
||||||
|
|
||||||
export type StreamOutput = StreamTextResult<ToolSet, unknown>
|
export type StreamOutput = {
|
||||||
|
result: StreamTextResult<ToolSet, unknown>
|
||||||
|
singleStepTools: boolean
|
||||||
|
}
|
||||||
|
|
||||||
export type RetryState = {
|
export type RetryState = {
|
||||||
attempt: number
|
attempt: number
|
||||||
@@ -178,7 +181,7 @@ export namespace LLM {
|
|||||||
|
|
||||||
ProviderSwitch.recordSuccess(input.model.providerID, input.model.id)
|
ProviderSwitch.recordSuccess(input.model.providerID, input.model.id)
|
||||||
|
|
||||||
return result
|
return { result, singleStepTools }
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
retryState.lastError = error as Error
|
retryState.lastError = error as Error
|
||||||
ProviderSwitch.recordFailure(input.model.providerID, input.model.id)
|
ProviderSwitch.recordFailure(input.model.providerID, input.model.id)
|
||||||
@@ -454,10 +457,16 @@ export namespace LLM {
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
export async function stream(input: StreamInput) {
|
export async function stream(input: StreamInput): Promise<StreamOutput> {
|
||||||
return streamWithProvider(input)
|
return streamWithProvider(input)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export function shouldLimitToolLoopForModel(model: Provider.Model): boolean {
|
||||||
|
if (model.providerID === "chutes") return true
|
||||||
|
const url = model.api.url?.toLowerCase() || ""
|
||||||
|
return url.includes("chutes.ai") || url.includes("/chutes/")
|
||||||
|
}
|
||||||
|
|
||||||
async function resolveTools(input: Pick<StreamInput, "tools" | "agent" | "user">) {
|
async function resolveTools(input: Pick<StreamInput, "tools" | "agent" | "user">) {
|
||||||
const disabled = PermissionNext.disabled(Object.keys(input.tools), input.agent.permission)
|
const disabled = PermissionNext.disabled(Object.keys(input.tools), input.agent.permission)
|
||||||
for (const tool of Object.keys(input.tools)) {
|
for (const tool of Object.keys(input.tools)) {
|
||||||
|
|||||||
@@ -57,7 +57,7 @@ export namespace SessionProcessor {
|
|||||||
const log = Log.create({ service: "session.processor" })
|
const log = Log.create({ service: "session.processor" })
|
||||||
|
|
||||||
export type Info = Awaited<ReturnType<typeof create>>
|
export type Info = Awaited<ReturnType<typeof create>>
|
||||||
export type Result = Awaited<ReturnType<Info["process"]>>
|
export type Result = Awaited<ReturnType<Info["process"]>> & { singleStepTools: boolean }
|
||||||
|
|
||||||
export function create(input: {
|
export function create(input: {
|
||||||
assistantMessage: MessageV2.Assistant
|
assistantMessage: MessageV2.Assistant
|
||||||
@@ -70,6 +70,7 @@ export namespace SessionProcessor {
|
|||||||
let blocked = false
|
let blocked = false
|
||||||
let attempt = 0
|
let attempt = 0
|
||||||
let needsCompaction = false
|
let needsCompaction = false
|
||||||
|
let singleStepTools = false
|
||||||
|
|
||||||
const result = {
|
const result = {
|
||||||
get message() {
|
get message() {
|
||||||
@@ -78,7 +79,7 @@ export namespace SessionProcessor {
|
|||||||
partFromToolCall(toolCallID: string) {
|
partFromToolCall(toolCallID: string) {
|
||||||
return toolcalls[toolCallID]
|
return toolcalls[toolCallID]
|
||||||
},
|
},
|
||||||
async process(streamInput: LLM.StreamInput & { allProviders?: Record<string, Provider.Info> }) {
|
async process(streamInput: LLM.StreamInput & { allProviders?: Record<string, Provider.Info> }): Promise<{ status: "compact" | "stop" | "continue"; singleStepTools: boolean }> {
|
||||||
log.info("process")
|
log.info("process")
|
||||||
needsCompaction = false
|
needsCompaction = false
|
||||||
const shouldBreak = (await Config.get()).experimental?.continue_loop_on_deny !== true
|
const shouldBreak = (await Config.get()).experimental?.continue_loop_on_deny !== true
|
||||||
@@ -86,12 +87,13 @@ export namespace SessionProcessor {
|
|||||||
try {
|
try {
|
||||||
let currentText: MessageV2.TextPart | undefined
|
let currentText: MessageV2.TextPart | undefined
|
||||||
let reasoningMap: Record<string, MessageV2.ReasoningPart> = {}
|
let reasoningMap: Record<string, MessageV2.ReasoningPart> = {}
|
||||||
const stream = await LLM.stream({
|
const streamOutput = await LLM.stream({
|
||||||
...streamInput,
|
...streamInput,
|
||||||
allProviders: streamInput.allProviders || await Provider.list(),
|
allProviders: streamInput.allProviders || await Provider.list(),
|
||||||
})
|
})
|
||||||
|
singleStepTools = streamOutput.singleStepTools
|
||||||
|
|
||||||
for await (const value of stream.fullStream) {
|
for await (const value of streamOutput.result.fullStream) {
|
||||||
input.abort.throwIfAborted()
|
input.abort.throwIfAborted()
|
||||||
switch (value.type) {
|
switch (value.type) {
|
||||||
case "start":
|
case "start":
|
||||||
@@ -438,11 +440,13 @@ export namespace SessionProcessor {
|
|||||||
}
|
}
|
||||||
input.assistantMessage.time.completed = Date.now()
|
input.assistantMessage.time.completed = Date.now()
|
||||||
await Session.updateMessage(input.assistantMessage)
|
await Session.updateMessage(input.assistantMessage)
|
||||||
if (needsCompaction) return "compact"
|
if (needsCompaction) return { status: "compact" as const, singleStepTools }
|
||||||
if (blocked) return "stop"
|
if (blocked) return { status: "stop" as const, singleStepTools }
|
||||||
if (input.assistantMessage.error) return "stop"
|
if (input.assistantMessage.error) return { status: "stop" as const, singleStepTools }
|
||||||
return "continue"
|
if (singleStepTools) return { status: "continue" as const, singleStepTools }
|
||||||
|
break
|
||||||
}
|
}
|
||||||
|
return { status: "continue" as const, singleStepTools }
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
return result
|
return result
|
||||||
|
|||||||
@@ -631,8 +631,8 @@ export namespace SessionPrompt {
|
|||||||
model,
|
model,
|
||||||
allProviders,
|
allProviders,
|
||||||
})
|
})
|
||||||
if (result === "stop") break
|
if (result.status === "stop") break
|
||||||
if (result === "compact") {
|
if (result.status === "compact") {
|
||||||
await SessionCompaction.create({
|
await SessionCompaction.create({
|
||||||
sessionID,
|
sessionID,
|
||||||
agent: lastUser.agent,
|
agent: lastUser.agent,
|
||||||
@@ -640,6 +640,7 @@ export namespace SessionPrompt {
|
|||||||
auto: true,
|
auto: true,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
if (result.singleStepTools) break
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
SessionCompaction.prune({ sessionID })
|
SessionCompaction.prune({ sessionID })
|
||||||
|
|||||||
Reference in New Issue
Block a user