feat(providers): add native tool calling support to LM Studio and Qwen-Code (#10208)

This commit is contained in:
Hannes Rudolph 2025-12-19 10:50:09 -07:00 • committed by GitHub
parent 3f1f8be2d1
commit 61903f9588
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
7 changed files with 832 additions and 5 deletions

View file

@ -10,6 +10,8 @@ export const lMStudioDefaultModelInfo: ModelInfo = {
contextWindow: 200_000,
supportsImages: true,
supportsPromptCache: true,
supportsNativeTools: true,
defaultToolProtocol: "native",
inputPrice: 0,
outputPrice: 0,
cacheWritesPrice: 0,

View file

@ -10,6 +10,8 @@ export const qwenCodeModels = {
contextWindow: 1_000_000,
supportsImages: false,
supportsPromptCache: false,
supportsNativeTools: true,
defaultToolProtocol: "native",
inputPrice: 0,
outputPrice: 0,
cacheWritesPrice: 0,
@ -21,6 +23,8 @@ export const qwenCodeModels = {
contextWindow: 1_000_000,
supportsImages: false,
supportsPromptCache: false,
supportsNativeTools: true,
defaultToolProtocol: "native",
inputPrice: 0,
outputPrice: 0,
cacheWritesPrice: 0,

View file

@ -0,0 +1,376 @@
// npx vitest run api/providers/__tests__/lmstudio-native-tools.spec.ts
// Mock OpenAI client - must come before other imports
const mockCreate = vi.fn()
vi.mock("openai", () => {
return {
__esModule: true,
default: vi.fn().mockImplementation(() => ({
chat: {
completions: {
create: mockCreate,
},
},
})),
}
})
import { LmStudioHandler } from "../lm-studio"
import { NativeToolCallParser } from "../../../core/assistant-message/NativeToolCallParser"
import type { ApiHandlerOptions } from "../../../shared/api"
describe("LmStudioHandler Native Tools", () => {
let handler: LmStudioHandler
let mockOptions: ApiHandlerOptions
const testTools = [
{
type: "function" as const,
function: {
name: "test_tool",
description: "A test tool",
parameters: {
type: "object",
properties: {
arg1: { type: "string", description: "First argument" },
},
required: ["arg1"],
},
},
},
]
beforeEach(() => {
vi.clearAllMocks()
mockOptions = {
apiModelId: "local-model",
lmStudioModelId: "local-model",
lmStudioBaseUrl: "http://localhost:1234",
}
handler = new LmStudioHandler(mockOptions)
// Clear NativeToolCallParser state before each test
NativeToolCallParser.clearRawChunkState()
})
describe("Native Tool Calling Support", () => {
it("should include tools in request when model supports native tools and tools are provided", async () => {
mockCreate.mockImplementationOnce(() => ({
[Symbol.asyncIterator]: async function* () {
yield {
choices: [{ delta: { content: "Test response" } }],
}
},
}))
const stream = handler.createMessage("test prompt", [], {
taskId: "test-task-id",
tools: testTools,
})
await stream.next()
expect(mockCreate).toHaveBeenCalledWith(
expect.objectContaining({
tools: expect.arrayContaining([
expect.objectContaining({
type: "function",
function: expect.objectContaining({
name: "test_tool",
}),
}),
]),
parallel_tool_calls: false,
}),
)
})
it("should include tool_choice when provided", async () => {
mockCreate.mockImplementationOnce(() => ({
[Symbol.asyncIterator]: async function* () {
yield {
choices: [{ delta: { content: "Test response" } }],
}
},
}))
const stream = handler.createMessage("test prompt", [], {
taskId: "test-task-id",
tools: testTools,
tool_choice: "auto",
})
await stream.next()
expect(mockCreate).toHaveBeenCalledWith(
expect.objectContaining({
tool_choice: "auto",
}),
)
})
it("should not include tools when toolProtocol is xml", async () => {
mockCreate.mockImplementationOnce(() => ({
[Symbol.asyncIterator]: async function* () {
yield {
choices: [{ delta: { content: "Test response" } }],
}
},
}))
const stream = handler.createMessage("test prompt", [], {
taskId: "test-task-id",
tools: testTools,
toolProtocol: "xml",
})
await stream.next()
const callArgs = mockCreate.mock.calls[mockCreate.mock.calls.length - 1][0]
expect(callArgs).not.toHaveProperty("tools")
expect(callArgs).not.toHaveProperty("tool_choice")
})
it("should yield tool_call_partial chunks during streaming", async () => {
mockCreate.mockImplementationOnce(() => ({
[Symbol.asyncIterator]: async function* () {
yield {
choices: [
{
delta: {
tool_calls: [
{
index: 0,
id: "call_lmstudio_123",
function: {
name: "test_tool",
arguments: '{"arg1":',
},
},
],
},
},
],
}
yield {
choices: [
{
delta: {
tool_calls: [
{
index: 0,
function: {
arguments: '"value"}',
},
},
],
},
},
],
}
},
}))
const stream = handler.createMessage("test prompt", [], {
taskId: "test-task-id",
tools: testTools,
})
const chunks = []
for await (const chunk of stream) {
chunks.push(chunk)
}
expect(chunks).toContainEqual({
type: "tool_call_partial",
index: 0,
id: "call_lmstudio_123",
name: "test_tool",
arguments: '{"arg1":',
})
expect(chunks).toContainEqual({
type: "tool_call_partial",
index: 0,
id: undefined,
name: undefined,
arguments: '"value"}',
})
})
it("should set parallel_tool_calls based on metadata", async () => {
mockCreate.mockImplementationOnce(() => ({
[Symbol.asyncIterator]: async function* () {
yield {
choices: [{ delta: { content: "Test response" } }],
}
},
}))
const stream = handler.createMessage("test prompt", [], {
taskId: "test-task-id",
tools: testTools,
parallelToolCalls: true,
})
await stream.next()
expect(mockCreate).toHaveBeenCalledWith(
expect.objectContaining({
parallel_tool_calls: true,
}),
)
})
it("should yield tool_call_end events when finish_reason is tool_calls", async () => {
mockCreate.mockImplementationOnce(() => ({
[Symbol.asyncIterator]: async function* () {
yield {
choices: [
{
delta: {
tool_calls: [
{
index: 0,
id: "call_lmstudio_test",
function: {
name: "test_tool",
arguments: '{"arg1":"value"}',
},
},
],
},
},
],
}
yield {
choices: [
{
delta: {},
finish_reason: "tool_calls",
},
],
}
},
}))
const stream = handler.createMessage("test prompt", [], {
taskId: "test-task-id",
tools: testTools,
})
const chunks = []
for await (const chunk of stream) {
// Simulate what Task.ts does: when we receive tool_call_partial,
// process it through NativeToolCallParser to populate rawChunkTracker
if (chunk.type === "tool_call_partial") {
NativeToolCallParser.processRawChunk({
index: chunk.index,
id: chunk.id,
name: chunk.name,
arguments: chunk.arguments,
})
}
chunks.push(chunk)
}
// Should have tool_call_partial and tool_call_end
const partialChunks = chunks.filter((chunk) => chunk.type === "tool_call_partial")
const endChunks = chunks.filter((chunk) => chunk.type === "tool_call_end")
expect(partialChunks).toHaveLength(1)
expect(endChunks).toHaveLength(1)
expect(endChunks[0].id).toBe("call_lmstudio_test")
})
it("should work with parallel tool calls disabled", async () => {
mockCreate.mockImplementationOnce(() => ({
[Symbol.asyncIterator]: async function* () {
yield {
choices: [{ delta: { content: "Response" } }],
}
},
}))
const stream = handler.createMessage("test prompt", [], {
taskId: "test-task-id",
tools: testTools,
parallelToolCalls: false,
})
await stream.next()
expect(mockCreate).toHaveBeenCalledWith(
expect.objectContaining({
parallel_tool_calls: false,
}),
)
})
it("should handle reasoning content alongside tool calls", async () => {
mockCreate.mockImplementationOnce(() => ({
[Symbol.asyncIterator]: async function* () {
yield {
choices: [
{
delta: {
content: "<think>Thinking about this...</think>",
},
},
],
}
yield {
choices: [
{
delta: {
tool_calls: [
{
index: 0,
id: "call_after_think",
function: {
name: "test_tool",
arguments: '{"arg1":"result"}',
},
},
],
},
},
],
}
yield {
choices: [
{
delta: {},
finish_reason: "tool_calls",
},
],
}
},
}))
const stream = handler.createMessage("test prompt", [], {
taskId: "test-task-id",
tools: testTools,
})
const chunks = []
for await (const chunk of stream) {
if (chunk.type === "tool_call_partial") {
NativeToolCallParser.processRawChunk({
index: chunk.index,
id: chunk.id,
name: chunk.name,
arguments: chunk.arguments,
})
}
chunks.push(chunk)
}
// Should have reasoning, tool_call_partial, and tool_call_end
const reasoningChunks = chunks.filter((chunk) => chunk.type === "reasoning")
const partialChunks = chunks.filter((chunk) => chunk.type === "tool_call_partial")
const endChunks = chunks.filter((chunk) => chunk.type === "tool_call_end")
expect(reasoningChunks).toHaveLength(1)
expect(reasoningChunks[0].text).toBe("Thinking about this...")
expect(partialChunks).toHaveLength(1)
expect(endChunks).toHaveLength(1)
})
})
})

View file

@ -0,0 +1,373 @@
// npx vitest run api/providers/__tests__/qwen-code-native-tools.spec.ts
// Mock filesystem - must come before other imports
vi.mock("node:fs", () => ({
promises: {
readFile: vi.fn(),
writeFile: vi.fn(),
},
}))
const mockCreate = vi.fn()
vi.mock("openai", () => {
return {
__esModule: true,
default: vi.fn().mockImplementation(() => ({
apiKey: "test-key",
baseURL: "https://dashscope.aliyuncs.com/compatible-mode/v1",
chat: {
completions: {
create: mockCreate,
},
},
})),
}
})
import { promises as fs } from "node:fs"
import { QwenCodeHandler } from "../qwen-code"
import { NativeToolCallParser } from "../../../core/assistant-message/NativeToolCallParser"
import type { ApiHandlerOptions } from "../../../shared/api"
describe("QwenCodeHandler Native Tools", () => {
let handler: QwenCodeHandler
let mockOptions: ApiHandlerOptions & { qwenCodeOauthPath?: string }
const testTools = [
{
type: "function" as const,
function: {
name: "test_tool",
description: "A test tool",
parameters: {
type: "object",
properties: {
arg1: { type: "string", description: "First argument" },
},
required: ["arg1"],
},
},
},
]
beforeEach(() => {
vi.clearAllMocks()
// Mock credentials file
const mockCredentials = {
access_token: "test-access-token",
refresh_token: "test-refresh-token",
token_type: "Bearer",
expiry_date: Date.now() + 3600000, // 1 hour from now
resource_url: "https://dashscope.aliyuncs.com/compatible-mode/v1",
}
;(fs.readFile as any).mockResolvedValue(JSON.stringify(mockCredentials))
;(fs.writeFile as any).mockResolvedValue(undefined)
mockOptions = {
apiModelId: "qwen3-coder-plus",
}
handler = new QwenCodeHandler(mockOptions)
// Clear NativeToolCallParser state before each test
NativeToolCallParser.clearRawChunkState()
})
describe("Native Tool Calling Support", () => {
it("should include tools in request when model supports native tools and tools are provided", async () => {
mockCreate.mockImplementationOnce(() => ({
[Symbol.asyncIterator]: async function* () {
yield {
choices: [{ delta: { content: "Test response" } }],
}
},
}))
const stream = handler.createMessage("test prompt", [], {
taskId: "test-task-id",
tools: testTools,
})
await stream.next()
expect(mockCreate).toHaveBeenCalledWith(
expect.objectContaining({
tools: expect.arrayContaining([
expect.objectContaining({
type: "function",
function: expect.objectContaining({
name: "test_tool",
}),
}),
]),
parallel_tool_calls: false,
}),
)
})
it("should include tool_choice when provided", async () => {
mockCreate.mockImplementationOnce(() => ({
[Symbol.asyncIterator]: async function* () {
yield {
choices: [{ delta: { content: "Test response" } }],
}
},
}))
const stream = handler.createMessage("test prompt", [], {
taskId: "test-task-id",
tools: testTools,
tool_choice: "auto",
})
await stream.next()
expect(mockCreate).toHaveBeenCalledWith(
expect.objectContaining({
tool_choice: "auto",
}),
)
})
it("should not include tools when toolProtocol is xml", async () => {
mockCreate.mockImplementationOnce(() => ({
[Symbol.asyncIterator]: async function* () {
yield {
choices: [{ delta: { content: "Test response" } }],
}
},
}))
const stream = handler.createMessage("test prompt", [], {
taskId: "test-task-id",
tools: testTools,
toolProtocol: "xml",
})
await stream.next()
const callArgs = mockCreate.mock.calls[mockCreate.mock.calls.length - 1][0]
expect(callArgs).not.toHaveProperty("tools")
expect(callArgs).not.toHaveProperty("tool_choice")
})
it("should yield tool_call_partial chunks during streaming", async () => {
mockCreate.mockImplementationOnce(() => ({
[Symbol.asyncIterator]: async function* () {
yield {
choices: [
{
delta: {
tool_calls: [
{
index: 0,
id: "call_qwen_123",
function: {
name: "test_tool",
arguments: '{"arg1":',
},
},
],
},
},
],
}
yield {
choices: [
{
delta: {
tool_calls: [
{
index: 0,
function: {
arguments: '"value"}',
},
},
],
},
},
],
}
},
}))
const stream = handler.createMessage("test prompt", [], {
taskId: "test-task-id",
tools: testTools,
})
const chunks = []
for await (const chunk of stream) {
chunks.push(chunk)
}
expect(chunks).toContainEqual({
type: "tool_call_partial",
index: 0,
id: "call_qwen_123",
name: "test_tool",
arguments: '{"arg1":',
})
expect(chunks).toContainEqual({
type: "tool_call_partial",
index: 0,
id: undefined,
name: undefined,
arguments: '"value"}',
})
})
it("should set parallel_tool_calls based on metadata", async () => {
mockCreate.mockImplementationOnce(() => ({
[Symbol.asyncIterator]: async function* () {
yield {
choices: [{ delta: { content: "Test response" } }],
}
},
}))
const stream = handler.createMessage("test prompt", [], {
taskId: "test-task-id",
tools: testTools,
parallelToolCalls: true,
})
await stream.next()
expect(mockCreate).toHaveBeenCalledWith(
expect.objectContaining({
parallel_tool_calls: true,
}),
)
})
it("should yield tool_call_end events when finish_reason is tool_calls", async () => {
mockCreate.mockImplementationOnce(() => ({
[Symbol.asyncIterator]: async function* () {
yield {
choices: [
{
delta: {
tool_calls: [
{
index: 0,
id: "call_qwen_test",
function: {
name: "test_tool",
arguments: '{"arg1":"value"}',
},
},
],
},
},
],
}
yield {
choices: [
{
delta: {},
finish_reason: "tool_calls",
},
],
usage: { prompt_tokens: 10, completion_tokens: 5, total_tokens: 15 },
}
},
}))
const stream = handler.createMessage("test prompt", [], {
taskId: "test-task-id",
tools: testTools,
})
const chunks = []
for await (const chunk of stream) {
// Simulate what Task.ts does: when we receive tool_call_partial,
// process it through NativeToolCallParser to populate rawChunkTracker
if (chunk.type === "tool_call_partial") {
NativeToolCallParser.processRawChunk({
index: chunk.index,
id: chunk.id,
name: chunk.name,
arguments: chunk.arguments,
})
}
chunks.push(chunk)
}
// Should have tool_call_partial and tool_call_end
const partialChunks = chunks.filter((chunk) => chunk.type === "tool_call_partial")
const endChunks = chunks.filter((chunk) => chunk.type === "tool_call_end")
expect(partialChunks).toHaveLength(1)
expect(endChunks).toHaveLength(1)
expect(endChunks[0].id).toBe("call_qwen_test")
})
it("should preserve thinking block handling alongside tool calls", async () => {
mockCreate.mockImplementationOnce(() => ({
[Symbol.asyncIterator]: async function* () {
yield {
choices: [
{
delta: {
reasoning_content: "Thinking about this...",
},
},
],
}
yield {
choices: [
{
delta: {
tool_calls: [
{
index: 0,
id: "call_after_think",
function: {
name: "test_tool",
arguments: '{"arg1":"result"}',
},
},
],
},
},
],
}
yield {
choices: [
{
delta: {},
finish_reason: "tool_calls",
},
],
}
},
}))
const stream = handler.createMessage("test prompt", [], {
taskId: "test-task-id",
tools: testTools,
})
const chunks = []
for await (const chunk of stream) {
if (chunk.type === "tool_call_partial") {
NativeToolCallParser.processRawChunk({
index: chunk.index,
id: chunk.id,
name: chunk.name,
arguments: chunk.arguments,
})
}
chunks.push(chunk)
}
// Should have reasoning, tool_call_partial, and tool_call_end
const reasoningChunks = chunks.filter((chunk) => chunk.type === "reasoning")
const partialChunks = chunks.filter((chunk) => chunk.type === "tool_call_partial")
const endChunks = chunks.filter((chunk) => chunk.type === "tool_call_end")
expect(reasoningChunks).toHaveLength(1)
expect(reasoningChunks[0].text).toBe("Thinking about this...")
expect(partialChunks).toHaveLength(1)
expect(endChunks).toHaveLength(1)
})
})
})

View file

@ -6,6 +6,7 @@ import { type ModelInfo, openAiModelInfoSaneDefaults, LMSTUDIO_DEFAULT_TEMPERATU
import type { ApiHandlerOptions } from "../../shared/api"
import { NativeToolCallParser } from "../../core/assistant-message/NativeToolCallParser"
import { XmlMatcher } from "../../utils/xml-matcher"
import { convertToOpenAiMessages } from "../transform/openai-format"
@ -13,7 +14,7 @@ import { ApiStream } from "../transform/stream"
import { BaseProvider } from "./base-provider"
import type { SingleCompletionHandler, ApiHandlerCreateMessageMetadata } from "../index"
import { getModels, getModelsFromCache } from "./fetchers/modelCache"
import { getModelsFromCache } from "./fetchers/modelCache"
import { getApiRequestTimeout } from "./utils/timeout-config"
import { handleOpenAIError } from "./utils/openai-error-handler"
@ -46,6 +47,9 @@ export class LmStudioHandler extends BaseProvider implements SingleCompletionHan
...convertToOpenAiMessages(messages),
]
// LM Studio always supports native tools (https://lmstudio.ai/docs/developer/core/tools)
const useNativeTools = metadata?.tools && metadata.tools.length > 0 && metadata?.toolProtocol !== "xml"
// -------------------------
// Track token usage
// -------------------------
@ -87,6 +91,9 @@ export class LmStudioHandler extends BaseProvider implements SingleCompletionHan
messages: openAiMessages,
temperature: this.options.modelTemperature ?? LMSTUDIO_DEFAULT_TEMPERATURE,
stream: true,
...(useNativeTools && { tools: this.convertToolsForOpenAI(metadata.tools) }),
...(useNativeTools && metadata.tool_choice && { tool_choice: metadata.tool_choice }),
...(useNativeTools && { parallel_tool_calls: metadata?.parallelToolCalls ?? false }),
}
if (this.options.lmStudioSpeculativeDecodingEnabled && this.options.lmStudioDraftModelId) {
@ -111,6 +118,7 @@ export class LmStudioHandler extends BaseProvider implements SingleCompletionHan
for await (const chunk of results) {
const delta = chunk.choices[0]?.delta
const finishReason = chunk.choices[0]?.finish_reason
if (delta?.content) {
assistantText += delta.content
@ -118,6 +126,27 @@ export class LmStudioHandler extends BaseProvider implements SingleCompletionHan
yield processedChunk
}
}
// Handle tool calls in stream - emit partial chunks for NativeToolCallParser
if (delta?.tool_calls) {
for (const toolCall of delta.tool_calls) {
yield {
type: "tool_call_partial",
index: toolCall.index,
id: toolCall.id,
name: toolCall.function?.name,
arguments: toolCall.function?.arguments,
}
}
}
// Process finish_reason to emit tool_call_end events
if (finishReason) {
const endEvents = NativeToolCallParser.processFinishReason(finishReason)
for (const event of endEvents) {
yield event
}
}
}
for (const processedChunk of matcher.final()) {

View file

@ -8,11 +8,13 @@ import { type ModelInfo, type QwenCodeModelId, qwenCodeModels, qwenCodeDefaultMo
import type { ApiHandlerOptions } from "../../shared/api"
import { NativeToolCallParser } from "../../core/assistant-message/NativeToolCallParser"
import { convertToOpenAiMessages } from "../transform/openai-format"
import { ApiStream } from "../transform/stream"
import { BaseProvider } from "./base-provider"
import type { SingleCompletionHandler } from "../index"
import type { SingleCompletionHandler, ApiHandlerCreateMessageMetadata } from "../index"
const QWEN_OAUTH_BASE_URL = "https://chat.qwen.ai"
const QWEN_OAUTH_TOKEN_ENDPOINT = `${QWEN_OAUTH_BASE_URL}/api/v1/oauth2/token`
@ -201,11 +203,20 @@ export class QwenCodeHandler extends BaseProvider implements SingleCompletionHan
}
}
override async *createMessage(systemPrompt: string, messages: Anthropic.Messages.MessageParam[]): ApiStream {
override async *createMessage(
systemPrompt: string,
messages: Anthropic.Messages.MessageParam[],
metadata?: ApiHandlerCreateMessageMetadata,
): ApiStream {
await this.ensureAuthenticated()
const client = this.ensureClient()
const model = this.getModel()
// Check if model supports native tools and tools are provided with native protocol
const supportsNativeTools = model.info.supportsNativeTools ?? false
const useNativeTools =
supportsNativeTools && metadata?.tools && metadata.tools.length > 0 && metadata?.toolProtocol !== "xml"
const systemMessage: OpenAI.Chat.ChatCompletionSystemMessageParam = {
role: "system",
content: systemPrompt,
@ -220,6 +231,9 @@ export class QwenCodeHandler extends BaseProvider implements SingleCompletionHan
stream: true,
stream_options: { include_usage: true },
max_completion_tokens: model.info.maxTokens,
...(useNativeTools && { tools: this.convertToolsForOpenAI(metadata.tools) }),
...(useNativeTools && metadata.tool_choice && { tool_choice: metadata.tool_choice }),
...(useNativeTools && { parallel_tool_calls: metadata?.parallelToolCalls ?? false }),
}
const stream = await this.callApiWithRetry(() => client.chat.completions.create(requestOptions))
@ -228,6 +242,7 @@ export class QwenCodeHandler extends BaseProvider implements SingleCompletionHan
for await (const apiChunk of stream) {
const delta = apiChunk.choices[0]?.delta ?? {}
const finishReason = apiChunk.choices[0]?.finish_reason
if (delta.content) {
let newText = delta.content
@ -274,6 +289,27 @@ export class QwenCodeHandler extends BaseProvider implements SingleCompletionHan
}
}
// Handle tool calls in stream - emit partial chunks for NativeToolCallParser
if (delta.tool_calls) {
for (const toolCall of delta.tool_calls) {
yield {
type: "tool_call_partial",
index: toolCall.index,
id: toolCall.id,
name: toolCall.function?.name,
arguments: toolCall.function?.arguments,
}
}
}
// Process finish_reason to emit tool_call_end events
if (finishReason) {
const endEvents = NativeToolCallParser.processFinishReason(finishReason)
for (const event of endEvents) {
yield event
}
}
if (apiChunk.usage) {
yield {
type: "usage",

View file

@ -29,6 +29,7 @@ import {
basetenModels,
qwenCodeModels,
litellmDefaultModelInfo,
lMStudioDefaultModelInfo,
BEDROCK_1M_CONTEXT_MODEL_IDS,
isDynamicProvider,
getProviderDefaultModelId,
@ -300,10 +301,16 @@ function getSelectedModel({
}
case "lmstudio": {
const id = apiConfiguration.lmStudioModelId ?? ""
const info = lmStudioModels && lmStudioModels[apiConfiguration.lmStudioModelId!]
const modelInfo = lmStudioModels && lmStudioModels[apiConfiguration.lmStudioModelId!]
// Only merge native tool call defaults, not prices or other model-specific info
const nativeToolDefaults = {
supportsNativeTools: lMStudioDefaultModelInfo.supportsNativeTools,
defaultToolProtocol: lMStudioDefaultModelInfo.defaultToolProtocol,
}
const info = modelInfo ? { ...nativeToolDefaults, ...modelInfo } : undefined
return {
id,
info: info || undefined,
info,
}
}
case "deepinfra": {