mirror of
https://github.com/RooVetGit/Roo-Code.git
synced 2026-10-07 02:58:15 +00:00
feat(providers): add native tool calling support to LM Studio and Qwen-Code (#10208)
This commit is contained in:
parent
3f1f8be2d1
commit
61903f9588
7 changed files with 832 additions and 5 deletions
|
|
@ -10,6 +10,8 @@ export const lMStudioDefaultModelInfo: ModelInfo = {
|
|||
contextWindow: 200_000,
|
||||
supportsImages: true,
|
||||
supportsPromptCache: true,
|
||||
supportsNativeTools: true,
|
||||
defaultToolProtocol: "native",
|
||||
inputPrice: 0,
|
||||
outputPrice: 0,
|
||||
cacheWritesPrice: 0,
|
||||
|
|
|
|||
|
|
@ -10,6 +10,8 @@ export const qwenCodeModels = {
|
|||
contextWindow: 1_000_000,
|
||||
supportsImages: false,
|
||||
supportsPromptCache: false,
|
||||
supportsNativeTools: true,
|
||||
defaultToolProtocol: "native",
|
||||
inputPrice: 0,
|
||||
outputPrice: 0,
|
||||
cacheWritesPrice: 0,
|
||||
|
|
@ -21,6 +23,8 @@ export const qwenCodeModels = {
|
|||
contextWindow: 1_000_000,
|
||||
supportsImages: false,
|
||||
supportsPromptCache: false,
|
||||
supportsNativeTools: true,
|
||||
defaultToolProtocol: "native",
|
||||
inputPrice: 0,
|
||||
outputPrice: 0,
|
||||
cacheWritesPrice: 0,
|
||||
|
|
|
|||
376
src/api/providers/__tests__/lmstudio-native-tools.spec.ts
Normal file
376
src/api/providers/__tests__/lmstudio-native-tools.spec.ts
Normal file
|
|
@ -0,0 +1,376 @@
|
|||
// npx vitest run api/providers/__tests__/lmstudio-native-tools.spec.ts
|
||||
|
||||
// Mock OpenAI client - must come before other imports
|
||||
const mockCreate = vi.fn()
|
||||
vi.mock("openai", () => {
|
||||
return {
|
||||
__esModule: true,
|
||||
default: vi.fn().mockImplementation(() => ({
|
||||
chat: {
|
||||
completions: {
|
||||
create: mockCreate,
|
||||
},
|
||||
},
|
||||
})),
|
||||
}
|
||||
})
|
||||
|
||||
import { LmStudioHandler } from "../lm-studio"
|
||||
import { NativeToolCallParser } from "../../../core/assistant-message/NativeToolCallParser"
|
||||
import type { ApiHandlerOptions } from "../../../shared/api"
|
||||
|
||||
describe("LmStudioHandler Native Tools", () => {
|
||||
let handler: LmStudioHandler
|
||||
let mockOptions: ApiHandlerOptions
|
||||
|
||||
const testTools = [
|
||||
{
|
||||
type: "function" as const,
|
||||
function: {
|
||||
name: "test_tool",
|
||||
description: "A test tool",
|
||||
parameters: {
|
||||
type: "object",
|
||||
properties: {
|
||||
arg1: { type: "string", description: "First argument" },
|
||||
},
|
||||
required: ["arg1"],
|
||||
},
|
||||
},
|
||||
},
|
||||
]
|
||||
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks()
|
||||
|
||||
mockOptions = {
|
||||
apiModelId: "local-model",
|
||||
lmStudioModelId: "local-model",
|
||||
lmStudioBaseUrl: "http://localhost:1234",
|
||||
}
|
||||
handler = new LmStudioHandler(mockOptions)
|
||||
|
||||
// Clear NativeToolCallParser state before each test
|
||||
NativeToolCallParser.clearRawChunkState()
|
||||
})
|
||||
|
||||
describe("Native Tool Calling Support", () => {
|
||||
it("should include tools in request when model supports native tools and tools are provided", async () => {
|
||||
mockCreate.mockImplementationOnce(() => ({
|
||||
[Symbol.asyncIterator]: async function* () {
|
||||
yield {
|
||||
choices: [{ delta: { content: "Test response" } }],
|
||||
}
|
||||
},
|
||||
}))
|
||||
|
||||
const stream = handler.createMessage("test prompt", [], {
|
||||
taskId: "test-task-id",
|
||||
tools: testTools,
|
||||
})
|
||||
await stream.next()
|
||||
|
||||
expect(mockCreate).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
tools: expect.arrayContaining([
|
||||
expect.objectContaining({
|
||||
type: "function",
|
||||
function: expect.objectContaining({
|
||||
name: "test_tool",
|
||||
}),
|
||||
}),
|
||||
]),
|
||||
parallel_tool_calls: false,
|
||||
}),
|
||||
)
|
||||
})
|
||||
|
||||
it("should include tool_choice when provided", async () => {
|
||||
mockCreate.mockImplementationOnce(() => ({
|
||||
[Symbol.asyncIterator]: async function* () {
|
||||
yield {
|
||||
choices: [{ delta: { content: "Test response" } }],
|
||||
}
|
||||
},
|
||||
}))
|
||||
|
||||
const stream = handler.createMessage("test prompt", [], {
|
||||
taskId: "test-task-id",
|
||||
tools: testTools,
|
||||
tool_choice: "auto",
|
||||
})
|
||||
await stream.next()
|
||||
|
||||
expect(mockCreate).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
tool_choice: "auto",
|
||||
}),
|
||||
)
|
||||
})
|
||||
|
||||
it("should not include tools when toolProtocol is xml", async () => {
|
||||
mockCreate.mockImplementationOnce(() => ({
|
||||
[Symbol.asyncIterator]: async function* () {
|
||||
yield {
|
||||
choices: [{ delta: { content: "Test response" } }],
|
||||
}
|
||||
},
|
||||
}))
|
||||
|
||||
const stream = handler.createMessage("test prompt", [], {
|
||||
taskId: "test-task-id",
|
||||
tools: testTools,
|
||||
toolProtocol: "xml",
|
||||
})
|
||||
await stream.next()
|
||||
|
||||
const callArgs = mockCreate.mock.calls[mockCreate.mock.calls.length - 1][0]
|
||||
expect(callArgs).not.toHaveProperty("tools")
|
||||
expect(callArgs).not.toHaveProperty("tool_choice")
|
||||
})
|
||||
|
||||
it("should yield tool_call_partial chunks during streaming", async () => {
|
||||
mockCreate.mockImplementationOnce(() => ({
|
||||
[Symbol.asyncIterator]: async function* () {
|
||||
yield {
|
||||
choices: [
|
||||
{
|
||||
delta: {
|
||||
tool_calls: [
|
||||
{
|
||||
index: 0,
|
||||
id: "call_lmstudio_123",
|
||||
function: {
|
||||
name: "test_tool",
|
||||
arguments: '{"arg1":',
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
],
|
||||
}
|
||||
yield {
|
||||
choices: [
|
||||
{
|
||||
delta: {
|
||||
tool_calls: [
|
||||
{
|
||||
index: 0,
|
||||
function: {
|
||||
arguments: '"value"}',
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
],
|
||||
}
|
||||
},
|
||||
}))
|
||||
|
||||
const stream = handler.createMessage("test prompt", [], {
|
||||
taskId: "test-task-id",
|
||||
tools: testTools,
|
||||
})
|
||||
|
||||
const chunks = []
|
||||
for await (const chunk of stream) {
|
||||
chunks.push(chunk)
|
||||
}
|
||||
|
||||
expect(chunks).toContainEqual({
|
||||
type: "tool_call_partial",
|
||||
index: 0,
|
||||
id: "call_lmstudio_123",
|
||||
name: "test_tool",
|
||||
arguments: '{"arg1":',
|
||||
})
|
||||
|
||||
expect(chunks).toContainEqual({
|
||||
type: "tool_call_partial",
|
||||
index: 0,
|
||||
id: undefined,
|
||||
name: undefined,
|
||||
arguments: '"value"}',
|
||||
})
|
||||
})
|
||||
|
||||
it("should set parallel_tool_calls based on metadata", async () => {
|
||||
mockCreate.mockImplementationOnce(() => ({
|
||||
[Symbol.asyncIterator]: async function* () {
|
||||
yield {
|
||||
choices: [{ delta: { content: "Test response" } }],
|
||||
}
|
||||
},
|
||||
}))
|
||||
|
||||
const stream = handler.createMessage("test prompt", [], {
|
||||
taskId: "test-task-id",
|
||||
tools: testTools,
|
||||
parallelToolCalls: true,
|
||||
})
|
||||
await stream.next()
|
||||
|
||||
expect(mockCreate).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
parallel_tool_calls: true,
|
||||
}),
|
||||
)
|
||||
})
|
||||
|
||||
it("should yield tool_call_end events when finish_reason is tool_calls", async () => {
|
||||
mockCreate.mockImplementationOnce(() => ({
|
||||
[Symbol.asyncIterator]: async function* () {
|
||||
yield {
|
||||
choices: [
|
||||
{
|
||||
delta: {
|
||||
tool_calls: [
|
||||
{
|
||||
index: 0,
|
||||
id: "call_lmstudio_test",
|
||||
function: {
|
||||
name: "test_tool",
|
||||
arguments: '{"arg1":"value"}',
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
],
|
||||
}
|
||||
yield {
|
||||
choices: [
|
||||
{
|
||||
delta: {},
|
||||
finish_reason: "tool_calls",
|
||||
},
|
||||
],
|
||||
}
|
||||
},
|
||||
}))
|
||||
|
||||
const stream = handler.createMessage("test prompt", [], {
|
||||
taskId: "test-task-id",
|
||||
tools: testTools,
|
||||
})
|
||||
|
||||
const chunks = []
|
||||
for await (const chunk of stream) {
|
||||
// Simulate what Task.ts does: when we receive tool_call_partial,
|
||||
// process it through NativeToolCallParser to populate rawChunkTracker
|
||||
if (chunk.type === "tool_call_partial") {
|
||||
NativeToolCallParser.processRawChunk({
|
||||
index: chunk.index,
|
||||
id: chunk.id,
|
||||
name: chunk.name,
|
||||
arguments: chunk.arguments,
|
||||
})
|
||||
}
|
||||
chunks.push(chunk)
|
||||
}
|
||||
|
||||
// Should have tool_call_partial and tool_call_end
|
||||
const partialChunks = chunks.filter((chunk) => chunk.type === "tool_call_partial")
|
||||
const endChunks = chunks.filter((chunk) => chunk.type === "tool_call_end")
|
||||
|
||||
expect(partialChunks).toHaveLength(1)
|
||||
expect(endChunks).toHaveLength(1)
|
||||
expect(endChunks[0].id).toBe("call_lmstudio_test")
|
||||
})
|
||||
|
||||
it("should work with parallel tool calls disabled", async () => {
|
||||
mockCreate.mockImplementationOnce(() => ({
|
||||
[Symbol.asyncIterator]: async function* () {
|
||||
yield {
|
||||
choices: [{ delta: { content: "Response" } }],
|
||||
}
|
||||
},
|
||||
}))
|
||||
|
||||
const stream = handler.createMessage("test prompt", [], {
|
||||
taskId: "test-task-id",
|
||||
tools: testTools,
|
||||
parallelToolCalls: false,
|
||||
})
|
||||
await stream.next()
|
||||
|
||||
expect(mockCreate).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
parallel_tool_calls: false,
|
||||
}),
|
||||
)
|
||||
})
|
||||
|
||||
it("should handle reasoning content alongside tool calls", async () => {
|
||||
mockCreate.mockImplementationOnce(() => ({
|
||||
[Symbol.asyncIterator]: async function* () {
|
||||
yield {
|
||||
choices: [
|
||||
{
|
||||
delta: {
|
||||
content: "<think>Thinking about this...</think>",
|
||||
},
|
||||
},
|
||||
],
|
||||
}
|
||||
yield {
|
||||
choices: [
|
||||
{
|
||||
delta: {
|
||||
tool_calls: [
|
||||
{
|
||||
index: 0,
|
||||
id: "call_after_think",
|
||||
function: {
|
||||
name: "test_tool",
|
||||
arguments: '{"arg1":"result"}',
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
],
|
||||
}
|
||||
yield {
|
||||
choices: [
|
||||
{
|
||||
delta: {},
|
||||
finish_reason: "tool_calls",
|
||||
},
|
||||
],
|
||||
}
|
||||
},
|
||||
}))
|
||||
|
||||
const stream = handler.createMessage("test prompt", [], {
|
||||
taskId: "test-task-id",
|
||||
tools: testTools,
|
||||
})
|
||||
|
||||
const chunks = []
|
||||
for await (const chunk of stream) {
|
||||
if (chunk.type === "tool_call_partial") {
|
||||
NativeToolCallParser.processRawChunk({
|
||||
index: chunk.index,
|
||||
id: chunk.id,
|
||||
name: chunk.name,
|
||||
arguments: chunk.arguments,
|
||||
})
|
||||
}
|
||||
chunks.push(chunk)
|
||||
}
|
||||
|
||||
// Should have reasoning, tool_call_partial, and tool_call_end
|
||||
const reasoningChunks = chunks.filter((chunk) => chunk.type === "reasoning")
|
||||
const partialChunks = chunks.filter((chunk) => chunk.type === "tool_call_partial")
|
||||
const endChunks = chunks.filter((chunk) => chunk.type === "tool_call_end")
|
||||
|
||||
expect(reasoningChunks).toHaveLength(1)
|
||||
expect(reasoningChunks[0].text).toBe("Thinking about this...")
|
||||
expect(partialChunks).toHaveLength(1)
|
||||
expect(endChunks).toHaveLength(1)
|
||||
})
|
||||
})
|
||||
})
|
||||
373
src/api/providers/__tests__/qwen-code-native-tools.spec.ts
Normal file
373
src/api/providers/__tests__/qwen-code-native-tools.spec.ts
Normal file
|
|
@ -0,0 +1,373 @@
|
|||
// npx vitest run api/providers/__tests__/qwen-code-native-tools.spec.ts
|
||||
|
||||
// Mock filesystem - must come before other imports
|
||||
vi.mock("node:fs", () => ({
|
||||
promises: {
|
||||
readFile: vi.fn(),
|
||||
writeFile: vi.fn(),
|
||||
},
|
||||
}))
|
||||
|
||||
const mockCreate = vi.fn()
|
||||
vi.mock("openai", () => {
|
||||
return {
|
||||
__esModule: true,
|
||||
default: vi.fn().mockImplementation(() => ({
|
||||
apiKey: "test-key",
|
||||
baseURL: "https://dashscope.aliyuncs.com/compatible-mode/v1",
|
||||
chat: {
|
||||
completions: {
|
||||
create: mockCreate,
|
||||
},
|
||||
},
|
||||
})),
|
||||
}
|
||||
})
|
||||
|
||||
import { promises as fs } from "node:fs"
|
||||
import { QwenCodeHandler } from "../qwen-code"
|
||||
import { NativeToolCallParser } from "../../../core/assistant-message/NativeToolCallParser"
|
||||
import type { ApiHandlerOptions } from "../../../shared/api"
|
||||
|
||||
describe("QwenCodeHandler Native Tools", () => {
|
||||
let handler: QwenCodeHandler
|
||||
let mockOptions: ApiHandlerOptions & { qwenCodeOauthPath?: string }
|
||||
|
||||
const testTools = [
|
||||
{
|
||||
type: "function" as const,
|
||||
function: {
|
||||
name: "test_tool",
|
||||
description: "A test tool",
|
||||
parameters: {
|
||||
type: "object",
|
||||
properties: {
|
||||
arg1: { type: "string", description: "First argument" },
|
||||
},
|
||||
required: ["arg1"],
|
||||
},
|
||||
},
|
||||
},
|
||||
]
|
||||
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks()
|
||||
|
||||
// Mock credentials file
|
||||
const mockCredentials = {
|
||||
access_token: "test-access-token",
|
||||
refresh_token: "test-refresh-token",
|
||||
token_type: "Bearer",
|
||||
expiry_date: Date.now() + 3600000, // 1 hour from now
|
||||
resource_url: "https://dashscope.aliyuncs.com/compatible-mode/v1",
|
||||
}
|
||||
;(fs.readFile as any).mockResolvedValue(JSON.stringify(mockCredentials))
|
||||
;(fs.writeFile as any).mockResolvedValue(undefined)
|
||||
|
||||
mockOptions = {
|
||||
apiModelId: "qwen3-coder-plus",
|
||||
}
|
||||
handler = new QwenCodeHandler(mockOptions)
|
||||
|
||||
// Clear NativeToolCallParser state before each test
|
||||
NativeToolCallParser.clearRawChunkState()
|
||||
})
|
||||
|
||||
describe("Native Tool Calling Support", () => {
|
||||
it("should include tools in request when model supports native tools and tools are provided", async () => {
|
||||
mockCreate.mockImplementationOnce(() => ({
|
||||
[Symbol.asyncIterator]: async function* () {
|
||||
yield {
|
||||
choices: [{ delta: { content: "Test response" } }],
|
||||
}
|
||||
},
|
||||
}))
|
||||
|
||||
const stream = handler.createMessage("test prompt", [], {
|
||||
taskId: "test-task-id",
|
||||
tools: testTools,
|
||||
})
|
||||
await stream.next()
|
||||
|
||||
expect(mockCreate).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
tools: expect.arrayContaining([
|
||||
expect.objectContaining({
|
||||
type: "function",
|
||||
function: expect.objectContaining({
|
||||
name: "test_tool",
|
||||
}),
|
||||
}),
|
||||
]),
|
||||
parallel_tool_calls: false,
|
||||
}),
|
||||
)
|
||||
})
|
||||
|
||||
it("should include tool_choice when provided", async () => {
|
||||
mockCreate.mockImplementationOnce(() => ({
|
||||
[Symbol.asyncIterator]: async function* () {
|
||||
yield {
|
||||
choices: [{ delta: { content: "Test response" } }],
|
||||
}
|
||||
},
|
||||
}))
|
||||
|
||||
const stream = handler.createMessage("test prompt", [], {
|
||||
taskId: "test-task-id",
|
||||
tools: testTools,
|
||||
tool_choice: "auto",
|
||||
})
|
||||
await stream.next()
|
||||
|
||||
expect(mockCreate).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
tool_choice: "auto",
|
||||
}),
|
||||
)
|
||||
})
|
||||
|
||||
it("should not include tools when toolProtocol is xml", async () => {
|
||||
mockCreate.mockImplementationOnce(() => ({
|
||||
[Symbol.asyncIterator]: async function* () {
|
||||
yield {
|
||||
choices: [{ delta: { content: "Test response" } }],
|
||||
}
|
||||
},
|
||||
}))
|
||||
|
||||
const stream = handler.createMessage("test prompt", [], {
|
||||
taskId: "test-task-id",
|
||||
tools: testTools,
|
||||
toolProtocol: "xml",
|
||||
})
|
||||
await stream.next()
|
||||
|
||||
const callArgs = mockCreate.mock.calls[mockCreate.mock.calls.length - 1][0]
|
||||
expect(callArgs).not.toHaveProperty("tools")
|
||||
expect(callArgs).not.toHaveProperty("tool_choice")
|
||||
})
|
||||
|
||||
it("should yield tool_call_partial chunks during streaming", async () => {
|
||||
mockCreate.mockImplementationOnce(() => ({
|
||||
[Symbol.asyncIterator]: async function* () {
|
||||
yield {
|
||||
choices: [
|
||||
{
|
||||
delta: {
|
||||
tool_calls: [
|
||||
{
|
||||
index: 0,
|
||||
id: "call_qwen_123",
|
||||
function: {
|
||||
name: "test_tool",
|
||||
arguments: '{"arg1":',
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
],
|
||||
}
|
||||
yield {
|
||||
choices: [
|
||||
{
|
||||
delta: {
|
||||
tool_calls: [
|
||||
{
|
||||
index: 0,
|
||||
function: {
|
||||
arguments: '"value"}',
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
],
|
||||
}
|
||||
},
|
||||
}))
|
||||
|
||||
const stream = handler.createMessage("test prompt", [], {
|
||||
taskId: "test-task-id",
|
||||
tools: testTools,
|
||||
})
|
||||
|
||||
const chunks = []
|
||||
for await (const chunk of stream) {
|
||||
chunks.push(chunk)
|
||||
}
|
||||
|
||||
expect(chunks).toContainEqual({
|
||||
type: "tool_call_partial",
|
||||
index: 0,
|
||||
id: "call_qwen_123",
|
||||
name: "test_tool",
|
||||
arguments: '{"arg1":',
|
||||
})
|
||||
|
||||
expect(chunks).toContainEqual({
|
||||
type: "tool_call_partial",
|
||||
index: 0,
|
||||
id: undefined,
|
||||
name: undefined,
|
||||
arguments: '"value"}',
|
||||
})
|
||||
})
|
||||
|
||||
it("should set parallel_tool_calls based on metadata", async () => {
|
||||
mockCreate.mockImplementationOnce(() => ({
|
||||
[Symbol.asyncIterator]: async function* () {
|
||||
yield {
|
||||
choices: [{ delta: { content: "Test response" } }],
|
||||
}
|
||||
},
|
||||
}))
|
||||
|
||||
const stream = handler.createMessage("test prompt", [], {
|
||||
taskId: "test-task-id",
|
||||
tools: testTools,
|
||||
parallelToolCalls: true,
|
||||
})
|
||||
await stream.next()
|
||||
|
||||
expect(mockCreate).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
parallel_tool_calls: true,
|
||||
}),
|
||||
)
|
||||
})
|
||||
|
||||
it("should yield tool_call_end events when finish_reason is tool_calls", async () => {
|
||||
mockCreate.mockImplementationOnce(() => ({
|
||||
[Symbol.asyncIterator]: async function* () {
|
||||
yield {
|
||||
choices: [
|
||||
{
|
||||
delta: {
|
||||
tool_calls: [
|
||||
{
|
||||
index: 0,
|
||||
id: "call_qwen_test",
|
||||
function: {
|
||||
name: "test_tool",
|
||||
arguments: '{"arg1":"value"}',
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
],
|
||||
}
|
||||
yield {
|
||||
choices: [
|
||||
{
|
||||
delta: {},
|
||||
finish_reason: "tool_calls",
|
||||
},
|
||||
],
|
||||
usage: { prompt_tokens: 10, completion_tokens: 5, total_tokens: 15 },
|
||||
}
|
||||
},
|
||||
}))
|
||||
|
||||
const stream = handler.createMessage("test prompt", [], {
|
||||
taskId: "test-task-id",
|
||||
tools: testTools,
|
||||
})
|
||||
|
||||
const chunks = []
|
||||
for await (const chunk of stream) {
|
||||
// Simulate what Task.ts does: when we receive tool_call_partial,
|
||||
// process it through NativeToolCallParser to populate rawChunkTracker
|
||||
if (chunk.type === "tool_call_partial") {
|
||||
NativeToolCallParser.processRawChunk({
|
||||
index: chunk.index,
|
||||
id: chunk.id,
|
||||
name: chunk.name,
|
||||
arguments: chunk.arguments,
|
||||
})
|
||||
}
|
||||
chunks.push(chunk)
|
||||
}
|
||||
|
||||
// Should have tool_call_partial and tool_call_end
|
||||
const partialChunks = chunks.filter((chunk) => chunk.type === "tool_call_partial")
|
||||
const endChunks = chunks.filter((chunk) => chunk.type === "tool_call_end")
|
||||
|
||||
expect(partialChunks).toHaveLength(1)
|
||||
expect(endChunks).toHaveLength(1)
|
||||
expect(endChunks[0].id).toBe("call_qwen_test")
|
||||
})
|
||||
|
||||
it("should preserve thinking block handling alongside tool calls", async () => {
|
||||
mockCreate.mockImplementationOnce(() => ({
|
||||
[Symbol.asyncIterator]: async function* () {
|
||||
yield {
|
||||
choices: [
|
||||
{
|
||||
delta: {
|
||||
reasoning_content: "Thinking about this...",
|
||||
},
|
||||
},
|
||||
],
|
||||
}
|
||||
yield {
|
||||
choices: [
|
||||
{
|
||||
delta: {
|
||||
tool_calls: [
|
||||
{
|
||||
index: 0,
|
||||
id: "call_after_think",
|
||||
function: {
|
||||
name: "test_tool",
|
||||
arguments: '{"arg1":"result"}',
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
],
|
||||
}
|
||||
yield {
|
||||
choices: [
|
||||
{
|
||||
delta: {},
|
||||
finish_reason: "tool_calls",
|
||||
},
|
||||
],
|
||||
}
|
||||
},
|
||||
}))
|
||||
|
||||
const stream = handler.createMessage("test prompt", [], {
|
||||
taskId: "test-task-id",
|
||||
tools: testTools,
|
||||
})
|
||||
|
||||
const chunks = []
|
||||
for await (const chunk of stream) {
|
||||
if (chunk.type === "tool_call_partial") {
|
||||
NativeToolCallParser.processRawChunk({
|
||||
index: chunk.index,
|
||||
id: chunk.id,
|
||||
name: chunk.name,
|
||||
arguments: chunk.arguments,
|
||||
})
|
||||
}
|
||||
chunks.push(chunk)
|
||||
}
|
||||
|
||||
// Should have reasoning, tool_call_partial, and tool_call_end
|
||||
const reasoningChunks = chunks.filter((chunk) => chunk.type === "reasoning")
|
||||
const partialChunks = chunks.filter((chunk) => chunk.type === "tool_call_partial")
|
||||
const endChunks = chunks.filter((chunk) => chunk.type === "tool_call_end")
|
||||
|
||||
expect(reasoningChunks).toHaveLength(1)
|
||||
expect(reasoningChunks[0].text).toBe("Thinking about this...")
|
||||
expect(partialChunks).toHaveLength(1)
|
||||
expect(endChunks).toHaveLength(1)
|
||||
})
|
||||
})
|
||||
})
|
||||
|
|
@ -6,6 +6,7 @@ import { type ModelInfo, openAiModelInfoSaneDefaults, LMSTUDIO_DEFAULT_TEMPERATU
|
|||
|
||||
import type { ApiHandlerOptions } from "../../shared/api"
|
||||
|
||||
import { NativeToolCallParser } from "../../core/assistant-message/NativeToolCallParser"
|
||||
import { XmlMatcher } from "../../utils/xml-matcher"
|
||||
|
||||
import { convertToOpenAiMessages } from "../transform/openai-format"
|
||||
|
|
@ -13,7 +14,7 @@ import { ApiStream } from "../transform/stream"
|
|||
|
||||
import { BaseProvider } from "./base-provider"
|
||||
import type { SingleCompletionHandler, ApiHandlerCreateMessageMetadata } from "../index"
|
||||
import { getModels, getModelsFromCache } from "./fetchers/modelCache"
|
||||
import { getModelsFromCache } from "./fetchers/modelCache"
|
||||
import { getApiRequestTimeout } from "./utils/timeout-config"
|
||||
import { handleOpenAIError } from "./utils/openai-error-handler"
|
||||
|
||||
|
|
@ -46,6 +47,9 @@ export class LmStudioHandler extends BaseProvider implements SingleCompletionHan
|
|||
...convertToOpenAiMessages(messages),
|
||||
]
|
||||
|
||||
// LM Studio always supports native tools (https://lmstudio.ai/docs/developer/core/tools)
|
||||
const useNativeTools = metadata?.tools && metadata.tools.length > 0 && metadata?.toolProtocol !== "xml"
|
||||
|
||||
// -------------------------
|
||||
// Track token usage
|
||||
// -------------------------
|
||||
|
|
@ -87,6 +91,9 @@ export class LmStudioHandler extends BaseProvider implements SingleCompletionHan
|
|||
messages: openAiMessages,
|
||||
temperature: this.options.modelTemperature ?? LMSTUDIO_DEFAULT_TEMPERATURE,
|
||||
stream: true,
|
||||
...(useNativeTools && { tools: this.convertToolsForOpenAI(metadata.tools) }),
|
||||
...(useNativeTools && metadata.tool_choice && { tool_choice: metadata.tool_choice }),
|
||||
...(useNativeTools && { parallel_tool_calls: metadata?.parallelToolCalls ?? false }),
|
||||
}
|
||||
|
||||
if (this.options.lmStudioSpeculativeDecodingEnabled && this.options.lmStudioDraftModelId) {
|
||||
|
|
@ -111,6 +118,7 @@ export class LmStudioHandler extends BaseProvider implements SingleCompletionHan
|
|||
|
||||
for await (const chunk of results) {
|
||||
const delta = chunk.choices[0]?.delta
|
||||
const finishReason = chunk.choices[0]?.finish_reason
|
||||
|
||||
if (delta?.content) {
|
||||
assistantText += delta.content
|
||||
|
|
@ -118,6 +126,27 @@ export class LmStudioHandler extends BaseProvider implements SingleCompletionHan
|
|||
yield processedChunk
|
||||
}
|
||||
}
|
||||
|
||||
// Handle tool calls in stream - emit partial chunks for NativeToolCallParser
|
||||
if (delta?.tool_calls) {
|
||||
for (const toolCall of delta.tool_calls) {
|
||||
yield {
|
||||
type: "tool_call_partial",
|
||||
index: toolCall.index,
|
||||
id: toolCall.id,
|
||||
name: toolCall.function?.name,
|
||||
arguments: toolCall.function?.arguments,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Process finish_reason to emit tool_call_end events
|
||||
if (finishReason) {
|
||||
const endEvents = NativeToolCallParser.processFinishReason(finishReason)
|
||||
for (const event of endEvents) {
|
||||
yield event
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (const processedChunk of matcher.final()) {
|
||||
|
|
|
|||
|
|
@ -8,11 +8,13 @@ import { type ModelInfo, type QwenCodeModelId, qwenCodeModels, qwenCodeDefaultMo
|
|||
|
||||
import type { ApiHandlerOptions } from "../../shared/api"
|
||||
|
||||
import { NativeToolCallParser } from "../../core/assistant-message/NativeToolCallParser"
|
||||
|
||||
import { convertToOpenAiMessages } from "../transform/openai-format"
|
||||
import { ApiStream } from "../transform/stream"
|
||||
|
||||
import { BaseProvider } from "./base-provider"
|
||||
import type { SingleCompletionHandler } from "../index"
|
||||
import type { SingleCompletionHandler, ApiHandlerCreateMessageMetadata } from "../index"
|
||||
|
||||
const QWEN_OAUTH_BASE_URL = "https://chat.qwen.ai"
|
||||
const QWEN_OAUTH_TOKEN_ENDPOINT = `${QWEN_OAUTH_BASE_URL}/api/v1/oauth2/token`
|
||||
|
|
@ -201,11 +203,20 @@ export class QwenCodeHandler extends BaseProvider implements SingleCompletionHan
|
|||
}
|
||||
}
|
||||
|
||||
override async *createMessage(systemPrompt: string, messages: Anthropic.Messages.MessageParam[]): ApiStream {
|
||||
override async *createMessage(
|
||||
systemPrompt: string,
|
||||
messages: Anthropic.Messages.MessageParam[],
|
||||
metadata?: ApiHandlerCreateMessageMetadata,
|
||||
): ApiStream {
|
||||
await this.ensureAuthenticated()
|
||||
const client = this.ensureClient()
|
||||
const model = this.getModel()
|
||||
|
||||
// Check if model supports native tools and tools are provided with native protocol
|
||||
const supportsNativeTools = model.info.supportsNativeTools ?? false
|
||||
const useNativeTools =
|
||||
supportsNativeTools && metadata?.tools && metadata.tools.length > 0 && metadata?.toolProtocol !== "xml"
|
||||
|
||||
const systemMessage: OpenAI.Chat.ChatCompletionSystemMessageParam = {
|
||||
role: "system",
|
||||
content: systemPrompt,
|
||||
|
|
@ -220,6 +231,9 @@ export class QwenCodeHandler extends BaseProvider implements SingleCompletionHan
|
|||
stream: true,
|
||||
stream_options: { include_usage: true },
|
||||
max_completion_tokens: model.info.maxTokens,
|
||||
...(useNativeTools && { tools: this.convertToolsForOpenAI(metadata.tools) }),
|
||||
...(useNativeTools && metadata.tool_choice && { tool_choice: metadata.tool_choice }),
|
||||
...(useNativeTools && { parallel_tool_calls: metadata?.parallelToolCalls ?? false }),
|
||||
}
|
||||
|
||||
const stream = await this.callApiWithRetry(() => client.chat.completions.create(requestOptions))
|
||||
|
|
@ -228,6 +242,7 @@ export class QwenCodeHandler extends BaseProvider implements SingleCompletionHan
|
|||
|
||||
for await (const apiChunk of stream) {
|
||||
const delta = apiChunk.choices[0]?.delta ?? {}
|
||||
const finishReason = apiChunk.choices[0]?.finish_reason
|
||||
|
||||
if (delta.content) {
|
||||
let newText = delta.content
|
||||
|
|
@ -274,6 +289,27 @@ export class QwenCodeHandler extends BaseProvider implements SingleCompletionHan
|
|||
}
|
||||
}
|
||||
|
||||
// Handle tool calls in stream - emit partial chunks for NativeToolCallParser
|
||||
if (delta.tool_calls) {
|
||||
for (const toolCall of delta.tool_calls) {
|
||||
yield {
|
||||
type: "tool_call_partial",
|
||||
index: toolCall.index,
|
||||
id: toolCall.id,
|
||||
name: toolCall.function?.name,
|
||||
arguments: toolCall.function?.arguments,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Process finish_reason to emit tool_call_end events
|
||||
if (finishReason) {
|
||||
const endEvents = NativeToolCallParser.processFinishReason(finishReason)
|
||||
for (const event of endEvents) {
|
||||
yield event
|
||||
}
|
||||
}
|
||||
|
||||
if (apiChunk.usage) {
|
||||
yield {
|
||||
type: "usage",
|
||||
|
|
|
|||
|
|
@ -29,6 +29,7 @@ import {
|
|||
basetenModels,
|
||||
qwenCodeModels,
|
||||
litellmDefaultModelInfo,
|
||||
lMStudioDefaultModelInfo,
|
||||
BEDROCK_1M_CONTEXT_MODEL_IDS,
|
||||
isDynamicProvider,
|
||||
getProviderDefaultModelId,
|
||||
|
|
@ -300,10 +301,16 @@ function getSelectedModel({
|
|||
}
|
||||
case "lmstudio": {
|
||||
const id = apiConfiguration.lmStudioModelId ?? ""
|
||||
const info = lmStudioModels && lmStudioModels[apiConfiguration.lmStudioModelId!]
|
||||
const modelInfo = lmStudioModels && lmStudioModels[apiConfiguration.lmStudioModelId!]
|
||||
// Only merge native tool call defaults, not prices or other model-specific info
|
||||
const nativeToolDefaults = {
|
||||
supportsNativeTools: lMStudioDefaultModelInfo.supportsNativeTools,
|
||||
defaultToolProtocol: lMStudioDefaultModelInfo.defaultToolProtocol,
|
||||
}
|
||||
const info = modelInfo ? { ...nativeToolDefaults, ...modelInfo } : undefined
|
||||
return {
|
||||
id,
|
||||
info: info || undefined,
|
||||
info,
|
||||
}
|
||||
}
|
||||
case "deepinfra": {
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue