mirror of
https://github.com/RooVetGit/Roo-Code.git
synced 2026-08-28 05:27:24 +00:00
refactor: migrate featherless provider to AI SDK (#11265)
* refactor: migrate featherless provider to AI SDK * fix: merge consecutive same-role messages in featherless R1 path convertToAiSdkMessages does not merge consecutive same-role messages like convertToR1Format did. When the system prompt is prepended as a user message and the conversation already starts with a user message, DeepSeek R1 can reject the request. Add mergeConsecutiveSameRoleMessages helper that collapses adjacent Anthropic messages sharing the same role before AI SDK conversion. Includes a test that verifies no two successive messages share a role.
This commit is contained in:
Daniel
GitHub
parent
a3a9048741
commit
36a986db1d
2 changed files with 418 additions and 294 deletions
|
|
@ -1,259 +1,356 @@
|
|||
// npx vitest run api/providers/__tests__/featherless.spec.ts
|
||||
|
||||
import { Anthropic } from "@anthropic-ai/sdk"
|
||||
import OpenAI from "openai"
|
||||
const { mockStreamText, mockGenerateText } = vi.hoisted(() => ({
|
||||
mockStreamText: vi.fn(),
|
||||
mockGenerateText: vi.fn(),
|
||||
}))
|
||||
|
||||
vi.mock("ai", async (importOriginal) => {
|
||||
const actual = await importOriginal<typeof import("ai")>()
|
||||
return {
|
||||
...actual,
|
||||
streamText: mockStreamText,
|
||||
generateText: mockGenerateText,
|
||||
}
|
||||
})
|
||||
|
||||
vi.mock("@ai-sdk/openai-compatible", () => ({
|
||||
createOpenAICompatible: vi.fn(() => {
|
||||
return vi.fn(() => ({
|
||||
modelId: "featherless-model",
|
||||
provider: "Featherless",
|
||||
}))
|
||||
}),
|
||||
}))
|
||||
|
||||
import type { Anthropic } from "@anthropic-ai/sdk"
|
||||
|
||||
import { type FeatherlessModelId, featherlessDefaultModelId, featherlessModels } from "@roo-code/types"
|
||||
|
||||
import type { ApiHandlerOptions } from "../../../shared/api"
|
||||
|
||||
import { FeatherlessHandler } from "../featherless"
|
||||
|
||||
// Create mock functions
|
||||
const mockCreate = vi.fn()
|
||||
|
||||
// Mock OpenAI module
|
||||
vi.mock("openai", () => ({
|
||||
default: vi.fn(() => ({
|
||||
chat: {
|
||||
completions: {
|
||||
create: mockCreate,
|
||||
},
|
||||
},
|
||||
})),
|
||||
}))
|
||||
|
||||
describe("FeatherlessHandler", () => {
|
||||
let handler: FeatherlessHandler
|
||||
let mockOptions: ApiHandlerOptions
|
||||
|
||||
beforeEach(() => {
|
||||
mockOptions = {
|
||||
featherlessApiKey: "test-api-key",
|
||||
}
|
||||
handler = new FeatherlessHandler(mockOptions)
|
||||
vi.clearAllMocks()
|
||||
// Set up default mock implementation
|
||||
mockCreate.mockImplementation(async () => ({
|
||||
[Symbol.asyncIterator]: async function* () {
|
||||
yield {
|
||||
choices: [
|
||||
{
|
||||
delta: { content: "Test response" },
|
||||
index: 0,
|
||||
},
|
||||
],
|
||||
usage: null,
|
||||
}
|
||||
yield {
|
||||
choices: [
|
||||
{
|
||||
delta: {},
|
||||
index: 0,
|
||||
},
|
||||
],
|
||||
usage: {
|
||||
prompt_tokens: 10,
|
||||
completion_tokens: 5,
|
||||
total_tokens: 15,
|
||||
})
|
||||
|
||||
describe("constructor", () => {
|
||||
it("should initialize with provided options", () => {
|
||||
expect(handler).toBeInstanceOf(FeatherlessHandler)
|
||||
expect(handler.getModel().id).toBe(featherlessDefaultModelId)
|
||||
})
|
||||
|
||||
it("should use specified model ID when provided", () => {
|
||||
const testModelId: FeatherlessModelId = "moonshotai/Kimi-K2-Instruct"
|
||||
const handlerWithModel = new FeatherlessHandler({
|
||||
apiModelId: testModelId,
|
||||
featherlessApiKey: "test-api-key",
|
||||
})
|
||||
expect(handlerWithModel.getModel().id).toBe(testModelId)
|
||||
})
|
||||
})
|
||||
|
||||
describe("getModel", () => {
|
||||
it("should return default model when no model is specified", () => {
|
||||
const model = handler.getModel()
|
||||
expect(model.id).toBe(featherlessDefaultModelId)
|
||||
expect(model.info).toEqual(expect.objectContaining(featherlessModels[featherlessDefaultModelId]))
|
||||
})
|
||||
|
||||
it("should return specified model when valid model is provided", () => {
|
||||
const testModelId: FeatherlessModelId = "moonshotai/Kimi-K2-Instruct"
|
||||
const handlerWithModel = new FeatherlessHandler({
|
||||
apiModelId: testModelId,
|
||||
featherlessApiKey: "test-api-key",
|
||||
})
|
||||
const model = handlerWithModel.getModel()
|
||||
expect(model.id).toBe(testModelId)
|
||||
expect(model.info).toEqual(expect.objectContaining(featherlessModels[testModelId]))
|
||||
})
|
||||
|
||||
it("should use default temperature for non-DeepSeek models", () => {
|
||||
const testModelId: FeatherlessModelId = "moonshotai/Kimi-K2-Instruct"
|
||||
const handlerWithModel = new FeatherlessHandler({
|
||||
apiModelId: testModelId,
|
||||
featherlessApiKey: "test-api-key",
|
||||
})
|
||||
const model = handlerWithModel.getModel()
|
||||
expect(model.temperature).toBe(0.5)
|
||||
})
|
||||
|
||||
it("should include model parameters from getModelParams", () => {
|
||||
const model = handler.getModel()
|
||||
expect(model).toHaveProperty("temperature")
|
||||
expect(model).toHaveProperty("maxTokens")
|
||||
})
|
||||
})
|
||||
|
||||
describe("createMessage", () => {
|
||||
const systemPrompt = "You are a helpful assistant."
|
||||
const messages: Anthropic.Messages.MessageParam[] = [
|
||||
{
|
||||
role: "user",
|
||||
content: [
|
||||
{
|
||||
type: "text" as const,
|
||||
text: "Hello!",
|
||||
},
|
||||
}
|
||||
],
|
||||
},
|
||||
}))
|
||||
handler = new FeatherlessHandler({ featherlessApiKey: "test-key" })
|
||||
})
|
||||
]
|
||||
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks()
|
||||
})
|
||||
|
||||
it("should use the correct Featherless base URL", () => {
|
||||
new FeatherlessHandler({ featherlessApiKey: "test-featherless-api-key" })
|
||||
expect(OpenAI).toHaveBeenCalledWith(expect.objectContaining({ baseURL: "https://api.featherless.ai/v1" }))
|
||||
})
|
||||
|
||||
it("should use the provided API key", () => {
|
||||
const featherlessApiKey = "test-featherless-api-key"
|
||||
new FeatherlessHandler({ featherlessApiKey })
|
||||
expect(OpenAI).toHaveBeenCalledWith(expect.objectContaining({ apiKey: featherlessApiKey }))
|
||||
})
|
||||
|
||||
it("should handle reasoning format from models that use <think> tags", async () => {
|
||||
// Override the mock for this specific test
|
||||
mockCreate.mockImplementationOnce(async () => ({
|
||||
[Symbol.asyncIterator]: async function* () {
|
||||
yield {
|
||||
choices: [
|
||||
{
|
||||
delta: { content: "<think>Thinking..." },
|
||||
index: 0,
|
||||
},
|
||||
],
|
||||
usage: null,
|
||||
}
|
||||
yield {
|
||||
choices: [
|
||||
{
|
||||
delta: { content: "</think>Hello" },
|
||||
index: 0,
|
||||
},
|
||||
],
|
||||
usage: null,
|
||||
}
|
||||
yield {
|
||||
choices: [
|
||||
{
|
||||
delta: {},
|
||||
index: 0,
|
||||
},
|
||||
],
|
||||
usage: { prompt_tokens: 10, completion_tokens: 5 },
|
||||
}
|
||||
},
|
||||
}))
|
||||
|
||||
const systemPrompt = "You are a helpful assistant."
|
||||
const messages: Anthropic.Messages.MessageParam[] = [{ role: "user", content: "Hi" }]
|
||||
vi.spyOn(handler, "getModel").mockReturnValue({
|
||||
id: "some-reasoning-model",
|
||||
info: { maxTokens: 1024, temperature: 0.7 },
|
||||
} as any)
|
||||
|
||||
const stream = handler.createMessage(systemPrompt, messages)
|
||||
const chunks = []
|
||||
for await (const chunk of stream) {
|
||||
chunks.push(chunk)
|
||||
}
|
||||
|
||||
expect(chunks[0]).toEqual({ type: "reasoning", text: "Thinking..." })
|
||||
expect(chunks[1]).toEqual({ type: "text", text: "Hello" })
|
||||
expect(chunks[2]).toMatchObject({ type: "usage", inputTokens: 10, outputTokens: 5 })
|
||||
})
|
||||
|
||||
it("should fall back to base provider for non-DeepSeek models", async () => {
|
||||
// Use default mock implementation which returns text content
|
||||
const systemPrompt = "You are a helpful assistant."
|
||||
const messages: Anthropic.Messages.MessageParam[] = [{ role: "user", content: "Hi" }]
|
||||
vi.spyOn(handler, "getModel").mockReturnValue({
|
||||
id: "some-other-model",
|
||||
info: { maxTokens: 1024, temperature: 0.7 },
|
||||
} as any)
|
||||
|
||||
const stream = handler.createMessage(systemPrompt, messages)
|
||||
const chunks = []
|
||||
for await (const chunk of stream) {
|
||||
chunks.push(chunk)
|
||||
}
|
||||
|
||||
expect(chunks[0]).toEqual({ type: "text", text: "Test response" })
|
||||
expect(chunks[1]).toMatchObject({ type: "usage", inputTokens: 10, outputTokens: 5 })
|
||||
})
|
||||
|
||||
it("should return default model when no model is specified", () => {
|
||||
const model = handler.getModel()
|
||||
expect(model.id).toBe(featherlessDefaultModelId)
|
||||
expect(model.info).toEqual(expect.objectContaining(featherlessModels[featherlessDefaultModelId]))
|
||||
})
|
||||
|
||||
it("should return specified model when valid model is provided", () => {
|
||||
const testModelId: FeatherlessModelId = "moonshotai/Kimi-K2-Instruct"
|
||||
const handlerWithModel = new FeatherlessHandler({
|
||||
apiModelId: testModelId,
|
||||
featherlessApiKey: "test-featherless-api-key",
|
||||
})
|
||||
const model = handlerWithModel.getModel()
|
||||
expect(model.id).toBe(testModelId)
|
||||
expect(model.info).toEqual(expect.objectContaining(featherlessModels[testModelId]))
|
||||
})
|
||||
|
||||
it("completePrompt method should return text from Featherless API", async () => {
|
||||
const expectedResponse = "This is a test response from Featherless"
|
||||
mockCreate.mockResolvedValueOnce({ choices: [{ message: { content: expectedResponse } }] })
|
||||
const result = await handler.completePrompt("test prompt")
|
||||
expect(result).toBe(expectedResponse)
|
||||
})
|
||||
|
||||
it("should handle errors in completePrompt", async () => {
|
||||
const errorMessage = "Featherless API error"
|
||||
mockCreate.mockRejectedValueOnce(new Error(errorMessage))
|
||||
await expect(handler.completePrompt("test prompt")).rejects.toThrow(
|
||||
`Featherless completion error: ${errorMessage}`,
|
||||
)
|
||||
})
|
||||
|
||||
it("createMessage should yield text content from stream", async () => {
|
||||
const testContent = "This is test content from Featherless stream"
|
||||
|
||||
mockCreate.mockImplementationOnce(() => {
|
||||
return {
|
||||
[Symbol.asyncIterator]: () => ({
|
||||
next: vi
|
||||
.fn()
|
||||
.mockResolvedValueOnce({
|
||||
done: false,
|
||||
value: { choices: [{ delta: { content: testContent } }] },
|
||||
})
|
||||
.mockResolvedValueOnce({ done: true }),
|
||||
}),
|
||||
it("should handle streaming responses for non-R1 models", async () => {
|
||||
async function* mockFullStream() {
|
||||
yield { type: "text-delta", text: "Test response" }
|
||||
}
|
||||
})
|
||||
|
||||
const stream = handler.createMessage("system prompt", [])
|
||||
const firstChunk = await stream.next()
|
||||
const mockUsage = Promise.resolve({
|
||||
inputTokens: 10,
|
||||
outputTokens: 5,
|
||||
})
|
||||
|
||||
expect(firstChunk.done).toBe(false)
|
||||
expect(firstChunk.value).toEqual({ type: "text", text: testContent })
|
||||
})
|
||||
mockStreamText.mockReturnValue({
|
||||
fullStream: mockFullStream(),
|
||||
usage: mockUsage,
|
||||
})
|
||||
|
||||
it("createMessage should yield usage data from stream", async () => {
|
||||
mockCreate.mockImplementationOnce(() => {
|
||||
return {
|
||||
[Symbol.asyncIterator]: () => ({
|
||||
next: vi
|
||||
.fn()
|
||||
.mockResolvedValueOnce({
|
||||
done: false,
|
||||
value: { choices: [{ delta: {} }], usage: { prompt_tokens: 10, completion_tokens: 20 } },
|
||||
})
|
||||
.mockResolvedValueOnce({ done: true }),
|
||||
}),
|
||||
const stream = handler.createMessage(systemPrompt, messages)
|
||||
const chunks: any[] = []
|
||||
for await (const chunk of stream) {
|
||||
chunks.push(chunk)
|
||||
}
|
||||
|
||||
expect(chunks.length).toBeGreaterThan(0)
|
||||
const textChunks = chunks.filter((chunk) => chunk.type === "text")
|
||||
expect(textChunks).toHaveLength(1)
|
||||
expect(textChunks[0].text).toBe("Test response")
|
||||
})
|
||||
|
||||
const stream = handler.createMessage("system prompt", [])
|
||||
const firstChunk = await stream.next()
|
||||
it("should include usage information", async () => {
|
||||
async function* mockFullStream() {
|
||||
yield { type: "text-delta", text: "Test response" }
|
||||
}
|
||||
|
||||
expect(firstChunk.done).toBe(false)
|
||||
expect(firstChunk.value).toMatchObject({ type: "usage", inputTokens: 10, outputTokens: 20 })
|
||||
const mockUsage = Promise.resolve({
|
||||
inputTokens: 10,
|
||||
outputTokens: 5,
|
||||
})
|
||||
|
||||
mockStreamText.mockReturnValue({
|
||||
fullStream: mockFullStream(),
|
||||
usage: mockUsage,
|
||||
})
|
||||
|
||||
const stream = handler.createMessage(systemPrompt, messages)
|
||||
const chunks: any[] = []
|
||||
for await (const chunk of stream) {
|
||||
chunks.push(chunk)
|
||||
}
|
||||
|
||||
const usageChunks = chunks.filter((chunk) => chunk.type === "usage")
|
||||
expect(usageChunks.length).toBeGreaterThan(0)
|
||||
expect(usageChunks[0].inputTokens).toBe(10)
|
||||
expect(usageChunks[0].outputTokens).toBe(5)
|
||||
})
|
||||
|
||||
it("should handle reasoning format from DeepSeek-R1 models using TagMatcher", async () => {
|
||||
async function* mockFullStream() {
|
||||
yield { type: "text-delta", text: "<think>Thinking..." }
|
||||
yield { type: "text-delta", text: "</think>Hello" }
|
||||
}
|
||||
|
||||
const mockUsage = Promise.resolve({
|
||||
inputTokens: 10,
|
||||
outputTokens: 5,
|
||||
})
|
||||
|
||||
mockStreamText.mockReturnValue({
|
||||
fullStream: mockFullStream(),
|
||||
usage: mockUsage,
|
||||
})
|
||||
|
||||
vi.spyOn(handler, "getModel").mockReturnValue({
|
||||
id: "some-DeepSeek-R1-model",
|
||||
info: { maxTokens: 1024, temperature: 0.6 },
|
||||
maxTokens: 1024,
|
||||
temperature: 0.6,
|
||||
} as any)
|
||||
|
||||
const stream = handler.createMessage(systemPrompt, messages)
|
||||
const chunks: any[] = []
|
||||
for await (const chunk of stream) {
|
||||
chunks.push(chunk)
|
||||
}
|
||||
|
||||
expect(chunks[0]).toEqual({ type: "reasoning", text: "Thinking..." })
|
||||
expect(chunks[1]).toEqual({ type: "text", text: "Hello" })
|
||||
expect(chunks[2]).toMatchObject({ type: "usage", inputTokens: 10, outputTokens: 5 })
|
||||
})
|
||||
|
||||
it("should delegate to super.createMessage for non-DeepSeek-R1 models", async () => {
|
||||
async function* mockFullStream() {
|
||||
yield { type: "text-delta", text: "Standard response" }
|
||||
}
|
||||
|
||||
const mockUsage = Promise.resolve({
|
||||
inputTokens: 15,
|
||||
outputTokens: 8,
|
||||
})
|
||||
|
||||
mockStreamText.mockReturnValue({
|
||||
fullStream: mockFullStream(),
|
||||
usage: mockUsage,
|
||||
})
|
||||
|
||||
vi.spyOn(handler, "getModel").mockReturnValue({
|
||||
id: "some-other-model",
|
||||
info: { maxTokens: 1024, temperature: 0.5 },
|
||||
maxTokens: 1024,
|
||||
temperature: 0.5,
|
||||
} as any)
|
||||
|
||||
const stream = handler.createMessage(systemPrompt, messages)
|
||||
const chunks: any[] = []
|
||||
for await (const chunk of stream) {
|
||||
chunks.push(chunk)
|
||||
}
|
||||
|
||||
expect(chunks[0]).toEqual({ type: "text", text: "Standard response" })
|
||||
expect(chunks[1]).toMatchObject({ type: "usage", inputTokens: 15, outputTokens: 8 })
|
||||
})
|
||||
|
||||
it("should pass correct model to streamText for R1 path", async () => {
|
||||
async function* mockFullStream() {
|
||||
yield { type: "text-delta", text: "response" }
|
||||
}
|
||||
|
||||
mockStreamText.mockReturnValue({
|
||||
fullStream: mockFullStream(),
|
||||
usage: Promise.resolve({ inputTokens: 0, outputTokens: 0 }),
|
||||
})
|
||||
|
||||
vi.spyOn(handler, "getModel").mockReturnValue({
|
||||
id: "some-DeepSeek-R1-model",
|
||||
info: { maxTokens: 2048, temperature: 0.6 },
|
||||
maxTokens: 2048,
|
||||
temperature: 0.6,
|
||||
} as any)
|
||||
|
||||
const stream = handler.createMessage(systemPrompt, messages)
|
||||
// Consume stream
|
||||
for await (const _ of stream) {
|
||||
// drain
|
||||
}
|
||||
|
||||
expect(mockStreamText).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
temperature: 0.6,
|
||||
}),
|
||||
)
|
||||
})
|
||||
|
||||
it("should not pass system prompt to streamText for R1 path", async () => {
|
||||
async function* mockFullStream() {
|
||||
yield { type: "text-delta", text: "response" }
|
||||
}
|
||||
|
||||
mockStreamText.mockReturnValue({
|
||||
fullStream: mockFullStream(),
|
||||
usage: Promise.resolve({ inputTokens: 0, outputTokens: 0 }),
|
||||
})
|
||||
|
||||
vi.spyOn(handler, "getModel").mockReturnValue({
|
||||
id: "some-DeepSeek-R1-model",
|
||||
info: { maxTokens: 2048, temperature: 0.6 },
|
||||
maxTokens: 2048,
|
||||
temperature: 0.6,
|
||||
} as any)
|
||||
|
||||
const stream = handler.createMessage(systemPrompt, messages)
|
||||
for await (const _ of stream) {
|
||||
// drain
|
||||
}
|
||||
|
||||
const callArgs = mockStreamText.mock.calls[0][0]
|
||||
expect(callArgs.system).toBeUndefined()
|
||||
expect(callArgs.messages).toBeDefined()
|
||||
})
|
||||
|
||||
it("should merge consecutive user messages in R1 path to avoid DeepSeek rejection", async () => {
|
||||
async function* mockFullStream() {
|
||||
yield { type: "text-delta", text: "response" }
|
||||
}
|
||||
|
||||
mockStreamText.mockReturnValue({
|
||||
fullStream: mockFullStream(),
|
||||
usage: Promise.resolve({ inputTokens: 0, outputTokens: 0 }),
|
||||
})
|
||||
|
||||
vi.spyOn(handler, "getModel").mockReturnValue({
|
||||
id: "some-DeepSeek-R1-model",
|
||||
info: { maxTokens: 2048, temperature: 0.6 },
|
||||
maxTokens: 2048,
|
||||
temperature: 0.6,
|
||||
} as any)
|
||||
|
||||
// messages starts with a user message, so after prepending the system
|
||||
// prompt as a user message we'd have two consecutive user messages.
|
||||
const userFirstMessages: Anthropic.Messages.MessageParam[] = [
|
||||
{ role: "user", content: "Hello!" },
|
||||
{ role: "assistant", content: "Hi there" },
|
||||
{ role: "user", content: "Follow-up" },
|
||||
]
|
||||
|
||||
const stream = handler.createMessage(systemPrompt, userFirstMessages)
|
||||
for await (const _ of stream) {
|
||||
// drain
|
||||
}
|
||||
|
||||
const callArgs = mockStreamText.mock.calls[0][0]
|
||||
const passedMessages = callArgs.messages
|
||||
|
||||
// Verify no two consecutive messages share the same role
|
||||
for (let i = 1; i < passedMessages.length; i++) {
|
||||
expect(passedMessages[i].role).not.toBe(passedMessages[i - 1].role)
|
||||
}
|
||||
|
||||
// The system prompt and first user message should be merged into a single user message
|
||||
expect(passedMessages[0].role).toBe("user")
|
||||
expect(passedMessages[1].role).toBe("assistant")
|
||||
expect(passedMessages[2].role).toBe("user")
|
||||
expect(passedMessages).toHaveLength(3)
|
||||
})
|
||||
})
|
||||
|
||||
it("createMessage should pass correct parameters to Featherless client", async () => {
|
||||
const modelId: FeatherlessModelId = "moonshotai/Kimi-K2-Instruct"
|
||||
describe("completePrompt", () => {
|
||||
it("should complete a prompt using generateText", async () => {
|
||||
mockGenerateText.mockResolvedValue({
|
||||
text: "Test completion from Featherless",
|
||||
})
|
||||
|
||||
// Clear previous mocks and set up new implementation
|
||||
mockCreate.mockClear()
|
||||
mockCreate.mockImplementationOnce(async () => ({
|
||||
[Symbol.asyncIterator]: async function* () {
|
||||
// Empty stream for this test
|
||||
},
|
||||
}))
|
||||
const result = await handler.completePrompt("Test prompt")
|
||||
|
||||
const handlerWithModel = new FeatherlessHandler({
|
||||
apiModelId: modelId,
|
||||
featherlessApiKey: "test-featherless-api-key",
|
||||
expect(result).toBe("Test completion from Featherless")
|
||||
expect(mockGenerateText).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
prompt: "Test prompt",
|
||||
}),
|
||||
)
|
||||
})
|
||||
|
||||
const systemPrompt = "Test system prompt for Featherless"
|
||||
const messages: Anthropic.Messages.MessageParam[] = [{ role: "user", content: "Test message for Featherless" }]
|
||||
|
||||
const messageGenerator = handlerWithModel.createMessage(systemPrompt, messages)
|
||||
await messageGenerator.next()
|
||||
|
||||
expect(mockCreate).toHaveBeenCalled()
|
||||
const callArgs = mockCreate.mock.calls[0][0]
|
||||
expect(callArgs.model).toBe(modelId)
|
||||
})
|
||||
|
||||
it("should use default temperature for non-DeepSeek models", () => {
|
||||
const testModelId: FeatherlessModelId = "moonshotai/Kimi-K2-Instruct"
|
||||
const handlerWithModel = new FeatherlessHandler({
|
||||
apiModelId: testModelId,
|
||||
featherlessApiKey: "test-featherless-api-key",
|
||||
describe("isAiSdkProvider", () => {
|
||||
it("should return true", () => {
|
||||
expect(handler.isAiSdkProvider()).toBe(true)
|
||||
})
|
||||
const model = handlerWithModel.getModel()
|
||||
expect(model.info.temperature).toBe(0.5)
|
||||
})
|
||||
})
|
||||
|
|
|
|||
|
|
@ -1,55 +1,88 @@
|
|||
import {
|
||||
DEEP_SEEK_DEFAULT_TEMPERATURE,
|
||||
type FeatherlessModelId,
|
||||
featherlessDefaultModelId,
|
||||
featherlessModels,
|
||||
} from "@roo-code/types"
|
||||
import { Anthropic } from "@anthropic-ai/sdk"
|
||||
import OpenAI from "openai"
|
||||
import { streamText } from "ai"
|
||||
|
||||
import { DEEP_SEEK_DEFAULT_TEMPERATURE, featherlessDefaultModelId, featherlessModels } from "@roo-code/types"
|
||||
|
||||
import type { ApiHandlerOptions } from "../../shared/api"
|
||||
import { TagMatcher } from "../../utils/tag-matcher"
|
||||
import { convertToR1Format } from "../transform/r1-format"
|
||||
import { convertToOpenAiMessages } from "../transform/openai-format"
|
||||
import { convertToAiSdkMessages, handleAiSdkError } from "../transform/ai-sdk"
|
||||
import { ApiStream } from "../transform/stream"
|
||||
import { getModelParams } from "../transform/model-params"
|
||||
|
||||
import type { ApiHandlerCreateMessageMetadata } from "../index"
|
||||
import { BaseOpenAiCompatibleProvider } from "./base-openai-compatible-provider"
|
||||
import { OpenAICompatibleHandler, OpenAICompatibleConfig } from "./openai-compatible"
|
||||
|
||||
export class FeatherlessHandler extends BaseOpenAiCompatibleProvider<FeatherlessModelId> {
|
||||
constructor(options: ApiHandlerOptions) {
|
||||
super({
|
||||
...options,
|
||||
providerName: "Featherless",
|
||||
baseURL: "https://api.featherless.ai/v1",
|
||||
apiKey: options.featherlessApiKey,
|
||||
defaultProviderModelId: featherlessDefaultModelId,
|
||||
providerModels: featherlessModels,
|
||||
defaultTemperature: 0.5,
|
||||
})
|
||||
/**
|
||||
* Merge consecutive Anthropic messages that share the same role.
|
||||
* DeepSeek R1 does not support successive messages with the same role,
|
||||
* so this is needed when the system prompt is injected as a user message
|
||||
* before the existing conversation (which may also start with a user message).
|
||||
*/
|
||||
function mergeConsecutiveSameRoleMessages(
|
||||
messages: Anthropic.Messages.MessageParam[],
|
||||
): Anthropic.Messages.MessageParam[] {
|
||||
if (messages.length <= 1) {
|
||||
return messages
|
||||
}
|
||||
|
||||
private getCompletionParams(
|
||||
systemPrompt: string,
|
||||
messages: Anthropic.Messages.MessageParam[],
|
||||
): OpenAI.Chat.Completions.ChatCompletionCreateParamsStreaming {
|
||||
const {
|
||||
id: model,
|
||||
info: { maxTokens: max_tokens },
|
||||
} = this.getModel()
|
||||
const merged: Anthropic.Messages.MessageParam[] = []
|
||||
|
||||
const temperature = this.options.modelTemperature ?? this.getModel().info.temperature
|
||||
for (const msg of messages) {
|
||||
const prev = merged[merged.length - 1]
|
||||
|
||||
return {
|
||||
model,
|
||||
max_tokens,
|
||||
temperature,
|
||||
messages: [{ role: "system", content: systemPrompt }, ...convertToOpenAiMessages(messages)],
|
||||
stream: true,
|
||||
stream_options: { include_usage: true },
|
||||
if (prev && prev.role === msg.role) {
|
||||
const prevBlocks: Anthropic.Messages.ContentBlockParam[] =
|
||||
typeof prev.content === "string" ? [{ type: "text", text: prev.content }] : prev.content
|
||||
const currBlocks: Anthropic.Messages.ContentBlockParam[] =
|
||||
typeof msg.content === "string" ? [{ type: "text", text: msg.content }] : msg.content
|
||||
|
||||
merged[merged.length - 1] = {
|
||||
role: prev.role,
|
||||
content: [...prevBlocks, ...currBlocks],
|
||||
}
|
||||
} else {
|
||||
merged.push(msg)
|
||||
}
|
||||
}
|
||||
|
||||
return merged
|
||||
}
|
||||
|
||||
export class FeatherlessHandler extends OpenAICompatibleHandler {
|
||||
constructor(options: ApiHandlerOptions) {
|
||||
const modelId = options.apiModelId ?? featherlessDefaultModelId
|
||||
const modelInfo =
|
||||
featherlessModels[modelId as keyof typeof featherlessModels] || featherlessModels[featherlessDefaultModelId]
|
||||
|
||||
const config: OpenAICompatibleConfig = {
|
||||
providerName: "Featherless",
|
||||
baseURL: "https://api.featherless.ai/v1",
|
||||
apiKey: options.featherlessApiKey ?? "not-provided",
|
||||
modelId,
|
||||
modelInfo,
|
||||
modelMaxTokens: options.modelMaxTokens ?? undefined,
|
||||
temperature: options.modelTemperature ?? undefined,
|
||||
}
|
||||
|
||||
super(options, config)
|
||||
}
|
||||
|
||||
override getModel() {
|
||||
const id = this.options.apiModelId ?? featherlessDefaultModelId
|
||||
const info =
|
||||
featherlessModels[id as keyof typeof featherlessModels] || featherlessModels[featherlessDefaultModelId]
|
||||
const isDeepSeekR1 = id.includes("DeepSeek-R1")
|
||||
const defaultTemp = isDeepSeekR1 ? DEEP_SEEK_DEFAULT_TEMPERATURE : 0.5
|
||||
const params = getModelParams({
|
||||
format: "openai",
|
||||
modelId: id,
|
||||
model: info,
|
||||
settings: this.options,
|
||||
defaultTemperature: defaultTemp,
|
||||
})
|
||||
return { id, info, ...params }
|
||||
}
|
||||
|
||||
override async *createMessage(
|
||||
systemPrompt: string,
|
||||
messages: Anthropic.Messages.MessageParam[],
|
||||
|
|
@ -58,9 +91,17 @@ export class FeatherlessHandler extends BaseOpenAiCompatibleProvider<Featherless
|
|||
const model = this.getModel()
|
||||
|
||||
if (model.id.includes("DeepSeek-R1")) {
|
||||
const stream = await this.client.chat.completions.create({
|
||||
...this.getCompletionParams(systemPrompt, messages),
|
||||
messages: convertToR1Format([{ role: "user", content: systemPrompt }, ...messages]),
|
||||
// R1 path: merge system prompt into user messages, use TagMatcher for <think> tags.
|
||||
// mergeConsecutiveSameRoleMessages ensures no two successive messages share the
|
||||
// same role (e.g. the injected system-as-user + original first user message).
|
||||
const r1Messages: Anthropic.Messages.MessageParam[] = [{ role: "user", content: systemPrompt }, ...messages]
|
||||
const aiSdkMessages = convertToAiSdkMessages(mergeConsecutiveSameRoleMessages(r1Messages))
|
||||
|
||||
const result = streamText({
|
||||
model: this.getLanguageModel(),
|
||||
messages: aiSdkMessages,
|
||||
temperature: model.temperature ?? 0,
|
||||
maxOutputTokens: this.getMaxOutputTokens(),
|
||||
})
|
||||
|
||||
const matcher = new TagMatcher(
|
||||
|
|
@ -72,42 +113,28 @@ export class FeatherlessHandler extends BaseOpenAiCompatibleProvider<Featherless
|
|||
}) as const,
|
||||
)
|
||||
|
||||
for await (const chunk of stream) {
|
||||
const delta = chunk.choices[0]?.delta
|
||||
|
||||
if (delta?.content) {
|
||||
for (const processedChunk of matcher.update(delta.content)) {
|
||||
yield processedChunk
|
||||
try {
|
||||
for await (const part of result.fullStream) {
|
||||
if (part.type === "text-delta") {
|
||||
for (const processedChunk of matcher.update(part.text)) {
|
||||
yield processedChunk
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (chunk.usage) {
|
||||
yield {
|
||||
type: "usage",
|
||||
inputTokens: chunk.usage.prompt_tokens || 0,
|
||||
outputTokens: chunk.usage.completion_tokens || 0,
|
||||
}
|
||||
for (const processedChunk of matcher.final()) {
|
||||
yield processedChunk
|
||||
}
|
||||
}
|
||||
|
||||
// Process any remaining content
|
||||
for (const processedChunk of matcher.final()) {
|
||||
yield processedChunk
|
||||
const usage = await result.usage
|
||||
if (usage) {
|
||||
yield this.processUsageMetrics(usage)
|
||||
}
|
||||
} catch (error) {
|
||||
throw handleAiSdkError(error, "Featherless")
|
||||
}
|
||||
} else {
|
||||
yield* super.createMessage(systemPrompt, messages, metadata)
|
||||
}
|
||||
}
|
||||
|
||||
override getModel() {
|
||||
const model = super.getModel()
|
||||
const isDeepSeekR1 = model.id.includes("DeepSeek-R1")
|
||||
return {
|
||||
...model,
|
||||
info: {
|
||||
...model.info,
|
||||
temperature: isDeepSeekR1 ? DEEP_SEEK_DEFAULT_TEMPERATURE : this.defaultTemperature,
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue