From adedc1c2c6d9912071672f62f87a717c9a2b9a98 Mon Sep 17 00:00:00 2001 From: Joe Manley Date: Fri, 7 Feb 2025 23:08:26 -0800 Subject: [PATCH] Remove magic numbers --- src/api/providers/anthropic.ts | 8 +++++--- src/api/providers/bedrock.ts | 6 ++++-- src/api/providers/deepseek.ts | 2 ++ src/api/providers/gemini.ts | 6 ++++-- src/api/providers/glama.ts | 7 ++++--- src/api/providers/lmstudio.ts | 6 ++++-- src/api/providers/mistral.ts | 4 +++- src/api/providers/ollama.ts | 9 +++++++-- src/api/providers/openai-native.ts | 6 ++++-- src/api/providers/openai.ts | 6 +++++- src/api/providers/openrouter.ts | 9 ++++++--- 11 files changed, 48 insertions(+), 21 deletions(-) diff --git a/src/api/providers/anthropic.ts b/src/api/providers/anthropic.ts index 2059804a54..9a14756f5d 100644 --- a/src/api/providers/anthropic.ts +++ b/src/api/providers/anthropic.ts @@ -10,6 +10,8 @@ import { import { ApiHandler, SingleCompletionHandler } from "../index" import { ApiStream } from "../transform/stream" +const ANTHROPIC_DEFAULT_TEMPERATURE = 0 + export class AnthropicHandler implements ApiHandler, SingleCompletionHandler { private options: ApiHandlerOptions private client: Anthropic @@ -44,7 +46,7 @@ export class AnthropicHandler implements ApiHandler, SingleCompletionHandler { { model: modelId, max_tokens: this.getModel().info.maxTokens || 8192, - temperature: this.options.modelTemperature ?? 0, + temperature: this.options.modelTemperature ?? ANTHROPIC_DEFAULT_TEMPERATURE, system: [{ text: systemPrompt, type: "text", cache_control: { type: "ephemeral" } }], // setting cache breakpoint for system prompt so new tasks can reuse it messages: messages.map((message, index) => { if (index === lastUserMsgIndex || index === secondLastMsgUserIndex) { @@ -96,7 +98,7 @@ export class AnthropicHandler implements ApiHandler, SingleCompletionHandler { stream = (await this.client.messages.create({ model: modelId, max_tokens: this.getModel().info.maxTokens || 8192, - temperature: this.options.modelTemperature ?? 0, + temperature: this.options.modelTemperature ?? ANTHROPIC_DEFAULT_TEMPERATURE, system: [{ text: systemPrompt, type: "text" }], messages, // tools, @@ -179,7 +181,7 @@ export class AnthropicHandler implements ApiHandler, SingleCompletionHandler { const response = await this.client.messages.create({ model: this.getModel().id, max_tokens: this.getModel().info.maxTokens || 8192, - temperature: this.options.modelTemperature ?? 0, + temperature: this.options.modelTemperature ?? ANTHROPIC_DEFAULT_TEMPERATURE, messages: [{ role: "user", content: prompt }], stream: false, }) diff --git a/src/api/providers/bedrock.ts b/src/api/providers/bedrock.ts index 17362e1f05..8f897fda2a 100644 --- a/src/api/providers/bedrock.ts +++ b/src/api/providers/bedrock.ts @@ -11,6 +11,8 @@ import { ApiHandlerOptions, BedrockModelId, ModelInfo, bedrockDefaultModelId, be import { ApiStream } from "../transform/stream" import { convertToBedrockConverseMessages, convertToAnthropicMessage } from "../transform/bedrock-converse-format" +const BEDROCK_DEFAULT_TEMPERATURE = 0.3 + // Define types for stream events based on AWS SDK export interface StreamEvent { messageStart?: { @@ -104,7 +106,7 @@ export class AwsBedrockHandler implements ApiHandler, SingleCompletionHandler { system: [{ text: systemPrompt }], inferenceConfig: { maxTokens: modelConfig.info.maxTokens || 5000, - temperature: this.options.modelTemperature ?? 0.3, + temperature: this.options.modelTemperature ?? BEDROCK_DEFAULT_TEMPERATURE, topP: 0.1, ...(this.options.awsUsePromptCache ? { @@ -262,7 +264,7 @@ export class AwsBedrockHandler implements ApiHandler, SingleCompletionHandler { ]), inferenceConfig: { maxTokens: modelConfig.info.maxTokens || 5000, - temperature: this.options.modelTemperature ?? 0.3, + temperature: this.options.modelTemperature ?? BEDROCK_DEFAULT_TEMPERATURE, topP: 0.1, }, } diff --git a/src/api/providers/deepseek.ts b/src/api/providers/deepseek.ts index 1c7186d48c..308fea871b 100644 --- a/src/api/providers/deepseek.ts +++ b/src/api/providers/deepseek.ts @@ -2,6 +2,8 @@ import { OpenAiHandler } from "./openai" import { ApiHandlerOptions, ModelInfo } from "../../shared/api" import { deepSeekModels, deepSeekDefaultModelId } from "../../shared/api" +export const DEEP_SEEK_DEFAULT_TEMPERATURE = 0.6 + export class DeepSeekHandler extends OpenAiHandler { constructor(options: ApiHandlerOptions) { super({ diff --git a/src/api/providers/gemini.ts b/src/api/providers/gemini.ts index e9a0015224..0d7179320c 100644 --- a/src/api/providers/gemini.ts +++ b/src/api/providers/gemini.ts @@ -5,6 +5,8 @@ import { ApiHandlerOptions, geminiDefaultModelId, GeminiModelId, geminiModels, M import { convertAnthropicMessageToGemini } from "../transform/gemini-format" import { ApiStream } from "../transform/stream" +const GEMINI_DEFAULT_TEMPERATURE = 0 + export class GeminiHandler implements ApiHandler, SingleCompletionHandler { private options: ApiHandlerOptions private client: GoogleGenerativeAI @@ -23,7 +25,7 @@ export class GeminiHandler implements ApiHandler, SingleCompletionHandler { contents: messages.map(convertAnthropicMessageToGemini), generationConfig: { // maxOutputTokens: this.getModel().info.maxTokens, - temperature: this.options.modelTemperature ?? 0, + temperature: this.options.modelTemperature ?? GEMINI_DEFAULT_TEMPERATURE, }, }) @@ -60,7 +62,7 @@ export class GeminiHandler implements ApiHandler, SingleCompletionHandler { const result = await model.generateContent({ contents: [{ role: "user", parts: [{ text: prompt }] }], generationConfig: { - temperature: this.options.modelTemperature ?? 0, + temperature: this.options.modelTemperature ?? GEMINI_DEFAULT_TEMPERATURE, }, }) diff --git a/src/api/providers/glama.ts b/src/api/providers/glama.ts index 226891b16a..72b41e5f58 100644 --- a/src/api/providers/glama.ts +++ b/src/api/providers/glama.ts @@ -5,7 +5,8 @@ import { ApiHandler, SingleCompletionHandler } from "../" import { ApiHandlerOptions, ModelInfo, glamaDefaultModelId, glamaDefaultModelInfo } from "../../shared/api" import { convertToOpenAiMessages } from "../transform/openai-format" import { ApiStream } from "../transform/stream" -import delay from "delay" + +const GLAMA_DEFAULT_TEMPERATURE = 0 export class GlamaHandler implements ApiHandler, SingleCompletionHandler { private options: ApiHandlerOptions @@ -79,7 +80,7 @@ export class GlamaHandler implements ApiHandler, SingleCompletionHandler { } if (this.supportsTemperature()) { - requestOptions.temperature = this.options.modelTemperature ?? 0 + requestOptions.temperature = this.options.modelTemperature ?? GLAMA_DEFAULT_TEMPERATURE } const { data: completion, response } = await this.client.chat.completions @@ -172,7 +173,7 @@ export class GlamaHandler implements ApiHandler, SingleCompletionHandler { } if (this.supportsTemperature()) { - requestOptions.temperature = this.options.modelTemperature ?? 0 + requestOptions.temperature = this.options.modelTemperature ?? GLAMA_DEFAULT_TEMPERATURE } if (this.getModel().id.startsWith("anthropic/")) { diff --git a/src/api/providers/lmstudio.ts b/src/api/providers/lmstudio.ts index cc164d240e..7efa037f46 100644 --- a/src/api/providers/lmstudio.ts +++ b/src/api/providers/lmstudio.ts @@ -5,6 +5,8 @@ import { ApiHandlerOptions, ModelInfo, openAiModelInfoSaneDefaults } from "../.. import { convertToOpenAiMessages } from "../transform/openai-format" import { ApiStream } from "../transform/stream" +const LMSTUDIO_DEFAULT_TEMPERATURE = 0 + export class LmStudioHandler implements ApiHandler, SingleCompletionHandler { private options: ApiHandlerOptions private client: OpenAI @@ -27,7 +29,7 @@ export class LmStudioHandler implements ApiHandler, SingleCompletionHandler { const stream = await this.client.chat.completions.create({ model: this.getModel().id, messages: openAiMessages, - temperature: this.options.modelTemperature ?? 0, + temperature: this.options.modelTemperature ?? LMSTUDIO_DEFAULT_TEMPERATURE, stream: true, }) for await (const chunk of stream) { @@ -59,7 +61,7 @@ export class LmStudioHandler implements ApiHandler, SingleCompletionHandler { const response = await this.client.chat.completions.create({ model: this.getModel().id, messages: [{ role: "user", content: prompt }], - temperature: this.options.modelTemperature ?? 0, + temperature: this.options.modelTemperature ?? LMSTUDIO_DEFAULT_TEMPERATURE, stream: false, }) return response.choices[0]?.message.content || "" diff --git a/src/api/providers/mistral.ts b/src/api/providers/mistral.ts index 4bcf1a191c..9ce70a297c 100644 --- a/src/api/providers/mistral.ts +++ b/src/api/providers/mistral.ts @@ -14,6 +14,8 @@ import { import { convertToMistralMessages } from "../transform/mistral-format" import { ApiStream } from "../transform/stream" +const MISTRAL_DEFAULT_TEMPERATURE = 0 + export class MistralHandler implements ApiHandler { private options: ApiHandlerOptions private client: Mistral @@ -30,7 +32,7 @@ export class MistralHandler implements ApiHandler { const stream = await this.client.chat.stream({ model: this.getModel().id, // max_completion_tokens: this.getModel().info.maxTokens, - temperature: this.options.modelTemperature ?? 0, + temperature: this.options.modelTemperature ?? MISTRAL_DEFAULT_TEMPERATURE, messages: [{ role: "system", content: systemPrompt }, ...convertToMistralMessages(messages)], stream: true, }) diff --git a/src/api/providers/ollama.ts b/src/api/providers/ollama.ts index b6b11ccc07..b8c2dfaeee 100644 --- a/src/api/providers/ollama.ts +++ b/src/api/providers/ollama.ts @@ -5,6 +5,9 @@ import { ApiHandlerOptions, ModelInfo, openAiModelInfoSaneDefaults } from "../.. import { convertToOpenAiMessages } from "../transform/openai-format" import { convertToR1Format } from "../transform/r1-format" import { ApiStream } from "../transform/stream" +import { DEEP_SEEK_DEFAULT_TEMPERATURE } from "./deepseek" + +const OLLAMA_DEFAULT_TEMPERATURE = 0 export class OllamaHandler implements ApiHandler, SingleCompletionHandler { private options: ApiHandlerOptions @@ -29,7 +32,7 @@ export class OllamaHandler implements ApiHandler, SingleCompletionHandler { const stream = await this.client.chat.completions.create({ model: this.getModel().id, messages: openAiMessages, - temperature: this.options.modelTemperature ?? 0, + temperature: this.options.modelTemperature ?? DEFAULT_TEMPERATURE, stream: true, }) for await (const chunk of stream) { @@ -59,7 +62,9 @@ export class OllamaHandler implements ApiHandler, SingleCompletionHandler { messages: useR1Format ? convertToR1Format([{ role: "user", content: prompt }]) : [{ role: "user", content: prompt }], - temperature: this.options.modelTemperature ?? (useR1Format ? 0.6 : 0), + temperature: + this.options.modelTemperature ?? + (useR1Format ? DEEP_SEEK_DEFAULT_TEMPERATURE : OLLAMA_DEFAULT_TEMPERATURE), stream: false, }) return response.choices[0]?.message.content || "" diff --git a/src/api/providers/openai-native.ts b/src/api/providers/openai-native.ts index a40e002ce1..1a4f9e613a 100644 --- a/src/api/providers/openai-native.ts +++ b/src/api/providers/openai-native.ts @@ -11,6 +11,8 @@ import { import { convertToOpenAiMessages } from "../transform/openai-format" import { ApiStream } from "../transform/stream" +const OPENAI_NATIVE_DEFAULT_TEMPERATURE = 0 + export class OpenAiNativeHandler implements ApiHandler, SingleCompletionHandler { private options: ApiHandlerOptions private client: OpenAI @@ -88,7 +90,7 @@ export class OpenAiNativeHandler implements ApiHandler, SingleCompletionHandler ): ApiStream { const stream = await this.client.chat.completions.create({ model: modelId, - temperature: this.options.modelTemperature ?? 0, + temperature: this.options.modelTemperature ?? OPENAI_NATIVE_DEFAULT_TEMPERATURE, messages: [{ role: "system", content: systemPrompt }, ...convertToOpenAiMessages(messages)], stream: true, stream_options: { include_usage: true }, @@ -189,7 +191,7 @@ export class OpenAiNativeHandler implements ApiHandler, SingleCompletionHandler return { model: modelId, messages: [{ role: "user", content: prompt }], - temperature: this.options.modelTemperature ?? 0, + temperature: this.options.modelTemperature ?? OPENAI_NATIVE_DEFAULT_TEMPERATURE, } } } diff --git a/src/api/providers/openai.ts b/src/api/providers/openai.ts index da3cf1b9e7..07bfbf7c28 100644 --- a/src/api/providers/openai.ts +++ b/src/api/providers/openai.ts @@ -11,6 +11,8 @@ import { ApiHandler, SingleCompletionHandler } from "../index" import { convertToOpenAiMessages } from "../transform/openai-format" import { convertToR1Format } from "../transform/r1-format" import { ApiStream } from "../transform/stream" +import { DEEP_SEEK_DEFAULT_TEMPERATURE } from "./deepseek" +const OPENAI_DEFAULT_TEMPERATURE = 0 export class OpenAiHandler implements ApiHandler, SingleCompletionHandler { protected options: ApiHandlerOptions @@ -57,7 +59,9 @@ export class OpenAiHandler implements ApiHandler, SingleCompletionHandler { } const requestOptions: OpenAI.Chat.Completions.ChatCompletionCreateParamsStreaming = { model: modelId, - temperature: this.options.modelTemperature ?? (deepseekReasoner ? 0.6 : 0), + temperature: + this.options.modelTemperature ?? + (deepseekReasoner ? DEEP_SEEK_DEFAULT_TEMPERATURE : OPENAI_DEFAULT_TEMPERATURE), messages: deepseekReasoner ? convertToR1Format([{ role: "user", content: systemPrompt }, ...messages]) : [systemMessage, ...convertToOpenAiMessages(messages)], diff --git a/src/api/providers/openrouter.ts b/src/api/providers/openrouter.ts index fa1c65d126..4334e6d48a 100644 --- a/src/api/providers/openrouter.ts +++ b/src/api/providers/openrouter.ts @@ -6,6 +6,9 @@ import { ApiHandlerOptions, ModelInfo, openRouterDefaultModelId, openRouterDefau import { convertToOpenAiMessages } from "../transform/openai-format" import { ApiStreamChunk, ApiStreamUsageChunk } from "../transform/stream" import delay from "delay" +import { DEEP_SEEK_DEFAULT_TEMPERATURE } from "./deepseek" + +const OPENROUTER_DEFAULT_TEMPERATURE = 0 // Add custom interface for OpenRouter params type OpenRouterChatCompletionParams = OpenAI.Chat.ChatCompletionCreateParams & { @@ -115,7 +118,7 @@ export class OpenRouterHandler implements ApiHandler, SingleCompletionHandler { break } - let defaultTemperature = 0 + let defaultTemperature = OPENROUTER_DEFAULT_TEMPERATURE let topP: number | undefined = undefined // Handle models based on deepseek-r1 @@ -124,7 +127,7 @@ export class OpenRouterHandler implements ApiHandler, SingleCompletionHandler { this.getModel().id === "perplexity/sonar-reasoning" ) { // Recommended temperature for DeepSeek reasoning models - defaultTemperature = 0.6 + defaultTemperature = DEEP_SEEK_DEFAULT_TEMPERATURE // DeepSeek highly recommends using user instead of system role openAiMessages = convertToR1Format([{ role: "user", content: systemPrompt }, ...messages]) // Some provider support topP and 0.95 is value that Deepseek used in their benchmarks @@ -223,7 +226,7 @@ export class OpenRouterHandler implements ApiHandler, SingleCompletionHandler { const response = await this.client.chat.completions.create({ model: this.getModel().id, messages: [{ role: "user", content: prompt }], - temperature: this.options.modelTemperature ?? 0, + temperature: this.options.modelTemperature ?? OPENROUTER_DEFAULT_TEMPERATURE, stream: false, })