From 3322e08cb73b5beab277070cae9cdfdcc7ecd70c Mon Sep 17 00:00:00 2001 From: Chris Estreich Date: Sun, 30 Mar 2025 23:07:34 -0700 Subject: [PATCH] Relax provider profiles schema and log parse error to PostHog (#2139) * Relax provider profiles schema and log parse error to PostHog * Create wise-moose-shop.md --------- Co-authored-by: Matt Rubens --- .changeset/wise-moose-shop.md | 5 +++++ src/api/providers/glama.ts | 2 +- src/api/providers/unbound.ts | 2 +- src/api/providers/vertex.ts | 2 +- src/core/config/ContextProxy.ts | 19 ++++++++++++++----- src/core/config/ProviderSettingsManager.ts | 7 ++++++- src/core/sliding-window/index.ts | 2 +- src/exports/roo-code.d.ts | 10 +++++----- src/exports/types.ts | 10 +++++----- src/schemas/index.ts | 2 +- src/services/telemetry/TelemetryService.ts | 13 +++++++++++++ .../src/components/settings/ModelInfoView.tsx | 2 +- webview-ui/src/utils/model-utils.ts | 4 ++-- 13 files changed, 56 insertions(+), 24 deletions(-) create mode 100644 .changeset/wise-moose-shop.md diff --git a/.changeset/wise-moose-shop.md b/.changeset/wise-moose-shop.md new file mode 100644 index 0000000000..775a7cea44 --- /dev/null +++ b/.changeset/wise-moose-shop.md @@ -0,0 +1,5 @@ +--- +"roo-cline": patch +--- + +Relax provider profiles schema and add telemetry diff --git a/src/api/providers/glama.ts b/src/api/providers/glama.ts index cc0b06e611..43b6ebfb7a 100644 --- a/src/api/providers/glama.ts +++ b/src/api/providers/glama.ts @@ -88,7 +88,7 @@ export class GlamaHandler extends BaseProvider implements SingleCompletionHandle let maxTokens: number | undefined if (this.getModel().id.startsWith("anthropic/")) { - maxTokens = this.getModel().info.maxTokens + maxTokens = this.getModel().info.maxTokens ?? undefined } const requestOptions: OpenAI.Chat.ChatCompletionCreateParams = { diff --git a/src/api/providers/unbound.ts b/src/api/providers/unbound.ts index f83ecd457b..0413c96f29 100644 --- a/src/api/providers/unbound.ts +++ b/src/api/providers/unbound.ts @@ -79,7 +79,7 @@ export class UnboundHandler extends BaseProvider implements SingleCompletionHand let maxTokens: number | undefined if (this.getModel().id.startsWith("anthropic/")) { - maxTokens = this.getModel().info.maxTokens + maxTokens = this.getModel().info.maxTokens ?? undefined } const requestOptions: OpenAI.Chat.Completions.ChatCompletionCreateParamsStreaming = { diff --git a/src/api/providers/vertex.ts b/src/api/providers/vertex.ts index bc888e0459..1f863c57cd 100644 --- a/src/api/providers/vertex.ts +++ b/src/api/providers/vertex.ts @@ -230,7 +230,7 @@ export class VertexHandler extends BaseProvider implements SingleCompletionHandl const result = await model.generateContentStream({ contents: messages.map(convertAnthropicMessageToVertexGemini), generationConfig: { - maxOutputTokens: this.getModel().info.maxTokens, + maxOutputTokens: this.getModel().info.maxTokens ?? undefined, temperature: this.options.modelTemperature ?? 0, }, }) diff --git a/src/core/config/ContextProxy.ts b/src/core/config/ContextProxy.ts index 835ca214f6..aa40477ad8 100644 --- a/src/core/config/ContextProxy.ts +++ b/src/core/config/ContextProxy.ts @@ -1,4 +1,5 @@ import * as vscode from "vscode" +import { ZodError } from "zod" import { PROVIDER_SETTINGS_KEYS, @@ -15,6 +16,7 @@ import { isSecretStateKey, } from "../../schemas" import { logger } from "../../utils/logging" +import { telemetryService } from "../../services/telemetry/TelemetryService" type GlobalStateKey = keyof GlobalState type SecretStateKey = keyof SecretState @@ -157,8 +159,10 @@ export class ContextProxy { try { return globalSettingsSchema.parse(values) } catch (error) { - // Log to Posthog? - // We'll want to know about bad type assumptions or bad ExtensionState data. + if (error instanceof ZodError) { + telemetryService.captureSchemaValidationError({ schemaName: "GlobalSettings", error }) + } + return GLOBAL_SETTINGS_KEYS.reduce((acc, key) => ({ ...acc, [key]: values[key] }), {} as GlobalSettings) } } @@ -173,8 +177,10 @@ export class ContextProxy { try { return providerSettingsSchema.parse(values) } catch (error) { - // Log to Posthog? - // We'll want to know about bad type assumptions or bad ExtensionState data. + if (error instanceof ZodError) { + telemetryService.captureSchemaValidationError({ schemaName: "ProviderSettings", error }) + } + return PROVIDER_SETTINGS_KEYS.reduce((acc, key) => ({ ...acc, [key]: values[key] }), {} as ProviderSettings) } } @@ -225,7 +231,10 @@ export class ContextProxy { const globalSettings = globalSettingsExportSchema.parse(this.getValues()) return Object.fromEntries(Object.entries(globalSettings).filter(([_, value]) => value !== undefined)) } catch (error) { - console.log(error.message) + if (error instanceof ZodError) { + telemetryService.captureSchemaValidationError({ schemaName: "GlobalSettings", error }) + } + return undefined } } diff --git a/src/core/config/ProviderSettingsManager.ts b/src/core/config/ProviderSettingsManager.ts index f4df9c0829..9729219a2d 100644 --- a/src/core/config/ProviderSettingsManager.ts +++ b/src/core/config/ProviderSettingsManager.ts @@ -1,8 +1,9 @@ import { ExtensionContext } from "vscode" -import { z } from "zod" +import { z, ZodError } from "zod" import { providerSettingsSchema, ApiConfigMeta } from "../../schemas" import { Mode, modes } from "../../shared/modes" +import { telemetryService } from "../../services/telemetry/TelemetryService" const providerSettingsWithIdSchema = providerSettingsSchema.extend({ id: z.string().optional() }) @@ -272,6 +273,10 @@ export class ProviderSettingsManager { const content = await this.context.secrets.get(this.secretsKey) return content ? providerProfilesSchema.parse(JSON.parse(content)) : this.defaultProviderProfiles } catch (error) { + if (error instanceof ZodError) { + telemetryService.captureSchemaValidationError({ schemaName: "ProviderProfiles", error }) + } + throw new Error(`Failed to read provider profiles from secrets: ${error}`) } } diff --git a/src/core/sliding-window/index.ts b/src/core/sliding-window/index.ts index 67c0028fab..75395ecd75 100644 --- a/src/core/sliding-window/index.ts +++ b/src/core/sliding-window/index.ts @@ -60,7 +60,7 @@ type TruncateOptions = { messages: Anthropic.Messages.MessageParam[] totalTokens: number contextWindow: number - maxTokens?: number + maxTokens?: number | null apiHandler: ApiHandler } diff --git a/src/exports/roo-code.d.ts b/src/exports/roo-code.d.ts index 255b58d94d..2804ab47f0 100644 --- a/src/exports/roo-code.d.ts +++ b/src/exports/roo-code.d.ts @@ -28,7 +28,7 @@ type ProviderSettings = { glamaModelId?: string | undefined glamaModelInfo?: | ({ - maxTokens?: number | undefined + maxTokens?: (number | null) | undefined contextWindow: number supportsImages?: boolean | undefined supportsComputerUse?: boolean | undefined @@ -47,7 +47,7 @@ type ProviderSettings = { openRouterModelId?: string | undefined openRouterModelInfo?: | ({ - maxTokens?: number | undefined + maxTokens?: (number | null) | undefined contextWindow: number supportsImages?: boolean | undefined supportsComputerUse?: boolean | undefined @@ -84,7 +84,7 @@ type ProviderSettings = { openAiModelId?: string | undefined openAiCustomModelInfo?: | ({ - maxTokens?: number | undefined + maxTokens?: (number | null) | undefined contextWindow: number supportsImages?: boolean | undefined supportsComputerUse?: boolean | undefined @@ -126,7 +126,7 @@ type ProviderSettings = { unboundModelId?: string | undefined unboundModelInfo?: | ({ - maxTokens?: number | undefined + maxTokens?: (number | null) | undefined contextWindow: number supportsImages?: boolean | undefined supportsComputerUse?: boolean | undefined @@ -144,7 +144,7 @@ type ProviderSettings = { requestyModelId?: string | undefined requestyModelInfo?: | ({ - maxTokens?: number | undefined + maxTokens?: (number | null) | undefined contextWindow: number supportsImages?: boolean | undefined supportsComputerUse?: boolean | undefined diff --git a/src/exports/types.ts b/src/exports/types.ts index 58f9ea33e1..f7d638ee0a 100644 --- a/src/exports/types.ts +++ b/src/exports/types.ts @@ -29,7 +29,7 @@ type ProviderSettings = { glamaModelId?: string | undefined glamaModelInfo?: | ({ - maxTokens?: number | undefined + maxTokens?: (number | null) | undefined contextWindow: number supportsImages?: boolean | undefined supportsComputerUse?: boolean | undefined @@ -48,7 +48,7 @@ type ProviderSettings = { openRouterModelId?: string | undefined openRouterModelInfo?: | ({ - maxTokens?: number | undefined + maxTokens?: (number | null) | undefined contextWindow: number supportsImages?: boolean | undefined supportsComputerUse?: boolean | undefined @@ -85,7 +85,7 @@ type ProviderSettings = { openAiModelId?: string | undefined openAiCustomModelInfo?: | ({ - maxTokens?: number | undefined + maxTokens?: (number | null) | undefined contextWindow: number supportsImages?: boolean | undefined supportsComputerUse?: boolean | undefined @@ -127,7 +127,7 @@ type ProviderSettings = { unboundModelId?: string | undefined unboundModelInfo?: | ({ - maxTokens?: number | undefined + maxTokens?: (number | null) | undefined contextWindow: number supportsImages?: boolean | undefined supportsComputerUse?: boolean | undefined @@ -145,7 +145,7 @@ type ProviderSettings = { requestyModelId?: string | undefined requestyModelInfo?: | ({ - maxTokens?: number | undefined + maxTokens?: (number | null) | undefined contextWindow: number supportsImages?: boolean | undefined supportsComputerUse?: boolean | undefined diff --git a/src/schemas/index.ts b/src/schemas/index.ts index 10d3e245f5..d0a789ead8 100644 --- a/src/schemas/index.ts +++ b/src/schemas/index.ts @@ -100,7 +100,7 @@ export type TelemetrySetting = z.infer */ export const modelInfoSchema = z.object({ - maxTokens: z.number().optional(), + maxTokens: z.number().nullish(), contextWindow: z.number(), supportsImages: z.boolean().optional(), supportsComputerUse: z.boolean().optional(), diff --git a/src/services/telemetry/TelemetryService.ts b/src/services/telemetry/TelemetryService.ts index d3ea8bfb5f..30145c6b98 100644 --- a/src/services/telemetry/TelemetryService.ts +++ b/src/services/telemetry/TelemetryService.ts @@ -1,5 +1,7 @@ import { PostHog } from "posthog-node" import * as vscode from "vscode" +import { ZodError } from "zod" + import { logger } from "../../utils/logging" // This forward declaration is needed to avoid circular dependencies @@ -26,6 +28,9 @@ class PostHogClient { CHECKPOINT_RESTORED: "Checkpoint Restored", CHECKPOINT_DIFFED: "Checkpoint Diffed", }, + ERRORS: { + SCHEMA_VALIDATION_ERROR: "Schema Validation Error", + }, } private static instance: PostHogClient @@ -261,6 +266,14 @@ class TelemetryService { this.captureEvent(PostHogClient.EVENTS.TASK.CHECKPOINT_RESTORED, { taskId }) } + public captureSchemaValidationError({ schemaName, error }: { schemaName: string; error: ZodError }): void { + this.captureEvent(PostHogClient.EVENTS.ERRORS.SCHEMA_VALIDATION_ERROR, { + schemaName, + // https://zod.dev/ERROR_HANDLING?id=formatting-errors + error: error.format(), + }) + } + /** * Checks if telemetry is currently enabled * @returns Whether telemetry is enabled diff --git a/webview-ui/src/components/settings/ModelInfoView.tsx b/webview-ui/src/components/settings/ModelInfoView.tsx index 603c5a5fe5..aa927f8ea2 100644 --- a/webview-ui/src/components/settings/ModelInfoView.tsx +++ b/webview-ui/src/components/settings/ModelInfoView.tsx @@ -43,7 +43,7 @@ export const ModelInfoView = ({ doesNotSupportLabel={t("settings:modelInfo.noPromptCache")} /> ), - modelInfo.maxTokens !== undefined && modelInfo.maxTokens > 0 && ( + typeof modelInfo.maxTokens === "number" && modelInfo.maxTokens > 0 && ( <> {t("settings:modelInfo.maxOutput")}:{" "} {modelInfo.maxTokens?.toLocaleString()} tokens diff --git a/webview-ui/src/utils/model-utils.ts b/webview-ui/src/utils/model-utils.ts index 8380062eef..29adb11a96 100644 --- a/webview-ui/src/utils/model-utils.ts +++ b/webview-ui/src/utils/model-utils.ts @@ -14,7 +14,7 @@ export interface ModelInfo { /** * Maximum number of tokens the model can process */ - maxTokens?: number + maxTokens?: number | null /** * Whether the model supports thinking/reasoning capabilities @@ -77,7 +77,7 @@ export const getMaxTokensForModel = ( if (modelInfo?.thinking) { return apiConfig?.modelMaxTokens || DEFAULT_THINKING_MODEL_MAX_TOKENS } - return modelInfo?.maxTokens + return modelInfo?.maxTokens ?? undefined } /**