From 1e59cd9e3561be8df396619fcf3e6f87a89a5e57 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 17 Jul 2026 14:14:54 -0400 Subject: [PATCH 01/34] fix(utils): honor string drop_params values from config and DB deployments --- litellm/litellm_core_utils/core_helpers.py | 12 +++++++ litellm/types/router.py | 7 ++++ litellm/utils.py | 9 +++-- .../litellm_core_utils/test_core_helpers.py | 22 ++++++++++++ tests/test_litellm/test_router.py | 34 +++++++++++++++++++ tests/test_litellm/test_utils.py | 28 +++++++++++++++ ui/litellm-dashboard/src/lib/http/schema.d.ts | 4 +++ 7 files changed, 113 insertions(+), 3 deletions(-) diff --git a/litellm/litellm_core_utils/core_helpers.py b/litellm/litellm_core_utils/core_helpers.py index 002a46771e3..838019264c9 100644 --- a/litellm/litellm_core_utils/core_helpers.py +++ b/litellm/litellm_core_utils/core_helpers.py @@ -36,6 +36,18 @@ def safe_divide_seconds(seconds: float, denominator: float, default: Optional[fl return float(seconds / denominator) +def normalize_drop_params(value: object) -> bool | None: + if isinstance(value, bool): + return value + if isinstance(value, str): + lowered = value.strip().lower() + if lowered == "true": + return True + if lowered == "false": + return False + return None + + def safe_divide( numerator: Union[int, float], denominator: Union[int, float], diff --git a/litellm/types/router.py b/litellm/types/router.py index 28e4a8272e8..77d1815d28d 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -23,6 +23,7 @@ from pydantic import BaseModel, ConfigDict, Field, field_validator, model_valida from typing_extensions import Protocol, Required, TypedDict, runtime_checkable from litellm._uuid import uuid +from litellm.litellm_core_utils.core_helpers import normalize_drop_params from .completion import CompletionRequest from .embedding import EmbeddingRequest @@ -233,6 +234,7 @@ class GenericLiteLLMParams(CredentialLiteLLMParams, CustomPricingLiteLLMParams): None # timeout when making stream=True calls, if str, pass in as os.environ/ ) max_retries: Optional[int] = None + drop_params: Optional[bool] = None organization: Optional[str] = None # for openai orgs configurable_clientside_auth_params: CONFIGURABLE_CLIENTSIDE_AUTH_PARAMS = None litellm_credential_name: Optional[str] = None @@ -311,6 +313,11 @@ class GenericLiteLLMParams(CredentialLiteLLMParams, CustomPricingLiteLLMParams): return filtered return data + @field_validator("drop_params", mode="before") + @classmethod + def coerce_drop_params(cls, value: object) -> Optional[bool]: + return normalize_drop_params(value) + def __contains__(self, key): # Define custom behavior for the 'in' operator return hasattr(self, key) diff --git a/litellm/utils.py b/litellm/utils.py index e19d2b36a52..e10b63ceb37 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -60,6 +60,7 @@ from litellm._lazy_imports import ( _get_token_counter_new, ) from litellm._uuid import uuid +from litellm.litellm_core_utils.core_helpers import normalize_drop_params from litellm.litellm_core_utils.fallback_generalizations import ( match_capability_generalizations, ) @@ -2852,7 +2853,7 @@ def get_optional_params_transcription( passed_params.pop("OPENAI_TRANSCRIPTION_PARAMS") custom_llm_provider = passed_params.pop("custom_llm_provider") - drop_params = passed_params.pop("drop_params") + drop_params = normalize_drop_params(passed_params.pop("drop_params")) special_params = passed_params.pop("kwargs") for k, v in special_params.items(): passed_params[k] = v @@ -2960,7 +2961,7 @@ def get_optional_params_image_gen( model = passed_params.pop("model", None) custom_llm_provider = passed_params.pop("custom_llm_provider") provider_config = passed_params.pop("provider_config", None) - drop_params = passed_params.pop("drop_params", None) + drop_params = normalize_drop_params(passed_params.pop("drop_params", None)) additional_drop_params = passed_params.pop("additional_drop_params", None) special_params = passed_params.pop("kwargs") for k, v in special_params.items(): @@ -3084,7 +3085,7 @@ def get_optional_params_embeddings( custom_llm_provider = passed_params.pop("custom_llm_provider", None) special_params = passed_params.pop("kwargs") - drop_params = passed_params.pop("drop_params", None) + drop_params = normalize_drop_params(passed_params.pop("drop_params", None)) additional_drop_params = passed_params.pop("additional_drop_params", None) allowed_openai_params = passed_params.pop("allowed_openai_params", None) or [] # Remove function objects from passed_params to avoid JSON serialization errors @@ -3797,6 +3798,8 @@ def get_optional_params( ): passed_params = locals().copy() special_params = passed_params.pop("kwargs") + drop_params = normalize_drop_params(drop_params) + passed_params["drop_params"] = drop_params # Remove base_model from passed_params so it doesn't interfere with # non_default_params / _check_valid_arg — it's a routing hint, not an # OpenAI param. diff --git a/tests/test_litellm/litellm_core_utils/test_core_helpers.py b/tests/test_litellm/litellm_core_utils/test_core_helpers.py index b67ea91bb0b..a7f93e3c997 100644 --- a/tests/test_litellm/litellm_core_utils/test_core_helpers.py +++ b/tests/test_litellm/litellm_core_utils/test_core_helpers.py @@ -5,6 +5,7 @@ import pytest from litellm.litellm_core_utils.core_helpers import ( _FINISH_REASON_MAP, map_finish_reason, + normalize_drop_params, reconstruct_model_name, redact_nested_match_and_regex_keys, ) @@ -201,3 +202,24 @@ class TestRedactNestedMatchAndRegexKeys: def test_passes_through_none_and_str(self): assert redact_nested_match_and_regex_keys(None) is None assert redact_nested_match_and_regex_keys("plain") == "plain" + + +@pytest.mark.parametrize( + "value, expected", + [ + (True, True), + (False, False), + ("true", True), + ("True", True), + (" TRUE ", True), + ("false", False), + ("False", False), + (None, None), + ("yes", None), + ("", None), + (1, None), + (0, None), + ], +) +def test_normalize_drop_params(value, expected): + assert normalize_drop_params(value) is expected diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index c2c98c8869c..81927d7e959 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -5430,3 +5430,37 @@ class TestRouterRequestTimeoutPropagation: ) == 60 ) + + +@pytest.mark.asyncio +async def test_router_deployment_drop_params_string_true_is_honored(monkeypatch): + from litellm import Router + + monkeypatch.setattr(litellm, "drop_params", False) + router = Router( + model_list=[ + { + "model_name": "gpt-5-nano", + "litellm_params": { + "model": "openai/gpt-5-nano", + "api_key": "sk-fake", + "temperature": 1, + "reasoning_effort": "minimal", + "drop_params": "true", + "mock_response": "Hello, world!", + }, + } + ], + num_retries=0, + ) + + deployment = router.get_deployment_by_model_group_name(model_group_name="gpt-5-nano") + assert deployment is not None + assert deployment.litellm_params.drop_params is True + + response = await router.acompletion( + model="gpt-5-nano", + messages=[{"role": "user", "content": "hi"}], + temperature=0.1, + ) + assert response.choices[0].message.content == "Hello, world!" diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index 073ff17991e..0fbf9c0db20 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -4814,3 +4814,31 @@ def test_is_prompt_caching_valid_prompt_explicit_min_token_count_overrides_model is_prompt_caching_valid_prompt(model="claude-opus-4-8", messages=PROMPT_CACHE_MESSAGES, min_token_count=8192) is False ) + + +class TestDropParamsStringCoercion: + @pytest.mark.parametrize("drop_params", ["true", "True", True]) + def test_truthy_drop_params_drops_unsupported_temperature(self, drop_params, monkeypatch): + from litellm.utils import get_optional_params + + monkeypatch.setattr(litellm, "drop_params", False) + result = get_optional_params( + model="gpt-5-nano", + custom_llm_provider="openai", + temperature=0.1, + drop_params=drop_params, + ) + assert "temperature" not in result + + @pytest.mark.parametrize("drop_params", ["false", False, None]) + def test_falsy_drop_params_still_raises(self, drop_params, monkeypatch): + from litellm.utils import get_optional_params + + monkeypatch.setattr(litellm, "drop_params", False) + with pytest.raises(litellm.UnsupportedParamsError): + get_optional_params( + model="gpt-5-nano", + custom_llm_provider="openai", + temperature=0.1, + drop_params=drop_params, + ) diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 0d8f55164f9..62103e4f742 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -25667,6 +25667,8 @@ export interface components { default_api_key_rpm_limit?: number | null; /** Default Api Key Tpm Limit */ default_api_key_tpm_limit?: number | null; + /** Drop Params */ + drop_params?: boolean | null; /** Gcs Bucket Name */ gcs_bucket_name?: string | null; /** Input Cost Per Audio Per Second */ @@ -33503,6 +33505,8 @@ export interface components { default_api_key_rpm_limit?: number | null; /** Default Api Key Tpm Limit */ default_api_key_tpm_limit?: number | null; + /** Drop Params */ + drop_params?: boolean | null; /** Gcs Bucket Name */ gcs_bucket_name?: string | null; /** Input Cost Per Audio Per Second */ From 2a2c49ad4cd711aded30da0384c1a200ddfca5d7 Mon Sep 17 00:00:00 2001 From: mubashir1osmani Date: Thu, 3 Sep 2026 16:09:44 -0400 Subject: [PATCH 02/34] feat(ui): surface batch results on the logs page --- .../LogDetailContent.test.tsx | 78 ++++++++++++++++ .../LogDetailsDrawer/LogDetailContent.tsx | 61 +++++++++++++ .../RequestLogsTableColumns.test.tsx | 53 +++++++++++ .../view_logs/RequestLogsTableColumns.tsx | 29 +++++- .../components/view_logs/TypeBadges.test.tsx | 14 ++- .../src/components/view_logs/TypeBadges.tsx | 26 ++++++ .../view_logs/batchLogUtils.test.ts | 90 +++++++++++++++++++ .../src/components/view_logs/batchLogUtils.ts | 67 ++++++++++++++ .../src/components/view_logs/constants.ts | 3 + 9 files changed, 418 insertions(+), 3 deletions(-) create mode 100644 ui/litellm-dashboard/src/components/view_logs/batchLogUtils.test.ts create mode 100644 ui/litellm-dashboard/src/components/view_logs/batchLogUtils.ts diff --git a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailContent.test.tsx b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailContent.test.tsx index a679dc49427..91778d2a98a 100644 --- a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailContent.test.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailContent.test.tsx @@ -113,6 +113,84 @@ describe("LogDetailContent", () => { expect(screen.getAllByText("$0.00200000").length).toBeGreaterThanOrEqual(1); }); + it("shows reasoning tokens in Metrics when the usage breakout carries them", () => { + render( + , + ); + + expect(screen.getByText("Reasoning Tokens")).toBeInTheDocument(); + expect(screen.getByText("224")).toBeInTheDocument(); + }); + + it("hides the reasoning metric when the breakout is absent or zero", () => { + render( + , + ); + + expect(screen.queryByText("Reasoning Tokens")).not.toBeInTheDocument(); + }); + + describe("Batch Results section", () => { + const batchCostEntry = (metadata: Record) => + createLogEntry({ + request_id: "batch_abc123_batch_cost", + call_type: "aretrieve_batch", + metadata: { status: "success", ...metadata }, + }); + + it("renders batch id, per-request outcome counts, and batch models for a batch cost row", () => { + render( + , + ); + + const section = screen.getByText("Batch Results").closest('[data-slot="card"]') as HTMLElement; + expect(within(section).getByText("batch_abc123")).toBeInTheDocument(); + expect(within(section).getByText("2")).toBeInTheDocument(); + expect(within(section).getByText("1")).toBeInTheDocument(); + expect(within(section).getByText("gemini-2.5-flash")).toBeInTheDocument(); + }); + + it("still renders the batch id when a legacy row carries no counts", () => { + render(); + + expect(screen.getByText("Batch Results")).toBeInTheDocument(); + expect(screen.getByText("batch_abc123")).toBeInTheDocument(); + expect(screen.queryByText("Successful Requests")).not.toBeInTheDocument(); + }); + + it("never renders for a non-batch call type", () => { + render( + , + ); + + expect(screen.queryByText("Batch Results")).not.toBeInTheDocument(); + }); + }); + it("should show Input Tokens and Output Tokens for anthropic_messages when uncached text_tokens exist", () => { render( + {/* Batch Results */} + {isBatchCallType(logEntry.call_type) && } + {/* Routing */} @@ -374,6 +384,53 @@ function MetricLabel({ label, tooltip, docsUrl }: { label: string; tooltip: stri ); } +/** + * Aggregate per-request outcomes for a batch cost row: batch id, success/failure counts + * from the parsed output and error files, and the models the batch actually ran on. + */ +function BatchResultsSection({ logEntry, metadata }: { logEntry: LogEntry; metadata: Record }) { + const counts = getBatchRequestCounts(metadata); + const batchId = getBatchIdFromRequestId(logEntry.request_id); + const batchModels = getBatchModels(metadata); + if (!counts && !batchId && !batchModels) return null; + + return ( +
+ + + Batch Results + + + + {batchId && ( + + + + )} + {counts && ( + <> + + {formatNumberWithCommas(counts.successful)} + + + {counts.failed > 0 ? ( + + {formatNumberWithCommas(counts.failed)} + + ) : ( + formatNumberWithCommas(counts.failed) + )} + + + )} + {batchModels && {batchModels.join(", ")}} + + + +
+ ); +} + function MetricsSection({ logEntry, metadata }: { logEntry: LogEntry; metadata: Record }) { const completionStartTime = logEntry.completionStartTime; const ttftMs = @@ -391,6 +448,7 @@ function MetricsSection({ logEntry, metadata }: { logEntry: LogEntry; metadata: const uncachedInputTokens = getUncachedInputTextTokens(metadata); const showAnthropicMessagesInputOutput = logEntry.call_type === "anthropic_messages" && uncachedInputTokens !== undefined; + const reasoningTokens = getReasoningTokens(metadata); return (
@@ -416,6 +474,9 @@ function MetricsSection({ logEntry, metadata }: { logEntry: LogEntry; metadata: /> )} + {reasoningTokens !== undefined && reasoningTokens > 0 && ( + {formatNumberWithCommas(reasoningTokens)} + )} ${formatNumberWithCommas(logEntry.spend || 0, 8)} {logEntry.request_duration_ms != null ? (logEntry.request_duration_ms / 1000).toFixed(3) : "-"} s diff --git a/ui/litellm-dashboard/src/components/view_logs/RequestLogsTableColumns.test.tsx b/ui/litellm-dashboard/src/components/view_logs/RequestLogsTableColumns.test.tsx index 3c9e6543c1c..3e8e522afe1 100644 --- a/ui/litellm-dashboard/src/components/view_logs/RequestLogsTableColumns.test.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/RequestLogsTableColumns.test.tsx @@ -134,6 +134,59 @@ describe("Type column", () => { expect(screen.getByText("MCP")).toBeInTheDocument(); }); + + it("marks a batch cost row with the Batch badge instead of LLM", () => { + renderRows([logEntry({ request_id: "batch_1_batch_cost", call_type: "aretrieve_batch" })]); + + expect(screen.getByText("Batch")).toBeInTheDocument(); + expect(screen.queryByText("LLM")).not.toBeInTheDocument(); + }); +}); + +describe("batch rows", () => { + const batchRow = (overrides: Partial): LogEntry => + logEntry({ + request_id: "batch_abc123_batch_cost", + call_type: "aretrieve_batch", + ...overrides, + }); + + it("rolls partial failures into the status badge instead of reporting blanket Success", async () => { + const user = userEvent.setup(); + renderRows([batchRow({ metadata: { batch_successful_requests: 2, batch_failed_requests: 1 } })]); + + expect(screen.queryByText("Success")).not.toBeInTheDocument(); + await user.hover(screen.getByText("2/3 succeeded")); + expect(await screen.findByText("1 of 3 batch requests failed")).toBeInTheDocument(); + }); + + it("keeps the Success badge when every batch request succeeded", () => { + renderRows([batchRow({ metadata: { batch_successful_requests: 3, batch_failed_requests: 0 } })]); + + expect(screen.getByText("Success")).toBeInTheDocument(); + }); + + it("keeps the Failure badge when the batch row itself failed, whatever the counts say", () => { + renderRows([batchRow({ metadata: { status: "failure", batch_successful_requests: 2, batch_failed_requests: 1 } })]); + + expect(screen.getByText("Failure")).toBeInTheDocument(); + expect(screen.queryByText("2/3 succeeded")).not.toBeInTheDocument(); + }); + + it("shows the provider batch id, not the synthetic _batch_cost request id", () => { + renderRows([batchRow({ metadata: { batch_successful_requests: 1, batch_failed_requests: 0 } })]); + + expect(screen.getByText("batch_abc123")).toBeInTheDocument(); + expect(screen.queryByText("batch_abc123_batch_cost")).not.toBeInTheDocument(); + expect(screen.getByText("batch cost")).toBeInTheDocument(); + }); + + it("leaves ordinary request ids untouched", () => { + renderRows([logEntry({ request_id: "chatcmpl-42" })]); + + expect(screen.getByText("chatcmpl-42")).toBeInTheDocument(); + expect(screen.queryByText("batch cost")).not.toBeInTheDocument(); + }); }); describe("Model column", () => { diff --git a/ui/litellm-dashboard/src/components/view_logs/RequestLogsTableColumns.tsx b/ui/litellm-dashboard/src/components/view_logs/RequestLogsTableColumns.tsx index 9d4dc4f7898..b4296abe266 100644 --- a/ui/litellm-dashboard/src/components/view_logs/RequestLogsTableColumns.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/RequestLogsTableColumns.tsx @@ -7,9 +7,10 @@ import { CellTooltip, DateCell, IdCell, MoneyCell, StatusBadge } from "@/compone import { getSpendString } from "@/utils/dataUtils"; import { getProviderLogoAndName } from "../provider_info_helpers"; +import { getBatchIdFromRequestId, getBatchRequestCounts, isBatchCallType } from "./batchLogUtils"; import type { LogEntry } from "./columns"; import { AGENT_CALL_TYPES, MCP_CALL_TYPES } from "./constants"; -import { AgentBadge, AgentIcon, LlmBadge, McpBadge, SparkleIcon, WrenchIcon } from "./TypeBadges"; +import { AgentBadge, AgentIcon, BatchBadge, LlmBadge, McpBadge, SparkleIcon, WrenchIcon } from "./TypeBadges"; export interface RequestLogsTableColumnsDeps { onKeyHashClick: (keyHash: string) => void; @@ -66,6 +67,7 @@ export const getRequestLogsTableColumns = ({ if (sessionCount <= 1) { if (isMcp) return ; if (isAgent) return ; + if (isBatchCallType(log.call_type)) return ; return ; } @@ -106,6 +108,17 @@ export const getRequestLogsTableColumns = ({ cell: ({ row }) => { const status = readMetaString(row.original.metadata, "status") ?? "Success"; const isSuccess = status.toLowerCase() !== "failure"; + const batchCounts = isSuccess ? getBatchRequestCounts(row.original.metadata) : undefined; + if (batchCounts && batchCounts.failed > 0) { + const total = batchCounts.successful + batchCounts.failed; + return ( + + ); + } return ; }, }, @@ -122,7 +135,19 @@ export const getRequestLogsTableColumns = ({ accessorKey: "request_id", header: "Request ID", enableSorting: false, - cell: ({ row }) => , + cell: ({ row }) => { + const log = row.original; + const batchId = isBatchCallType(log.call_type) ? getBatchIdFromRequestId(log.request_id) : undefined; + if (batchId) { + return ( +
+ + batch cost +
+ ); + } + return ; + }, }, { id: "spend", diff --git a/ui/litellm-dashboard/src/components/view_logs/TypeBadges.test.tsx b/ui/litellm-dashboard/src/components/view_logs/TypeBadges.test.tsx index e3467310265..8a964ea2124 100644 --- a/ui/litellm-dashboard/src/components/view_logs/TypeBadges.test.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/TypeBadges.test.tsx @@ -1,6 +1,6 @@ import { render, screen } from "@testing-library/react"; import { describe, expect, it } from "vitest"; -import { LlmBadge, McpBadge, AgentBadge } from "./TypeBadges"; +import { LlmBadge, McpBadge, AgentBadge, BatchBadge } from "./TypeBadges"; describe("TypeBadges", () => { describe("LlmBadge", () => { @@ -43,4 +43,16 @@ describe("TypeBadges", () => { expect(screen.getByText("12")).toBeInTheDocument(); }); }); + + describe("BatchBadge", () => { + it("should render with default 'Batch' text when no count is provided", () => { + render(); + expect(screen.getByText("Batch")).toBeInTheDocument(); + }); + + it("should render the count when provided", () => { + render(); + expect(screen.getByText("4")).toBeInTheDocument(); + }); + }); }); diff --git a/ui/litellm-dashboard/src/components/view_logs/TypeBadges.tsx b/ui/litellm-dashboard/src/components/view_logs/TypeBadges.tsx index 1ba4365f66e..84079848487 100644 --- a/ui/litellm-dashboard/src/components/view_logs/TypeBadges.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/TypeBadges.tsx @@ -57,6 +57,25 @@ export const AgentIcon = ({ size = 12 }: { size?: number }) => ( ); +/** Stacked-layers icon for Batch API call types (Lucide Layers-style). */ +export const LayersIcon = ({ size = 12 }: { size?: number }) => ( + + + + + +); + export const LlmBadge = ({ count }: { count?: number }) => ( @@ -77,3 +96,10 @@ export const AgentBadge = ({ count }: { count?: number }) => ( {count != null ? count : "Agent"} ); + +export const BatchBadge = ({ count }: { count?: number }) => ( + + + {count != null ? count : "Batch"} + +); diff --git a/ui/litellm-dashboard/src/components/view_logs/batchLogUtils.test.ts b/ui/litellm-dashboard/src/components/view_logs/batchLogUtils.test.ts new file mode 100644 index 00000000000..4da1e389646 --- /dev/null +++ b/ui/litellm-dashboard/src/components/view_logs/batchLogUtils.test.ts @@ -0,0 +1,90 @@ +import { describe, expect, it } from "vitest"; + +import { + getBatchIdFromRequestId, + getBatchModels, + getBatchRequestCounts, + getReasoningTokens, + isBatchCallType, +} from "./batchLogUtils"; + +/** Metadata shape the batch cost poller writes on an aretrieve_batch spend row. */ +const batchCostMetadata = { + batch_models: ["gemini-2.5-flash"], + batch_successful_requests: 2, + batch_failed_requests: 1, + usage_object: { + total_tokens: 270, + prompt_tokens: 14, + completion_tokens: 256, + completion_tokens_details: { text_tokens: 32, reasoning_tokens: 224 }, + }, +}; + +describe("isBatchCallType", () => { + it("recognizes the poller's aretrieve_batch and the create call types", () => { + for (const callType of ["aretrieve_batch", "retrieve_batch", "acreate_batch", "create_batch"]) { + expect(isBatchCallType(callType)).toBe(true); + } + expect(isBatchCallType("acompletion")).toBe(false); + }); +}); + +describe("getBatchRequestCounts", () => { + it("reads both counts off a batch cost row", () => { + expect(getBatchRequestCounts(batchCostMetadata)).toEqual({ successful: 2, failed: 1 }); + }); + + it("returns undefined for a non-batch row and for null counts, so no rollup renders", () => { + expect(getBatchRequestCounts({ status: "success" })).toBeUndefined(); + expect(getBatchRequestCounts({ batch_successful_requests: null, batch_failed_requests: null })).toBeUndefined(); + expect(getBatchRequestCounts(undefined)).toBeUndefined(); + }); + + it("treats a lone present count as the other being 0, for rows logged mid-rollout", () => { + expect(getBatchRequestCounts({ batch_successful_requests: 3 })).toEqual({ successful: 3, failed: 0 }); + }); +}); + +describe("getBatchIdFromRequestId", () => { + it("strips the poller's synthetic _batch_cost suffix down to the provider batch id", () => { + expect(getBatchIdFromRequestId("batch_abc123_batch_cost")).toBe("batch_abc123"); + }); + + it("returns undefined for ordinary request ids and a bare suffix", () => { + expect(getBatchIdFromRequestId("chatcmpl-123")).toBeUndefined(); + expect(getBatchIdFromRequestId("_batch_cost")).toBeUndefined(); + }); +}); + +describe("getBatchModels", () => { + it("returns the model list from metadata.batch_models", () => { + expect(getBatchModels(batchCostMetadata)).toEqual(["gemini-2.5-flash"]); + }); + + it("returns undefined when absent, null, or empty", () => { + expect(getBatchModels({})).toBeUndefined(); + expect(getBatchModels({ batch_models: null })).toBeUndefined(); + expect(getBatchModels({ batch_models: [] })).toBeUndefined(); + }); +}); + +describe("getReasoningTokens", () => { + it("reads reasoning tokens from usage_object on a batch cost row", () => { + expect(getReasoningTokens(batchCostMetadata)).toBe(224); + }); + + it("prefers additional_usage_values, which per-request rows carry", () => { + const metadata = { + additional_usage_values: { completion_tokens_details: { reasoning_tokens: 40 } }, + usage_object: { completion_tokens_details: { reasoning_tokens: 999 } }, + }; + expect(getReasoningTokens(metadata)).toBe(40); + }); + + it("returns undefined when the breakout is null or missing", () => { + expect(getReasoningTokens({ usage_object: { completion_tokens_details: null } })).toBeUndefined(); + expect(getReasoningTokens({})).toBeUndefined(); + expect(getReasoningTokens(undefined)).toBeUndefined(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/view_logs/batchLogUtils.ts b/ui/litellm-dashboard/src/components/view_logs/batchLogUtils.ts new file mode 100644 index 00000000000..ce7793065d2 --- /dev/null +++ b/ui/litellm-dashboard/src/components/view_logs/batchLogUtils.ts @@ -0,0 +1,67 @@ +/** + * Helpers for reading batch-specific fields off a spend log row. + * + * The proxy's batch cost poller (CheckBatchCost) writes one spend log per completed batch + * with request_id "_batch_cost" and call_type "aretrieve_batch", carrying + * batch_models / batch_successful_requests / batch_failed_requests in metadata + * (see litellm/proxy/spend_tracking/spend_tracking_utils.py). + */ + +import { BATCH_CALL_TYPES } from "./constants"; + +export const BATCH_COST_REQUEST_ID_SUFFIX = "_batch_cost"; + +export interface BatchRequestCounts { + successful: number; + failed: number; +} + +export const isBatchCallType = (callType: string): boolean => BATCH_CALL_TYPES.includes(callType); + +const readMetaNumber = (metadata: Record | undefined, key: string): number | undefined => { + const value = metadata?.[key]; + return typeof value === "number" && Number.isFinite(value) ? value : undefined; +}; + +/** + * Per-request outcome counts of a batch cost row. Undefined when the row carries neither + * count (a non-batch row, or a batch logged before counts were tracked). + */ +export const getBatchRequestCounts = ( + metadata: Record | undefined, +): BatchRequestCounts | undefined => { + const successful = readMetaNumber(metadata, "batch_successful_requests"); + const failed = readMetaNumber(metadata, "batch_failed_requests"); + if (successful === undefined && failed === undefined) return undefined; + return { successful: successful ?? 0, failed: failed ?? 0 }; +}; + +/** The provider batch id behind a poller-written "_batch_cost" spend row. */ +export const getBatchIdFromRequestId = (requestId: string): string | undefined => + requestId.endsWith(BATCH_COST_REQUEST_ID_SUFFIX) && requestId.length > BATCH_COST_REQUEST_ID_SUFFIX.length + ? requestId.slice(0, -BATCH_COST_REQUEST_ID_SUFFIX.length) + : undefined; + +/** The models the batch's requests actually ran on, from metadata.batch_models. */ +export const getBatchModels = (metadata: Record | undefined): string[] | undefined => { + const models = metadata?.["batch_models"]; + if (!Array.isArray(models)) return undefined; + const names = models.filter((model): model is string => typeof model === "string" && model !== ""); + return names.length > 0 ? names : undefined; +}; + +/** + * Reasoning tokens aggregated across the row's completion usage. Read from the same two + * metadata containers the drawer already uses for prompt-token details: per-request rows + * carry additional_usage_values, batch cost rows carry usage_object. + */ +export const getReasoningTokens = (metadata: Record | undefined): number | undefined => { + const readDetails = (container: unknown): number | undefined => { + if (typeof container !== "object" || container === null) return undefined; + const details = (container as Record)["completion_tokens_details"]; + if (typeof details !== "object" || details === null) return undefined; + const reasoning = (details as Record)["reasoning_tokens"]; + return typeof reasoning === "number" && Number.isFinite(reasoning) ? reasoning : undefined; + }; + return readDetails(metadata?.["additional_usage_values"]) ?? readDetails(metadata?.["usage_object"]); +}; diff --git a/ui/litellm-dashboard/src/components/view_logs/constants.ts b/ui/litellm-dashboard/src/components/view_logs/constants.ts index 1c17f398e35..51889ae7d04 100644 --- a/ui/litellm-dashboard/src/components/view_logs/constants.ts +++ b/ui/litellm-dashboard/src/components/view_logs/constants.ts @@ -21,6 +21,9 @@ export const MCP_CALL_TYPES = ["call_mcp_tool", "list_mcp_tools"]; /** Call types that represent agent/A2A requests (e.g. asend_message). */ export const AGENT_CALL_TYPES = ["asend_message"]; +/** Call types that represent Batch API operations (creation and retrieval, sync and async). */ +export const BATCH_CALL_TYPES = ["acreate_batch", "create_batch", "aretrieve_batch", "retrieve_batch"]; + export const QUICK_SELECT_OPTIONS: { label: string; value: number; unit: string }[] = [ { label: "Last Minute", value: 1, unit: "minutes" }, { label: "Last 15 Minutes", value: 15, unit: "minutes" }, From c276813cb46f57b1934d5e18bedfc53a09e74df0 Mon Sep 17 00:00:00 2001 From: mubashir1osmani Date: Thu, 3 Sep 2026 17:23:53 -0400 Subject: [PATCH 03/34] feat(batches): enrich batch cost rows with breakdown, identity, session, and org spend --- .../proxy/common_utils/check_batch_cost.py | 48 ++++++++++---- litellm/batches/batch_utils.py | 52 +++++++++------- litellm/litellm_core_utils/litellm_logging.py | 15 +++++ .../spend_tracking/spend_tracking_utils.py | 27 +++++++- .../test_batches_logging_unit_tests.py | 60 ++++++++++++++++-- .../proxy_unit_tests/test_check_batch_cost.py | 45 ++++++++++++++ .../test_litellm/batches/test_batch_utils.py | 34 +++++++--- .../test_spend_tracking_utils.py | 62 +++++++++++++++++++ .../view_logs/RequestLogsTableColumns.tsx | 4 +- 9 files changed, 295 insertions(+), 52 deletions(-) diff --git a/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py b/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py index 354a6ed2fd0..be6e52681fb 100644 --- a/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py +++ b/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py @@ -112,35 +112,39 @@ class CheckBatchCost: verbose_proxy_logger.error(f"CheckBatchCost: could not look up user {user_id} for batch {batch_id}: {e}") return {} - async def _get_key_alias(self, batch_id: str, api_key: str | None) -> str | None: - """Resolve the creating virtual key's alias from its hashed token.""" + async def _get_key_attribution(self, batch_id: str, api_key: str | None) -> tuple[str | None, str | None]: + """Resolve the creating virtual key's (alias, org_id) from its hashed token.""" if not api_key: - return None + return None, None try: key_row: prisma_models.LiteLLM_VerificationToken | None = ( await self.prisma_client.db.litellm_verificationtoken.find_unique( where={"token": api_key} ) ) - return getattr(key_row, "key_alias", None) if key_row is not None else None + if key_row is None: + return None, None + return getattr(key_row, "key_alias", None), getattr(key_row, "org_id", None) except Exception as e: verbose_proxy_logger.error(f"CheckBatchCost: could not look up key alias for batch {batch_id}: {e}") - return None + return None, None - async def _get_team_alias(self, team_id: str | None) -> str | None: - """Resolve a team's alias from its id.""" + async def _get_team_attribution(self, team_id: str | None) -> tuple[str | None, str | None]: + """Resolve a team's (alias, organization_id) from its id.""" if not team_id: - return None + return None, None try: team_row: prisma_models.LiteLLM_TeamTable | None = ( await self.prisma_client.db.litellm_teamtable.find_unique( where={"team_id": team_id} ) ) - return getattr(team_row, "team_alias", None) if team_row is not None else None + if team_row is None: + return None, None + return getattr(team_row, "team_alias", None), getattr(team_row, "organization_id", None) except Exception as e: verbose_proxy_logger.error(f"CheckBatchCost: could not look up team alias for team {team_id}: {e}") - return None + return None, None async def _build_creator_attribution_metadata( self, job: "LiteLLM_ManagedObjectTable", batch_id: str @@ -153,6 +157,10 @@ class CheckBatchCost: user_api_key_alias; when it has no alias, or the key has since been rotated or deleted, the field keeps the creating user's alias that _get_user_info filled in, because a resolvable name is more useful on the spend row than a null. + + user_api_key_org_id must be resolved here too: the spend update writer reads it + off this metadata to increment organization spend, so leaving it out silently + drops batch cost from org accounting for keys and teams that belong to one. """ api_key = getattr(job, "api_key", None) team_id = getattr(job, "team_id", None) @@ -165,12 +173,15 @@ class CheckBatchCost: **(await self._get_user_info(batch_id, job.created_by)), } - key_alias = await self._get_key_alias(batch_id, api_key) + key_alias, key_org_id = await self._get_key_attribution(batch_id, api_key) if key_alias is not None: metadata["user_api_key_alias"] = key_alias - team_alias = await self._get_team_alias(team_id) + team_alias, team_org_id = await self._get_team_attribution(team_id) if team_alias is not None: metadata["user_api_key_team_alias"] = team_alias + org_id: Final = key_org_id or team_org_id + if org_id is not None: + metadata["user_api_key_org_id"] = org_id if isinstance(request_tags, list) and request_tags: metadata["tags"] = [tag for tag in request_tags if isinstance(tag, str)] @@ -804,6 +815,7 @@ class CheckBatchCost: function_id=str(uuid.uuid4()), ) + deployment_api_base: Final = deployment_info.litellm_params.api_base logging_obj.update_environment_variables( litellm_params={ # set the user-agent header so that S3 callback consumers can easily identify CheckBatchCost callbacks @@ -812,9 +824,17 @@ class CheckBatchCost: "user-agent": CHECK_BATCH_COST_USER_AGENT, } }, - "metadata": await self._build_creator_attribution_metadata(job, batch_id), + **({"api_base": deployment_api_base} if deployment_api_base else {}), + "metadata": { + **(await self._build_creator_attribution_metadata(job, batch_id)), + # spend logs read the deployment identity off these metadata keys, so + # without them the batch cost row carries no model_id or model_group + "model_info": {"id": model_id}, + "model_group": deployment_info.model_name, + }, }, optional_params={}, + custom_llm_provider=str(llm_provider) if llm_provider else None, ) if not await self._claim_job_for_costing(job): @@ -832,6 +852,8 @@ class CheckBatchCost: batch_models=batch_result.models, batch_successful_requests=batch_result.successful_requests, batch_failed_requests=batch_result.failed_requests, + batch_prompt_cost=batch_result.prompt_cost, + batch_completion_cost=batch_result.completion_cost, ) except Exception: await self._release_job_claim(job) diff --git a/litellm/batches/batch_utils.py b/litellm/batches/batch_utils.py index 3831f57a10d..a688e9f69fb 100644 --- a/litellm/batches/batch_utils.py +++ b/litellm/batches/batch_utils.py @@ -10,7 +10,7 @@ from litellm._logging import verbose_logger from litellm.litellm_core_utils.get_litellm_params import AWS_CREDENTIAL_KWARGS_KEYS from litellm.litellm_core_utils.llm_cost_calc.utils import parse_prompt_tokens_details from litellm.types.llms.openai import Batch -from litellm.types.utils import CallTypes, ModelInfo, Usage +from litellm.types.utils import ModelInfo, Usage from litellm.utils import token_counter @@ -23,6 +23,8 @@ class BatchCostUsageResult: models: list[str] successful_requests: int failed_requests: int + prompt_cost: float = 0.0 + completion_cost: float = 0.0 async def calculate_batch_cost_and_usage( @@ -130,7 +132,8 @@ class _LineOutcome(Enum): @dataclass(frozen=True, slots=True) class _BatchOutputLineStats: - cost: float + prompt_cost: float + completion_cost: float prompt_tokens: int completion_tokens: int total_tokens: int @@ -193,15 +196,16 @@ def _compute_output_line_stats( raw_model: Final = response_body.get("model") response_model: Final = raw_model if isinstance(raw_model, str) and raw_model else None completion_details: Final = usage.completion_tokens_details + line_prompt_cost, line_completion_cost = _output_line_cost( + usage=usage, + custom_llm_provider=custom_llm_provider, + model_name=model_name, + response_model=response_model, + model_info=model_info, + ) return _BatchOutputLineStats( - cost=_output_line_cost( - response_body=response_body, - usage=usage, - custom_llm_provider=custom_llm_provider, - model_name=model_name, - response_model=response_model, - model_info=model_info, - ), + prompt_cost=line_prompt_cost, + completion_cost=line_completion_cost, prompt_tokens=usage.prompt_tokens, completion_tokens=usage.completion_tokens, total_tokens=usage.total_tokens, @@ -213,31 +217,24 @@ def _compute_output_line_stats( def _output_line_cost( - response_body: Mapping[str, Any], usage: Usage, custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic", "bedrock"], model_name: str | None, response_model: str | None, model_info: ModelInfo | None, -) -> float: +) -> tuple[float, float]: + """(prompt_cost, completion_cost) for one output line, priced at batch rates.""" from litellm.cost_calculator import batch_cost_calculator - if model_info is None and custom_llm_provider not in ("anthropic", "bedrock"): - return litellm.completion_cost( - completion_response=response_body, - custom_llm_provider=custom_llm_provider, - call_type=CallTypes.aretrieve_batch.value, - ) cost_model: Final = ( model_name if custom_llm_provider == "bedrock" and model_name else response_model or model_name or "" ) - prompt_cost, completion_cost = batch_cost_calculator( + return batch_cost_calculator( usage=usage, model=cost_model, custom_llm_provider=custom_llm_provider, model_info=model_info, ) - return prompt_cost + completion_cost def _aggregate_batch_cost_usage_models( @@ -270,7 +267,9 @@ def _aggregate_batch_cost_usage_models( **cache_token_params, ) batch_models: Final = [model_name] if model_name else [stats.model for stats in line_stats if stats.model] - total_cost: Final = sum((stats.cost for stats in line_stats), 0.0) + total_prompt_cost: Final = sum((stats.prompt_cost for stats in line_stats), 0.0) + total_completion_cost: Final = sum((stats.completion_cost for stats in line_stats), 0.0) + total_cost: Final = total_prompt_cost + total_completion_cost verbose_logger.debug( "batch output aggregate: cost=%s usage=%s models=%s successful=%d failed=%d", total_cost, @@ -285,6 +284,8 @@ def _aggregate_batch_cost_usage_models( models=batch_models, successful_requests=successful_requests, failed_requests=failed_requests, + prompt_cost=total_prompt_cost, + completion_cost=total_completion_cost, ) @@ -309,7 +310,8 @@ def calculate_vertex_ai_batch_cost_and_usage( """ from litellm.cost_calculator import batch_cost_calculator - total_cost = 0.0 + total_prompt_cost = 0.0 # rebind-ok: loop accumulator, matches total_tokens below + total_completion_cost = 0.0 # rebind-ok: loop accumulator, matches total_tokens below total_tokens = 0 prompt_tokens = 0 completion_tokens = 0 @@ -341,7 +343,8 @@ def calculate_vertex_ai_batch_cost_and_usage( model=actual_model_name, custom_llm_provider="vertex_ai", ) - total_cost += p_cost + c_cost + total_prompt_cost += p_cost + total_completion_cost += c_cost except Exception as e: verbose_logger.debug("vertex_ai batch cost calculation error for line: %s", str(e)) @@ -349,6 +352,7 @@ def calculate_vertex_ai_batch_cost_and_usage( completion_tokens += _completion total_tokens += _total + total_cost: Final = total_prompt_cost + total_completion_cost verbose_logger.info( "vertex_ai batch cost: cost=%s, prompt=%d, completion=%d, total=%d, successful=%d, failed=%d", total_cost, @@ -369,6 +373,8 @@ def calculate_vertex_ai_batch_cost_and_usage( models=[actual_model_name], successful_requests=successful_requests, failed_requests=failed_requests, + prompt_cost=total_prompt_cost, + completion_cost=total_completion_cost, ) diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index f54eeca5178..3a616e7961d 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -2904,6 +2904,15 @@ class Logging(LiteLLMLoggingBaseClass): result._hidden_params["batch_successful_requests"] = batch_successful_requests # pyright: ignore[reportPrivateUsage] # rebind-ok: same result._hidden_params pattern as response_cost/batch_models above result._hidden_params["batch_failed_requests"] = batch_failed_requests # pyright: ignore[reportPrivateUsage] # rebind-ok: same pattern as above result.usage = batch_usage + batch_prompt_cost: Final = kwargs.get("batch_prompt_cost", None) + batch_completion_cost: Final = kwargs.get("batch_completion_cost", None) + if batch_prompt_cost is not None and batch_completion_cost is not None: + self.set_cost_breakdown( + input_cost=batch_prompt_cost, + output_cost=batch_completion_cost, + total_cost=batch_cost, + cost_for_built_in_tools_cost_usd_dollar=0.0, + ) elif should_compute_batch_data: batch_result: Final = await _handle_completed_batch( @@ -2919,6 +2928,12 @@ class Logging(LiteLLMLoggingBaseClass): result._hidden_params["batch_successful_requests"] = batch_result.successful_requests # pyright: ignore[reportPrivateUsage] # rebind-ok: same pattern as above result._hidden_params["batch_failed_requests"] = batch_result.failed_requests # pyright: ignore[reportPrivateUsage] # rebind-ok: same pattern as above result.usage = batch_result.usage + self.set_cost_breakdown( + input_cost=batch_result.prompt_cost, + output_cost=batch_result.completion_cost, + total_cost=batch_result.cost, + cost_for_built_in_tools_cost_usd_dollar=0.0, + ) start_time, end_time, result = self._success_handler_helper_fn( start_time=start_time, diff --git a/litellm/proxy/spend_tracking/spend_tracking_utils.py b/litellm/proxy/spend_tracking/spend_tracking_utils.py index a37c3ba4405..10db82d1b64 100644 --- a/litellm/proxy/spend_tracking/spend_tracking_utils.py +++ b/litellm/proxy/spend_tracking/spend_tracking_utils.py @@ -582,6 +582,7 @@ def get_logging_payload(kwargs, response_obj, start_time, end_time) -> SpendLogs metadata=metadata, standard_logging_payload=standard_logging_payload, omit_when_missing=_omits_session_id_when_missing(metadata), + batch_trace_session_id=_get_batch_trace_session_id(call_type=call_type, request_id=id), ), request_duration_ms=_get_request_duration_ms(start_time, end_time), status=_get_status_for_spend_log( @@ -620,20 +621,44 @@ def _omits_session_id_when_missing(metadata: Mapping[str, object] | None) -> boo return general_settings.get("missing_session_id") == "omit" +_BATCH_TRACE_CALL_TYPES: Final = frozenset( + { + CallTypes.create_batch.value, + CallTypes.acreate_batch.value, + CallTypes.retrieve_batch.value, + CallTypes.aretrieve_batch.value, + } +) + + +def _get_batch_trace_session_id(call_type: str | None, request_id: str | None) -> str | None: + """A batch's create row and its poller-written cost row both derive their request id + from the same batch id (the cost row appends BATCH_COST_REQUEST_ID_SUFFIX), so using + that id as the session groups the batch lifecycle into one trace on the logs UI. The + poller builds its own logging context, so per-request trace ids can never link them.""" + if call_type not in _BATCH_TRACE_CALL_TYPES or not request_id: + return None + return request_id.removesuffix(BATCH_COST_REQUEST_ID_SUFFIX) + + def _get_session_id_for_spend_log( kwargs: Mapping[str, object], metadata: Mapping[str, object] | None, standard_logging_payload: StandardLoggingPayload | None, omit_when_missing: bool, + batch_trace_session_id: str | None = None, ) -> str | None: """Under `omit` only `metadata.session_id`, the key Langfuse reads, counts as a session; `litellm_session_id` may - be a copied trace id.""" + be a copied trace id. Batch call types carry a deterministic session derived from the batch id, which outranks + the per-request trace ids because those differ between the create call and the cost poller's row.""" if omit_when_missing: session_id: Final = metadata.get("session_id") if metadata else None return str(session_id) if session_id else None from litellm._uuid import uuid + if batch_trace_session_id is not None: + return batch_trace_session_id if standard_logging_payload is not None and standard_logging_payload.get("trace_id") is not None: return str(standard_logging_payload.get("trace_id")) if kwargs.get("litellm_trace_id") is not None: diff --git a/tests/batches_tests/test_batches_logging_unit_tests.py b/tests/batches_tests/test_batches_logging_unit_tests.py index 5bde40d90b0..73adb391481 100644 --- a/tests/batches_tests/test_batches_logging_unit_tests.py +++ b/tests/batches_tests/test_batches_logging_unit_tests.py @@ -144,18 +144,20 @@ def test_get_batch_job_total_usage_from_file_content(sample_file_content_dict): @pytest.mark.asyncio async def test_batch_cost_calculator(sample_file_content_dict): """ - mock litellm.completion_cost to return 0.5 + mock batch_cost_calculator to return (0.3, 0.2) per line we know sample_file_content_dict has 2 successful responses - so we expect the cost to be 0.5 * 2 = 1.0 + so we expect the cost to be (0.3 + 0.2) * 2 = 1.0, split 0.6 / 0.4 """ - with patch("litellm.completion_cost", return_value=0.5): + with patch("litellm.cost_calculator.batch_cost_calculator", return_value=(0.3, 0.2)): result = _aggregate_batch_cost_usage_models( entries=sample_file_content_dict, custom_llm_provider="openai", ) - assert result.cost == 1.0 # 0.5 * 2 successful responses + assert result.cost == pytest.approx(1.0) # (0.3 + 0.2) * 2 successful responses + assert result.prompt_cost == pytest.approx(0.6) + assert result.completion_cost == pytest.approx(0.4) def test_get_response_from_batch_job_output_file(sample_file_content_dict): @@ -402,6 +404,56 @@ async def test_batch_retrieve_cost_tracking_with_explicit_cost_data(): assert mock_batch.usage == explicit_usage +@pytest.mark.asyncio +async def test_batch_retrieve_explicit_cost_split_sets_cost_breakdown(): + """The poller passes the batch's prompt/completion cost split so the spend row's + cost_breakdown carries real input/output costs; without it the UI's Cost Breakdown + card renders blank for every batch. Regression for the split being dropped.""" + from litellm.litellm_core_utils.litellm_logging import Logging + from litellm.types.utils import CallTypes, LiteLLMBatch + + mock_batch = LiteLLMBatch( + id="batch-breakdown-1", + object="batch", + endpoint="/v1/chat/completions", + errors=None, + input_file_id="file-input-1", + completion_window="24h", + status="completed", + output_file_id="file-output-1", + created_at=1234567890, + ) + mock_batch._hidden_params = {} + + logging_obj = Logging( + model="gpt-5-mini", + messages=[{"role": "user", "content": "test"}], + stream=False, + call_type=CallTypes.aretrieve_batch.value, + litellm_call_id="test-call-breakdown", + function_id="test-function", + start_time=time.time(), + dynamic_success_callbacks=[], + ) + logging_obj.custom_llm_provider = "openai" + + await logging_obj.async_success_handler( + result=mock_batch, + start_time=time.time(), + end_time=time.time() + 1, + batch_cost=0.10, + batch_usage=litellm.Usage(prompt_tokens=200, completion_tokens=100, total_tokens=300), + batch_models=["gpt-5-mini"], + batch_prompt_cost=0.06, + batch_completion_cost=0.04, + ) + + assert logging_obj.cost_breakdown is not None + assert logging_obj.cost_breakdown["input_cost"] == 0.06 + assert logging_obj.cost_breakdown["output_cost"] == 0.04 + assert logging_obj.cost_breakdown["total_cost"] == 0.10 + + @pytest.mark.asyncio async def test_batch_retrieve_cost_tracking_with_unified_file_id_incomplete_batch(): """ diff --git a/tests/proxy_unit_tests/test_check_batch_cost.py b/tests/proxy_unit_tests/test_check_batch_cost.py index ff5e8f89d64..9ee0a36f00b 100644 --- a/tests/proxy_unit_tests/test_check_batch_cost.py +++ b/tests/proxy_unit_tests/test_check_batch_cost.py @@ -2553,6 +2553,51 @@ class TestBatchCostAttribution: assert metadata["user_api_key_alias"] == "prod-key" + @pytest.mark.asyncio + async def test_org_id_comes_from_the_creating_key(self): + """The spend update writer increments organization spend from user_api_key_org_id, + so an org-scoped key's batch cost must carry the key's org id.""" + from types import SimpleNamespace + + instance = self._instance( + key_row=SimpleNamespace(key_alias="prod-key", org_id="org-42"), + team_row=SimpleNamespace(team_alias="Team Alpha", organization_id="org-team"), + ) + + metadata = await instance._build_creator_attribution_metadata(self._job(), "batch-1") + + assert metadata["user_api_key_org_id"] == "org-42" + + @pytest.mark.asyncio + async def test_org_id_falls_back_to_the_team_organization(self): + """A key with no org of its own still books batch spend against its team's + organization, matching how the request path resolves org attribution.""" + from types import SimpleNamespace + + instance = self._instance( + key_row=SimpleNamespace(key_alias="prod-key", org_id=None), + team_row=SimpleNamespace(team_alias="Team Alpha", organization_id="org-team"), + ) + + metadata = await instance._build_creator_attribution_metadata(self._job(), "batch-1") + + assert metadata["user_api_key_org_id"] == "org-team" + + @pytest.mark.asyncio + async def test_no_org_leaves_the_key_unset(self): + """Without any org the key is absent entirely, so the spend writer's org update + stays skipped instead of matching an empty-string organization.""" + from types import SimpleNamespace + + instance = self._instance( + key_row=SimpleNamespace(key_alias="prod-key", org_id=None), + team_row=SimpleNamespace(team_alias="Team Alpha", organization_id=None), + ) + + metadata = await instance._build_creator_attribution_metadata(self._job(), "batch-1") + + assert "user_api_key_org_id" not in metadata + class TestPollPageStarvation: """LIT-5462 regression: a row that can never be costed used to keep its slot in the diff --git a/tests/test_litellm/batches/test_batch_utils.py b/tests/test_litellm/batches/test_batch_utils.py index c86c7c4df03..1fd78870481 100644 --- a/tests/test_litellm/batches/test_batch_utils.py +++ b/tests/test_litellm/batches/test_batch_utils.py @@ -489,7 +489,9 @@ def test_aggregate_counts_successful_and_failed_requests(monkeypatch): def test_aggregate_returns_batch_cost_usage_result_dataclass(monkeypatch): - monkeypatch.setattr(litellm, "completion_cost", lambda **kw: 1.0) + import litellm.cost_calculator as cc + + monkeypatch.setattr(cc, "batch_cost_calculator", lambda **kw: (0.4, 0.6)) result = bu._aggregate_batch_cost_usage_models( entries=[_success_row(usage=_usage(10, 5))], custom_llm_provider="openai" ) @@ -500,6 +502,7 @@ def test_aggregate_returns_batch_cost_usage_result_dataclass(monkeypatch): 1, 0, ) + assert (result.prompt_cost, result.completion_cost) == (0.4, 0.6) # =========================================================================== # @@ -507,15 +510,17 @@ def test_aggregate_returns_batch_cost_usage_result_dataclass(monkeypatch): # =========================================================================== # -def test_cost_from_content_completion_cost_path(monkeypatch): - # model_info is None -> litellm.completion_cost per successful row. +def test_cost_without_model_info_prices_each_row_by_its_response_model(monkeypatch): + # model_info is None -> batch_cost_calculator per successful row, model from the response body. + import litellm.cost_calculator as cc + calls = [] - def _completion_cost(**kw): + def _batch_cost(**kw): calls.append(kw) - return 0.5 + return (0.3, 0.2) - monkeypatch.setattr(litellm, "completion_cost", _completion_cost) + monkeypatch.setattr(cc, "batch_cost_calculator", _batch_cost) rows = [ _success_row(usage=_usage(10, 5)), _failed_row(), # excluded -> not costed @@ -524,8 +529,10 @@ def test_cost_from_content_completion_cost_path(monkeypatch): result = bu._aggregate_batch_cost_usage_models(entries=rows, custom_llm_provider="openai") - assert result.cost == 1.0 # 2 successful * 0.5 + assert result.cost == pytest.approx(1.0) # 2 successful * (0.3 + 0.2) + assert (result.prompt_cost, result.completion_cost) == (pytest.approx(0.6), pytest.approx(0.4)) assert len(calls) == 2 # failed row not costed + assert all(call["model"] == "gpt-4o" and call["model_info"] is None for call in calls) assert result.successful_requests == 2 assert result.failed_requests == 1 @@ -578,7 +585,9 @@ def test_aggregate_consumes_entries_in_a_single_pass(monkeypatch): """A one-shot generator: any implementation that iterates the entries twice (e.g. separate cost and usage passes) sees nothing on the second pass and returns wrong totals for at least one of cost/usage/models.""" - monkeypatch.setattr(litellm, "completion_cost", lambda **kw: 0.5) + import litellm.cost_calculator as cc + + monkeypatch.setattr(cc, "batch_cost_calculator", lambda **kw: (0.25, 0.25)) one_shot = (row for row in [_success_row(usage=_usage(10, 5)), _failed_row(), _success_row(usage=_usage(20, 10))]) result = bu._aggregate_batch_cost_usage_models(entries=one_shot, custom_llm_provider="openai") @@ -753,12 +762,15 @@ def test_vertex_cost_error_in_line_is_swallowed(monkeypatch): @pytest.mark.asyncio async def test_calculate_batch_cost_and_usage_orchestration(monkeypatch): + import litellm.cost_calculator as cc + rows = [_success_row(model="gpt-4o", usage=_usage(10, 5))] - monkeypatch.setattr(litellm, "completion_cost", lambda **kw: 2.5) + monkeypatch.setattr(cc, "batch_cost_calculator", lambda **kw: (1.5, 1.0)) result = await bu.calculate_batch_cost_and_usage(file_content_dictionary=rows, custom_llm_provider="openai") assert result.cost == 2.5 + assert (result.prompt_cost, result.completion_cost) == (1.5, 1.0) assert (result.usage.prompt_tokens, result.usage.completion_tokens, result.usage.total_tokens) == (10, 5, 15) assert result.models == ["gpt-4o"] @@ -1107,8 +1119,10 @@ async def test_handle_completed_batch_orchestration(monkeypatch): async def fake_fetch(batch, custom_llm_provider, litellm_params=None): return _vertex_jsonl(rows) + import litellm.cost_calculator as cc + monkeypatch.setattr(bu, "_fetch_batch_output_file_content", fake_fetch) - monkeypatch.setattr(litellm, "completion_cost", lambda **kw: 3.3) + monkeypatch.setattr(cc, "batch_cost_calculator", lambda **kw: (2.0, 1.3)) result = await bu._handle_completed_batch(_batch("of"), custom_llm_provider="openai") diff --git a/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py b/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py index 323930eee60..be274befd25 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py +++ b/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py @@ -128,6 +128,68 @@ def test_legacy_policy_keeps_trace_id_fallback(): assert len(str(generated)) == 36 +def test_batch_lifecycle_rows_derive_the_same_session_from_the_batch_id(): + """The create call's request id IS the batch id and the poller's cost row appends + _batch_cost to it, so deriving the session from the request id lands both rows in one + trace on the logs UI even though the poller builds a fresh logging context per cycle.""" + from litellm.proxy.spend_tracking.spend_tracking_utils import _get_batch_trace_session_id + + create_session: Final = _get_batch_trace_session_id(call_type="acreate_batch", request_id="batch-uid-1") + cost_session: Final = _get_batch_trace_session_id( + call_type="aretrieve_batch", request_id="batch-uid-1_batch_cost" + ) + assert create_session == cost_session == "batch-uid-1" + + +def test_non_batch_call_types_derive_no_batch_session(): + from litellm.proxy.spend_tracking.spend_tracking_utils import _get_batch_trace_session_id + + assert _get_batch_trace_session_id(call_type="acompletion", request_id="chatcmpl-1") is None + + +def test_batch_session_outranks_the_per_request_trace_id(): + """Each batch lifecycle call carries its own auto-generated trace id, so letting the + trace id win would scatter the rows across sessions again.""" + session_id: Final = _get_session_id_for_spend_log( + kwargs={"litellm_trace_id": "trace-abc"}, + metadata={"trace_id": "trace-abc"}, + standard_logging_payload=_TRACE_ONLY_STANDARD_LOGGING, + omit_when_missing=False, + batch_trace_session_id="batch-uid-1", + ) + assert session_id == "batch-uid-1" + + +def test_omit_policy_still_suppresses_batch_sessions(): + session_id: Final = _get_session_id_for_spend_log( + kwargs={}, + metadata=None, + standard_logging_payload=None, + omit_when_missing=True, + batch_trace_session_id="batch-uid-1", + ) + assert session_id is None + + +def test_get_logging_payload_groups_batch_create_and_cost_rows_in_one_session(): + def _payload(call_type: str) -> SpendLogsPayload: + return get_logging_payload( + kwargs={ + "call_type": call_type, + "model": "gpt-4o-mini", + "litellm_params": {"metadata": {"user_api_key": "test-key"}}, + }, + response_obj=litellm.ModelResponse(id="batch-uid-1", choices=[], usage=litellm.Usage()), + start_time=datetime.datetime.now(timezone.utc), + end_time=datetime.datetime.now(timezone.utc), + ) + + create_payload: Final = _payload("acreate_batch") + cost_payload: Final = _payload("aretrieve_batch") + assert cost_payload["request_id"] == "batch-uid-1_batch_cost" + assert create_payload["session_id"] == cost_payload["session_id"] == "batch-uid-1" + + @pytest.mark.parametrize( ("request_metadata", "expected"), [ diff --git a/ui/litellm-dashboard/src/components/view_logs/RequestLogsTableColumns.tsx b/ui/litellm-dashboard/src/components/view_logs/RequestLogsTableColumns.tsx index b4296abe266..a444acb9517 100644 --- a/ui/litellm-dashboard/src/components/view_logs/RequestLogsTableColumns.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/RequestLogsTableColumns.tsx @@ -64,10 +64,12 @@ export const getRequestLogsTableColumns = ({ const sessionAgentCount = log.session_agent_count ?? (isAgent ? sessionCount : 0); const sessionMcpCount = log.mcp_tool_call_count ?? (isMcp ? sessionCount : 0); + if (isBatchCallType(log.call_type)) { + return 1 ? sessionCount : undefined} />; + } if (sessionCount <= 1) { if (isMcp) return ; if (isAgent) return ; - if (isBatchCallType(log.call_type)) return ; return ; } From 06c860b60a5c4a2d14a02dead9c809b8b7eaf22f Mon Sep 17 00:00:00 2001 From: mubashir1osmani Date: Thu, 3 Sep 2026 18:09:18 -0400 Subject: [PATCH 04/34] fix(ui): type batch results metadata as unknown instead of any --- .../components/view_logs/LogDetailsDrawer/LogDetailContent.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailContent.tsx b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailContent.tsx index 8e0f041b88d..710d82b2f9d 100644 --- a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailContent.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailContent.tsx @@ -388,7 +388,7 @@ function MetricLabel({ label, tooltip, docsUrl }: { label: string; tooltip: stri * Aggregate per-request outcomes for a batch cost row: batch id, success/failure counts * from the parsed output and error files, and the models the batch actually ran on. */ -function BatchResultsSection({ logEntry, metadata }: { logEntry: LogEntry; metadata: Record }) { +function BatchResultsSection({ logEntry, metadata }: { logEntry: LogEntry; metadata: Record }) { const counts = getBatchRequestCounts(metadata); const batchId = getBatchIdFromRequestId(logEntry.request_id); const batchModels = getBatchModels(metadata); From 538cd2c3b00d31fcb516c8326de87883e0e27f94 Mon Sep 17 00:00:00 2001 From: mubashir1osmani Date: Thu, 3 Sep 2026 18:16:31 -0400 Subject: [PATCH 05/34] fix(batches): narrow batch cost kwargs before the breakdown and drop node access in test --- litellm/litellm_core_utils/litellm_logging.py | 6 +++++- .../LogDetailsDrawer/LogDetailContent.test.tsx | 12 +++++++----- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 3a616e7961d..03486f4f729 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -2906,7 +2906,11 @@ class Logging(LiteLLMLoggingBaseClass): result.usage = batch_usage batch_prompt_cost: Final = kwargs.get("batch_prompt_cost", None) batch_completion_cost: Final = kwargs.get("batch_completion_cost", None) - if batch_prompt_cost is not None and batch_completion_cost is not None: + if ( + isinstance(batch_prompt_cost, float) + and isinstance(batch_completion_cost, float) + and isinstance(batch_cost, float) + ): self.set_cost_breakdown( input_cost=batch_prompt_cost, output_cost=batch_completion_cost, diff --git a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailContent.test.tsx b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailContent.test.tsx index 91778d2a98a..a893e9bffd0 100644 --- a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailContent.test.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailContent.test.tsx @@ -163,11 +163,13 @@ describe("LogDetailContent", () => { />, ); - const section = screen.getByText("Batch Results").closest('[data-slot="card"]') as HTMLElement; - expect(within(section).getByText("batch_abc123")).toBeInTheDocument(); - expect(within(section).getByText("2")).toBeInTheDocument(); - expect(within(section).getByText("1")).toBeInTheDocument(); - expect(within(section).getByText("gemini-2.5-flash")).toBeInTheDocument(); + expect(screen.getByText("Batch Results")).toBeInTheDocument(); + expect(screen.getByText("batch_abc123")).toBeInTheDocument(); + expect(screen.getByText("Successful Requests")).toBeInTheDocument(); + expect(screen.getByText("2")).toBeInTheDocument(); + expect(screen.getByText("Failed Requests")).toBeInTheDocument(); + expect(screen.getByText("1")).toBeInTheDocument(); + expect(screen.getByText("gemini-2.5-flash")).toBeInTheDocument(); }); it("still renders the batch id when a legacy row carries no counts", () => { From 7fb4b427cec0db7d3bc85dd6c2055ca987f2390b Mon Sep 17 00:00:00 2001 From: mubashir1osmani Date: Thu, 3 Sep 2026 18:36:05 -0400 Subject: [PATCH 06/34] feat(batches): snapshot the creating key's org on the managed object row --- .../proxy/common_utils/check_batch_cost.py | 61 +++++++++++++------ .../proxy/hooks/managed_files.py | 1 + .../migration.sql | 4 ++ .../litellm_proxy_extras/schema.prisma | 1 + litellm/models/managed_files.py | 1 + schema.prisma | 1 + .../proxy_unit_tests/test_check_batch_cost.py | 21 ++++++- ..._batch_update_db_managed_output_file_id.py | 4 +- 8 files changed, 74 insertions(+), 20 deletions(-) create mode 100644 litellm-proxy-extras/litellm_proxy_extras/migrations/20260903230000_add_org_id_to_managed_object_table/migration.sql diff --git a/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py b/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py index be6e52681fb..595a8d04bed 100644 --- a/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py +++ b/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py @@ -112,39 +112,66 @@ class CheckBatchCost: verbose_proxy_logger.error(f"CheckBatchCost: could not look up user {user_id} for batch {batch_id}: {e}") return {} - async def _get_key_attribution(self, batch_id: str, api_key: str | None) -> tuple[str | None, str | None]: - """Resolve the creating virtual key's (alias, org_id) from its hashed token.""" + async def _get_key_alias(self, batch_id: str, api_key: str | None) -> str | None: + """Resolve the creating virtual key's alias from its hashed token.""" if not api_key: - return None, None + return None try: key_row: prisma_models.LiteLLM_VerificationToken | None = ( await self.prisma_client.db.litellm_verificationtoken.find_unique( where={"token": api_key} ) ) - if key_row is None: - return None, None - return getattr(key_row, "key_alias", None), getattr(key_row, "org_id", None) + return getattr(key_row, "key_alias", None) if key_row is not None else None except Exception as e: verbose_proxy_logger.error(f"CheckBatchCost: could not look up key alias for batch {batch_id}: {e}") - return None, None + return None - async def _get_team_attribution(self, team_id: str | None) -> tuple[str | None, str | None]: - """Resolve a team's (alias, organization_id) from its id.""" + async def _get_team_alias(self, team_id: str | None) -> str | None: + """Resolve a team's alias from its id.""" if not team_id: - return None, None + return None try: team_row: prisma_models.LiteLLM_TeamTable | None = ( await self.prisma_client.db.litellm_teamtable.find_unique( where={"team_id": team_id} ) ) - if team_row is None: - return None, None - return getattr(team_row, "team_alias", None), getattr(team_row, "organization_id", None) + return getattr(team_row, "team_alias", None) if team_row is not None else None except Exception as e: verbose_proxy_logger.error(f"CheckBatchCost: could not look up team alias for team {team_id}: {e}") - return None, None + return None + + async def _get_org_id(self, job: "LiteLLM_ManagedObjectTable", batch_id: str) -> str | None: + """Organization to bill the batch against, snapshotted on the row at creation + like team_id. Rows created before the org_id column existed carry None, so they + fall back to the creating key's org (or its team's) as resolved today.""" + org_id = getattr(job, "org_id", None) + if org_id: + return org_id + api_key = getattr(job, "api_key", None) + team_id = getattr(job, "team_id", None) + try: + if api_key: + key_row: prisma_models.LiteLLM_VerificationToken | None = ( + await self.prisma_client.db.litellm_verificationtoken.find_unique( + where={"token": api_key} + ) + ) + key_org_id = getattr(key_row, "org_id", None) if key_row is not None else None + if key_org_id: + return key_org_id + if team_id: + team_row: prisma_models.LiteLLM_TeamTable | None = ( + await self.prisma_client.db.litellm_teamtable.find_unique( + where={"team_id": team_id} + ) + ) + return getattr(team_row, "organization_id", None) if team_row is not None else None + return None + except Exception as e: + verbose_proxy_logger.error(f"CheckBatchCost: could not resolve org for batch {batch_id}: {e}") + return None async def _build_creator_attribution_metadata( self, job: "LiteLLM_ManagedObjectTable", batch_id: str @@ -173,13 +200,13 @@ class CheckBatchCost: **(await self._get_user_info(batch_id, job.created_by)), } - key_alias, key_org_id = await self._get_key_attribution(batch_id, api_key) + key_alias = await self._get_key_alias(batch_id, api_key) if key_alias is not None: metadata["user_api_key_alias"] = key_alias - team_alias, team_org_id = await self._get_team_attribution(team_id) + team_alias = await self._get_team_alias(team_id) if team_alias is not None: metadata["user_api_key_team_alias"] = team_alias - org_id: Final = key_org_id or team_org_id + org_id: Final = await self._get_org_id(job, batch_id) if org_id is not None: metadata["user_api_key_org_id"] = org_id if isinstance(request_tags, list) and request_tags: diff --git a/enterprise/litellm_enterprise/proxy/hooks/managed_files.py b/enterprise/litellm_enterprise/proxy/hooks/managed_files.py index 5cfcf6129f0..4f0ca787280 100644 --- a/enterprise/litellm_enterprise/proxy/hooks/managed_files.py +++ b/enterprise/litellm_enterprise/proxy/hooks/managed_files.py @@ -349,6 +349,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): "file_purpose": file_purpose, "created_by": resolve_resource_owner_id(user_api_key_dict), "team_id": user_api_key_dict.team_id, + "org_id": user_api_key_dict.org_id, "updated_by": user_api_key_dict.user_id, "status": file_object.status, **attribution_columns, diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20260903230000_add_org_id_to_managed_object_table/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260903230000_add_org_id_to_managed_object_table/migration.sql new file mode 100644 index 00000000000..bbe980bb66f --- /dev/null +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260903230000_add_org_id_to_managed_object_table/migration.sql @@ -0,0 +1,4 @@ +-- Add org_id column to LiteLLM_ManagedObjectTable +-- Snapshots the creating key's organization at submission time, like team_id, +-- so CheckBatchCost can bill organization spend hours later without re-resolving +ALTER TABLE "LiteLLM_ManagedObjectTable" ADD COLUMN IF NOT EXISTS "org_id" TEXT; diff --git a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma index 7604ceadf7a..e386446d5bd 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma +++ b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma @@ -1034,6 +1034,7 @@ model LiteLLM_ManagedObjectTable { // for batches or finetuning jobs which use t created_at DateTime @default(now()) created_by String? team_id String? + org_id String? // creating key's organization at submission time; CheckBatchCost bills org spend against it api_key String? request_tags Json? @default("[]") updated_at DateTime @updatedAt diff --git a/litellm/models/managed_files.py b/litellm/models/managed_files.py index 23d70ef5c48..c90f9b535ea 100644 --- a/litellm/models/managed_files.py +++ b/litellm/models/managed_files.py @@ -32,6 +32,7 @@ class LiteLLM_ManagedObjectTable(LiteLLMPydanticObjectBase): file_object: LiteLLMBatch | LiteLLMFineTuningJob | ResponsesAPIResponse created_by: str | None = None team_id: str | None = None + org_id: str | None = None class LiteLLM_ManagedVectorStoreTable(LiteLLMPydanticObjectBase): diff --git a/schema.prisma b/schema.prisma index 7604ceadf7a..e386446d5bd 100644 --- a/schema.prisma +++ b/schema.prisma @@ -1034,6 +1034,7 @@ model LiteLLM_ManagedObjectTable { // for batches or finetuning jobs which use t created_at DateTime @default(now()) created_by String? team_id String? + org_id String? // creating key's organization at submission time; CheckBatchCost bills org spend against it api_key String? request_tags Json? @default("[]") updated_at DateTime @updatedAt diff --git a/tests/proxy_unit_tests/test_check_batch_cost.py b/tests/proxy_unit_tests/test_check_batch_cost.py index 9ee0a36f00b..92b946e19b1 100644 --- a/tests/proxy_unit_tests/test_check_batch_cost.py +++ b/tests/proxy_unit_tests/test_check_batch_cost.py @@ -2553,10 +2553,27 @@ class TestBatchCostAttribution: assert metadata["user_api_key_alias"] == "prod-key" + @pytest.mark.asyncio + async def test_org_id_snapshotted_on_the_row_wins(self): + """The org_id column captures the creating key's organization at submission time, + like team_id, so a key later moved to another org still bills the original one.""" + from types import SimpleNamespace + + instance = self._instance( + key_row=SimpleNamespace(key_alias="prod-key", org_id="org-moved-to"), + team_row=SimpleNamespace(team_alias="Team Alpha", organization_id="org-team"), + ) + + metadata = await instance._build_creator_attribution_metadata( + self._job(org_id="org-at-creation"), "batch-1" + ) + + assert metadata["user_api_key_org_id"] == "org-at-creation" + @pytest.mark.asyncio async def test_org_id_comes_from_the_creating_key(self): - """The spend update writer increments organization spend from user_api_key_org_id, - so an org-scoped key's batch cost must carry the key's org id.""" + """The spend update writer increments organization spend from user_api_key_org_id. + A legacy row without the org_id column falls back to the creating key's org.""" from types import SimpleNamespace instance = self._instance( diff --git a/tests/test_litellm/enterprise/proxy/test_batch_update_db_managed_output_file_id.py b/tests/test_litellm/enterprise/proxy/test_batch_update_db_managed_output_file_id.py index ebd33aa2e53..63884e0a779 100644 --- a/tests/test_litellm/enterprise/proxy/test_batch_update_db_managed_output_file_id.py +++ b/tests/test_litellm/enterprise/proxy/test_batch_update_db_managed_output_file_id.py @@ -390,7 +390,7 @@ async def test_store_unified_object_id_persists_key_and_tags_on_create(): """Regression (spend loss): the batch create persists the creating key hash and tags so CheckBatchCost can write an attributed spend row instead of a blank one the DB drops.""" instance, store = _in_memory_managed_files() - creator = UserAPIKeyAuth(user_id="alice", team_id="team-alpha", api_key="hash-alice") + creator = UserAPIKeyAuth(user_id="alice", team_id="team-alpha", api_key="hash-alice", org_id="org-acme") await instance.store_unified_object_id( unified_object_id="unified-b", @@ -407,6 +407,7 @@ async def test_store_unified_object_id_persists_key_and_tags_on_create(): assert row["api_key"] == "hash-alice" assert row["created_by"] == "alice" assert row["team_id"] == "team-alpha" + assert row["org_id"] == "org-acme" assert row["request_tags"].data == ["env:prod"] @@ -471,6 +472,7 @@ async def test_store_unified_object_id_attribution_columns_are_write_once(): upsert_data = instance.prisma_client.db.litellm_managedobjecttable.upsert.call_args.kwargs["data"] assert "api_key" not in upsert_data["update"] assert "request_tags" not in upsert_data["update"] + assert "org_id" not in upsert_data["update"] @pytest.mark.asyncio From 9859d1e64ca1e6930d2f3e440a51222e834c7a5f Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 3 Sep 2026 22:36:28 +0000 Subject: [PATCH 07/34] chore: sync schema.prisma copies from root --- litellm/proxy/schema.prisma | 1 + 1 file changed, 1 insertion(+) diff --git a/litellm/proxy/schema.prisma b/litellm/proxy/schema.prisma index 7604ceadf7a..e386446d5bd 100644 --- a/litellm/proxy/schema.prisma +++ b/litellm/proxy/schema.prisma @@ -1034,6 +1034,7 @@ model LiteLLM_ManagedObjectTable { // for batches or finetuning jobs which use t created_at DateTime @default(now()) created_by String? team_id String? + org_id String? // creating key's organization at submission time; CheckBatchCost bills org spend against it api_key String? request_tags Json? @default("[]") updated_at DateTime @updatedAt From 0d7976116eff0a9090f846e0066067bd867e67fb Mon Sep 17 00:00:00 2001 From: mubashir1osmani Date: Thu, 3 Sep 2026 18:51:18 -0400 Subject: [PATCH 08/34] fix(batches): resolve team-scoped keys' org at creation for the snapshot --- .../proxy/hooks/managed_files.py | 20 +++++++++++++- ..._batch_update_db_managed_output_file_id.py | 27 +++++++++++++++++++ 2 files changed, 46 insertions(+), 1 deletion(-) diff --git a/enterprise/litellm_enterprise/proxy/hooks/managed_files.py b/enterprise/litellm_enterprise/proxy/hooks/managed_files.py index 4f0ca787280..2ed446ca44e 100644 --- a/enterprise/litellm_enterprise/proxy/hooks/managed_files.py +++ b/enterprise/litellm_enterprise/proxy/hooks/managed_files.py @@ -277,6 +277,24 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): ) verbose_logger.debug(f"LiteLLM Managed File object with id={file_id} stored in db: {result}") + async def _resolve_creator_org_id(self, user_api_key_dict: UserAPIKeyAuth) -> Optional[str]: + """Organization to snapshot on the managed object row, like team_id. A key that + belongs to an org only through its team carries no org_id on the auth object, so + resolve the team's organization at creation time; costing then bills the org the + batch was submitted under even if the key or team moves before it completes.""" + if user_api_key_dict.org_id: + return user_api_key_dict.org_id + if not user_api_key_dict.team_id: + return None + try: + team_row = await self.prisma_client.db.litellm_teamtable.find_unique( + where={"team_id": user_api_key_dict.team_id} + ) + return getattr(team_row, "organization_id", None) if team_row is not None else None + except Exception as e: + verbose_logger.warning(f"could not resolve org for managed object attribution: {e}") + return None + async def store_unified_object_id( self, unified_object_id: str, @@ -349,7 +367,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): "file_purpose": file_purpose, "created_by": resolve_resource_owner_id(user_api_key_dict), "team_id": user_api_key_dict.team_id, - "org_id": user_api_key_dict.org_id, + "org_id": await self._resolve_creator_org_id(user_api_key_dict), "updated_by": user_api_key_dict.user_id, "status": file_object.status, **attribution_columns, diff --git a/tests/test_litellm/enterprise/proxy/test_batch_update_db_managed_output_file_id.py b/tests/test_litellm/enterprise/proxy/test_batch_update_db_managed_output_file_id.py index 63884e0a779..b5865ab4a13 100644 --- a/tests/test_litellm/enterprise/proxy/test_batch_update_db_managed_output_file_id.py +++ b/tests/test_litellm/enterprise/proxy/test_batch_update_db_managed_output_file_id.py @@ -375,6 +375,7 @@ def _in_memory_managed_files(): table.upsert = AsyncMock(side_effect=_upsert) prisma = MagicMock() prisma.db.litellm_managedobjecttable = table + prisma.db.litellm_teamtable.find_unique = AsyncMock(return_value=None) cache = MagicMock() cache.async_set_cache = AsyncMock() @@ -411,6 +412,32 @@ async def test_store_unified_object_id_persists_key_and_tags_on_create(): assert row["request_tags"].data == ["env:prod"] +@pytest.mark.asyncio +async def test_store_unified_object_id_resolves_org_through_the_team(): + """Most keys belong to an org only through their team, so the auth object carries no + org_id. The create resolves the team's organization so org spend is snapshotted at + submission time instead of never being billed.""" + from types import SimpleNamespace + + instance, store = _in_memory_managed_files() + instance.prisma_client.db.litellm_teamtable.find_unique = AsyncMock( + return_value=SimpleNamespace(organization_id="org-via-team") + ) + creator = UserAPIKeyAuth(user_id="alice", team_id="team-alpha", api_key="hash-alice") + + await instance.store_unified_object_id( + unified_object_id="unified-b", + file_object=_build_batch_response(batch_id="b", status="validating"), + litellm_parent_otel_span=None, + model_object_id="b", + file_purpose="batch", + user_api_key_dict=creator, + persist_attribution=True, + ) + + assert store["unified-b"]["org_id"] == "org-via-team" + + @pytest.mark.asyncio async def test_store_unified_object_id_omits_key_and_tags_without_persist_attribution(): """Regression (spend redirect): a caller that is not the batch create (a poll, or the From 35371a34c1c873a0afafe9dffc1873fcee42928e Mon Sep 17 00:00:00 2001 From: mubashir1osmani Date: Thu, 3 Sep 2026 19:09:45 -0400 Subject: [PATCH 09/34] fix(batches): keep team org attribution when the key lookup fails --- .../proxy/common_utils/check_batch_cost.py | 24 ++++++++++++------- .../proxy_unit_tests/test_check_batch_cost.py | 17 +++++++++++++ 2 files changed, 32 insertions(+), 9 deletions(-) diff --git a/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py b/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py index 595a8d04bed..bd7f3c93ff3 100644 --- a/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py +++ b/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py @@ -151,8 +151,8 @@ class CheckBatchCost: return org_id api_key = getattr(job, "api_key", None) team_id = getattr(job, "team_id", None) - try: - if api_key: + if api_key: + try: key_row: prisma_models.LiteLLM_VerificationToken | None = ( await self.prisma_client.db.litellm_verificationtoken.find_unique( where={"token": api_key} @@ -161,16 +161,22 @@ class CheckBatchCost: key_org_id = getattr(key_row, "org_id", None) if key_row is not None else None if key_org_id: return key_org_id - if team_id: - team_row: prisma_models.LiteLLM_TeamTable | None = ( - await self.prisma_client.db.litellm_teamtable.find_unique( - where={"team_id": team_id} - ) + except Exception as e: + verbose_proxy_logger.error( + f"CheckBatchCost: could not resolve the key's org for batch {batch_id}, " + f"still trying the team's: {e}" ) - return getattr(team_row, "organization_id", None) if team_row is not None else None + if not team_id: return None + try: + team_row: prisma_models.LiteLLM_TeamTable | None = ( + await self.prisma_client.db.litellm_teamtable.find_unique( + where={"team_id": team_id} + ) + ) + return getattr(team_row, "organization_id", None) if team_row is not None else None except Exception as e: - verbose_proxy_logger.error(f"CheckBatchCost: could not resolve org for batch {batch_id}: {e}") + verbose_proxy_logger.error(f"CheckBatchCost: could not resolve the team's org for batch {batch_id}: {e}") return None async def _build_creator_attribution_metadata( diff --git a/tests/proxy_unit_tests/test_check_batch_cost.py b/tests/proxy_unit_tests/test_check_batch_cost.py index 92b946e19b1..59dfb67c486 100644 --- a/tests/proxy_unit_tests/test_check_batch_cost.py +++ b/tests/proxy_unit_tests/test_check_batch_cost.py @@ -2600,6 +2600,23 @@ class TestBatchCostAttribution: assert metadata["user_api_key_org_id"] == "org-team" + @pytest.mark.asyncio + async def test_key_lookup_failure_still_bills_the_team_org(self): + """A key-table error while resolving a legacy row's org must not drop the team's + organization: the two lookups fail independently, so org spend still lands.""" + from types import SimpleNamespace + + instance = self._instance( + team_row=SimpleNamespace(team_alias="Team Alpha", organization_id="org-team"), + ) + instance.prisma_client.db.litellm_verificationtoken.find_unique = AsyncMock( + side_effect=Exception("db down") + ) + + metadata = await instance._build_creator_attribution_metadata(self._job(), "batch-1") + + assert metadata["user_api_key_org_id"] == "org-team" + @pytest.mark.asyncio async def test_no_org_leaves_the_key_unset(self): """Without any org the key is absent entirely, so the spend writer's org update From 8debf8294cbe5e8d66d470b8dda1254668aa65df Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 4 Sep 2026 12:42:39 -0700 Subject: [PATCH 10/34] fix(batches): resolve a legacy row's org from the key's organization_id and keep the Batch label on grouped rows --- .../proxy/common_utils/check_batch_cost.py | 2 +- tests/proxy_unit_tests/test_check_batch_cost.py | 8 ++++---- .../view_logs/RequestLogsTableColumns.test.tsx | 13 +++++++++++++ .../view_logs/RequestLogsTableColumns.tsx | 2 +- .../src/components/view_logs/TypeBadges.test.tsx | 7 +------ .../src/components/view_logs/TypeBadges.tsx | 4 ++-- 6 files changed, 22 insertions(+), 14 deletions(-) diff --git a/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py b/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py index ce80ba6493f..6b3ccfd3694 100644 --- a/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py +++ b/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py @@ -158,7 +158,7 @@ class CheckBatchCost: where={"token": api_key} ) ) - key_org_id = getattr(key_row, "org_id", None) if key_row is not None else None + key_org_id = getattr(key_row, "organization_id", None) if key_row is not None else None if key_org_id: return key_org_id except Exception as e: diff --git a/tests/proxy_unit_tests/test_check_batch_cost.py b/tests/proxy_unit_tests/test_check_batch_cost.py index 9ce91a8aee4..f9fb782b052 100644 --- a/tests/proxy_unit_tests/test_check_batch_cost.py +++ b/tests/proxy_unit_tests/test_check_batch_cost.py @@ -2561,7 +2561,7 @@ class TestBatchCostAttribution: from types import SimpleNamespace instance = self._instance( - key_row=SimpleNamespace(key_alias="prod-key", org_id="org-moved-to"), + key_row=SimpleNamespace(key_alias="prod-key", organization_id="org-moved-to"), team_row=SimpleNamespace(team_alias="Team Alpha", organization_id="org-team"), ) @@ -2578,7 +2578,7 @@ class TestBatchCostAttribution: from types import SimpleNamespace instance = self._instance( - key_row=SimpleNamespace(key_alias="prod-key", org_id="org-42"), + key_row=SimpleNamespace(key_alias="prod-key", organization_id="org-42"), team_row=SimpleNamespace(team_alias="Team Alpha", organization_id="org-team"), ) @@ -2593,7 +2593,7 @@ class TestBatchCostAttribution: from types import SimpleNamespace instance = self._instance( - key_row=SimpleNamespace(key_alias="prod-key", org_id=None), + key_row=SimpleNamespace(key_alias="prod-key", organization_id=None), team_row=SimpleNamespace(team_alias="Team Alpha", organization_id="org-team"), ) @@ -2625,7 +2625,7 @@ class TestBatchCostAttribution: from types import SimpleNamespace instance = self._instance( - key_row=SimpleNamespace(key_alias="prod-key", org_id=None), + key_row=SimpleNamespace(key_alias="prod-key", organization_id=None), team_row=SimpleNamespace(team_alias="Team Alpha", organization_id=None), ) diff --git a/ui/litellm-dashboard/src/components/view_logs/RequestLogsTableColumns.test.tsx b/ui/litellm-dashboard/src/components/view_logs/RequestLogsTableColumns.test.tsx index 3e8e522afe1..9f0e659cb1f 100644 --- a/ui/litellm-dashboard/src/components/view_logs/RequestLogsTableColumns.test.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/RequestLogsTableColumns.test.tsx @@ -141,6 +141,19 @@ describe("Type column", () => { expect(screen.getByText("Batch")).toBeInTheDocument(); expect(screen.queryByText("LLM")).not.toBeInTheDocument(); }); + + it("keeps the Batch label on the grouped create-plus-cost session instead of a row count", () => { + const groupedCostRow: Partial = { + request_id: "batch_1_batch_cost", + call_type: "aretrieve_batch", + session_id: "batch_1", + session_total_count: 2, + }; + renderRows([logEntry(groupedCostRow)]); + + expect(screen.getByText("Batch")).toBeInTheDocument(); + expect(screen.queryByText("2")).not.toBeInTheDocument(); + }); }); describe("batch rows", () => { diff --git a/ui/litellm-dashboard/src/components/view_logs/RequestLogsTableColumns.tsx b/ui/litellm-dashboard/src/components/view_logs/RequestLogsTableColumns.tsx index a444acb9517..1ec1087a1a4 100644 --- a/ui/litellm-dashboard/src/components/view_logs/RequestLogsTableColumns.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/RequestLogsTableColumns.tsx @@ -65,7 +65,7 @@ export const getRequestLogsTableColumns = ({ const sessionMcpCount = log.mcp_tool_call_count ?? (isMcp ? sessionCount : 0); if (isBatchCallType(log.call_type)) { - return 1 ? sessionCount : undefined} />; + return ; } if (sessionCount <= 1) { if (isMcp) return ; diff --git a/ui/litellm-dashboard/src/components/view_logs/TypeBadges.test.tsx b/ui/litellm-dashboard/src/components/view_logs/TypeBadges.test.tsx index 8a964ea2124..9a3b53685ca 100644 --- a/ui/litellm-dashboard/src/components/view_logs/TypeBadges.test.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/TypeBadges.test.tsx @@ -45,14 +45,9 @@ describe("TypeBadges", () => { }); describe("BatchBadge", () => { - it("should render with default 'Batch' text when no count is provided", () => { + it("should render 'Batch'", () => { render(); expect(screen.getByText("Batch")).toBeInTheDocument(); }); - - it("should render the count when provided", () => { - render(); - expect(screen.getByText("4")).toBeInTheDocument(); - }); }); }); diff --git a/ui/litellm-dashboard/src/components/view_logs/TypeBadges.tsx b/ui/litellm-dashboard/src/components/view_logs/TypeBadges.tsx index 84079848487..db64bfbbe73 100644 --- a/ui/litellm-dashboard/src/components/view_logs/TypeBadges.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/TypeBadges.tsx @@ -97,9 +97,9 @@ export const AgentBadge = ({ count }: { count?: number }) => ( ); -export const BatchBadge = ({ count }: { count?: number }) => ( +export const BatchBadge = () => ( - {count != null ? count : "Batch"} + Batch ); From 217cb7da653e025f7c6d10eaae8e291b826fd76f Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 4 Sep 2026 20:30:33 -0700 Subject: [PATCH 11/34] fix(managed_files): resolve the creator org through the cached team lookup Batch creation snapshotted the team's organization with a direct litellm_teamtable query on every create. Go through get_team_object instead, which serves the team auth already cached and only falls back to the database when the team was never cached. --- .../proxy/hooks/managed_files.py | 13 +++- ..._batch_update_db_managed_output_file_id.py | 68 ++++++++++++++----- 2 files changed, 61 insertions(+), 20 deletions(-) diff --git a/enterprise/litellm_enterprise/proxy/hooks/managed_files.py b/enterprise/litellm_enterprise/proxy/hooks/managed_files.py index 850e5aadf0e..68782c5516d 100644 --- a/enterprise/litellm_enterprise/proxy/hooks/managed_files.py +++ b/enterprise/litellm_enterprise/proxy/hooks/managed_files.py @@ -288,11 +288,18 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): return user_api_key_dict.org_id if not user_api_key_dict.team_id: return None + from litellm.proxy.auth.auth_checks import get_team_object + from litellm.proxy.proxy_server import proxy_logging_obj, user_api_key_cache + try: - team_row = await self.prisma_client.db.litellm_teamtable.find_unique( - where={"team_id": user_api_key_dict.team_id} + team: Final = await get_team_object( + team_id=user_api_key_dict.team_id, + prisma_client=self.prisma_client, + user_api_key_cache=user_api_key_cache, + parent_otel_span=user_api_key_dict.parent_otel_span, + proxy_logging_obj=proxy_logging_obj, ) - return getattr(team_row, "organization_id", None) if team_row is not None else None + return team.organization_id except Exception as e: verbose_logger.warning(f"could not resolve org for managed object attribution: {e}") return None diff --git a/tests/test_litellm/enterprise/proxy/test_batch_update_db_managed_output_file_id.py b/tests/test_litellm/enterprise/proxy/test_batch_update_db_managed_output_file_id.py index b5865ab4a13..d3e668b8987 100644 --- a/tests/test_litellm/enterprise/proxy/test_batch_update_db_managed_output_file_id.py +++ b/tests/test_litellm/enterprise/proxy/test_batch_update_db_managed_output_file_id.py @@ -413,29 +413,63 @@ async def test_store_unified_object_id_persists_key_and_tags_on_create(): @pytest.mark.asyncio -async def test_store_unified_object_id_resolves_org_through_the_team(): +async def test_store_unified_object_id_resolves_org_through_the_cached_team(): """Most keys belong to an org only through their team, so the auth object carries no - org_id. The create resolves the team's organization so org spend is snapshotted at - submission time instead of never being billed.""" - from types import SimpleNamespace + org_id. The create reads the team that auth already cached, so org spend is snapshotted + at submission time without a database query in the request path.""" + from litellm.models.team import LiteLLM_TeamTableCachedObj + from litellm.proxy.proxy_server import user_api_key_cache + + instance, store = _in_memory_managed_files() + creator = UserAPIKeyAuth(user_id="alice", team_id="team-cached", api_key="hash-alice") + await user_api_key_cache.async_set_cache( + key="team_id:team-cached", + value=LiteLLM_TeamTableCachedObj(team_id="team-cached", organization_id="org-via-team"), + model_type=LiteLLM_TeamTableCachedObj, + ) + try: + await instance.store_unified_object_id( + unified_object_id="unified-b", + file_object=_build_batch_response(batch_id="b", status="validating"), + litellm_parent_otel_span=None, + model_object_id="b", + file_purpose="batch", + user_api_key_dict=creator, + persist_attribution=True, + ) + finally: + user_api_key_cache.delete_cache(key="team_id:team-cached") + + assert store["unified-b"]["org_id"] == "org-via-team" + instance.prisma_client.db.litellm_teamtable.find_unique.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_store_unified_object_id_resolves_org_from_the_db_when_the_team_is_not_cached(): + """A team no request has run under yet is absent from the auth cache; its organization + still comes back from the table so the org is billed rather than dropped.""" + from litellm.models.team import LiteLLM_TeamTable + from litellm.proxy.proxy_server import user_api_key_cache instance, store = _in_memory_managed_files() instance.prisma_client.db.litellm_teamtable.find_unique = AsyncMock( - return_value=SimpleNamespace(organization_id="org-via-team") + return_value=LiteLLM_TeamTable(team_id="team-uncached", organization_id="org-via-db") ) - creator = UserAPIKeyAuth(user_id="alice", team_id="team-alpha", api_key="hash-alice") + creator = UserAPIKeyAuth(user_id="alice", team_id="team-uncached", api_key="hash-alice") + try: + await instance.store_unified_object_id( + unified_object_id="unified-b", + file_object=_build_batch_response(batch_id="b", status="validating"), + litellm_parent_otel_span=None, + model_object_id="b", + file_purpose="batch", + user_api_key_dict=creator, + persist_attribution=True, + ) + finally: + user_api_key_cache.delete_cache(key="team_id:team-uncached") - await instance.store_unified_object_id( - unified_object_id="unified-b", - file_object=_build_batch_response(batch_id="b", status="validating"), - litellm_parent_otel_span=None, - model_object_id="b", - file_purpose="batch", - user_api_key_dict=creator, - persist_attribution=True, - ) - - assert store["unified-b"]["org_id"] == "org-via-team" + assert store["unified-b"]["org_id"] == "org-via-db" @pytest.mark.asyncio From 814c151b02b41e4fb333f4f505203d7db7c4343a Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 5 Sep 2026 01:35:09 -0700 Subject: [PATCH 12/34] fix(batches): mask api base credentials on batch cost rows --- .../proxy/common_utils/check_batch_cost.py | 4 +- litellm/litellm_core_utils/litellm_logging.py | 16 ++-- .../proxy_unit_tests/test_check_batch_cost.py | 84 +++++++++++++++++++ 3 files changed, 94 insertions(+), 10 deletions(-) diff --git a/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py b/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py index 6b3ccfd3694..64a86387c39 100644 --- a/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py +++ b/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py @@ -685,7 +685,7 @@ class CheckBatchCost: from litellm.files.main import afile_content from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging - from litellm.litellm_core_utils.litellm_logging import deployment_pricing_model_info + from litellm.litellm_core_utils.litellm_logging import deployment_pricing_model_info, mask_api_base_credentials from litellm.proxy.openai_files_endpoints.common_utils import ( _is_base64_encoded_unified_file_id, ) @@ -858,7 +858,7 @@ class CheckBatchCost: "user-agent": CHECK_BATCH_COST_USER_AGENT, } }, - **({"api_base": deployment_api_base} if deployment_api_base else {}), + **({"api_base": mask_api_base_credentials(deployment_api_base)} if deployment_api_base else {}), "metadata": { **(await self._build_creator_attribution_metadata(job, batch_id)), # spend logs read the deployment identity off these metadata keys, so diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 73c06b891fb..7e800139228 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -419,6 +419,13 @@ def _provider_response_id(source: object) -> str | None: return candidate if isinstance(candidate, str) and candidate else None +def mask_api_base_credentials(api_base: str) -> str: + if "key=" not in api_base: + return api_base + key_end: Final = api_base.find("key=") + 4 + return api_base[:key_end] + "*" * 5 + api_base[-4:] + + class Logging(LiteLLMLoggingBaseClass): global \ supabaseClient, \ @@ -1160,14 +1167,7 @@ class Logging(LiteLLMLoggingBaseClass): return data def _get_masked_api_base(self, api_base: str) -> str: - if "key=" in api_base: - # Find the position of "key=" in the string - key_index: Final = api_base.find("key=") + 4 - # Mask the last 5 characters after "key=" - masked_api_base = api_base[:key_index] + "*" * 5 + api_base[-4:] - else: - masked_api_base = api_base - return str(masked_api_base) + return str(mask_api_base_credentials(api_base)) def _pre_call(self, input, api_key, model=None, additional_args={}): """ diff --git a/tests/proxy_unit_tests/test_check_batch_cost.py b/tests/proxy_unit_tests/test_check_batch_cost.py index f9fb782b052..36177b44930 100644 --- a/tests/proxy_unit_tests/test_check_batch_cost.py +++ b/tests/proxy_unit_tests/test_check_batch_cost.py @@ -583,6 +583,90 @@ class TestCheckBatchCost: assert passed_model_info["input_cost_per_token_batches"] == 2e-06 assert passed_model_info["output_cost_per_token_batches"] == 4e-06 + @pytest.mark.asyncio + async def test_poller_masks_api_base_credentials_before_logging( + self, check_batch_cost_instance, mock_prisma_client, mock_llm_router + ): + """Request rows mask `key=` query credentials out of api_base before it is + logged, but the poller skips that pre-call step, so an unmasked deployment + api_base would land verbatim on the batch cost row: regression test for the + poller masking the same way. + """ + import base64 + from unittest.mock import patch + + import httpx + import respx + + from litellm.litellm_core_utils.litellm_logging import Logging + + mock_prisma_client.db.litellm_managedobjecttable.update_many = AsyncMock(return_value=1) + mock_prisma_client.db.litellm_managedobjecttable.update = AsyncMock() + mock_prisma_client.db.litellm_usertable.find_unique = AsyncMock(return_value=None) + + mock_job = MagicMock() + mock_job.id = "job-masked-api-base-1" + mock_job.unified_object_id = base64.urlsafe_b64encode( + b"litellm_proxy;model_id:model-123;llm_batch_id:batch-456" + ).decode() + mock_job.created_by = "user-1" + mock_prisma_client.db.litellm_managedobjecttable.find_many = AsyncMock(return_value=[mock_job]) + + mock_response = MagicMock() + mock_response.status = "completed" + mock_response.output_file_id = "file-output-123" + mock_response.error_file_id = None + mock_response.model_dump_json.return_value = '{"id":"batch-1","status":"completed"}' + mock_llm_router.aretrieve_batch = AsyncMock(return_value=mock_response) + mock_llm_router.get_deployment_credentials_with_provider = MagicMock(return_value={"api_key": "sk-test"}) + + mock_deployment = MagicMock() + mock_deployment.litellm_params.custom_llm_provider = "openai" + mock_deployment.litellm_params.model = "gpt-5.4-mini" + mock_deployment.litellm_params.api_base = "https://gateway.example.com/v1?key=AIzaSyVERYSECRET7890" + mock_deployment.model_info.model_dump.return_value = {} + mock_llm_router.get_deployment = MagicMock(return_value=mock_deployment) + + output_line = json.dumps( + { + "custom_id": "req-1", + "response": { + "status_code": 200, + "body": { + "id": "chatcmpl-1", + "object": "chat.completion", + "model": "gpt-5.4-mini", + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "hi"}, + "finish_reason": "stop", + } + ], + "usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15}, + }, + }, + "error": None, + } + ) + + with ( + respx.mock(assert_all_called=True) as provider, + patch.object( # test-quality-ok: the poller builds Logging inline, the only seam to the row it logs + Logging, "async_success_handler", autospec=True + ) as success_handler, + ): + provider.get("https://api.openai.com/v1/files/file-output-123/content").mock( + return_value=httpx.Response(200, content=f"{output_line}\n".encode()) + ) + await check_batch_cost_instance.check_batch_cost() + + cost_row_calls = [call for call in success_handler.await_args_list if "batch_cost" in call.kwargs] + assert len(cost_row_calls) == 1 + logged_api_base = cost_row_calls[0].args[0].litellm_params["api_base"] + assert logged_api_base == "https://gateway.example.com/v1?key=*****7890" + assert "VERYSECRET" not in logged_api_base + @pytest.mark.asyncio async def test_primary_path_completion_update_includes_batch_processed( self, check_batch_cost_instance, mock_prisma_client, mock_llm_router From 95d721ffe60e6fa8409e2ff3e57ab9a85093fcc7 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 5 Sep 2026 02:06:40 -0700 Subject: [PATCH 13/34] refactor(batches): drop docstrings restating the org fallbacks --- .../litellm_enterprise/proxy/common_utils/check_batch_cost.py | 3 --- enterprise/litellm_enterprise/proxy/hooks/managed_files.py | 4 ---- 2 files changed, 7 deletions(-) diff --git a/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py b/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py index 64a86387c39..13e9e5093a8 100644 --- a/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py +++ b/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py @@ -143,9 +143,6 @@ class CheckBatchCost: return None async def _get_org_id(self, job: "LiteLLM_ManagedObjectTable", batch_id: str) -> str | None: - """Organization to bill the batch against, snapshotted on the row at creation - like team_id. Rows created before the org_id column existed carry None, so they - fall back to the creating key's org (or its team's) as resolved today.""" org_id = getattr(job, "org_id", None) if org_id: return org_id diff --git a/enterprise/litellm_enterprise/proxy/hooks/managed_files.py b/enterprise/litellm_enterprise/proxy/hooks/managed_files.py index 6474475f1bf..486904d0abe 100644 --- a/enterprise/litellm_enterprise/proxy/hooks/managed_files.py +++ b/enterprise/litellm_enterprise/proxy/hooks/managed_files.py @@ -281,10 +281,6 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): verbose_logger.debug(f"LiteLLM Managed File object with id={file_id} stored in db: {result}") async def _resolve_creator_org_id(self, user_api_key_dict: UserAPIKeyAuth) -> Optional[str]: - """Organization to snapshot on the managed object row, like team_id. A key that - belongs to an org only through its team carries no org_id on the auth object, so - resolve the team's organization at creation time; costing then bills the org the - batch was submitted under even if the key or team moves before it completes.""" if user_api_key_dict.org_id: return user_api_key_dict.org_id if not user_api_key_dict.team_id: From 4cc0180eab7482a59722e68999def58e11077a72 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 7 Sep 2026 17:06:34 -0700 Subject: [PATCH 14/34] fix(router): keep unresolved drop_params strings so DB rows and env refs survive The drop_params validator collapsed every string it did not recognize to None. A pre-fix DB row holds the flag as ciphertext, so a partial PATCH rebuilt the deployment without it and dropped the key from the stored row, and /model/new turned an os.environ/ reference into nothing before the loader could resolve it. The validator now returns the raw value when it is not a boolean flag, the field admits strings the way timeout already does, and the flag set follows pydantic's lax bool parsing instead of a hand-rolled true/false pair --- litellm/litellm_core_utils/core_helpers.py | 15 +++++----- litellm/types/router.py | 7 +++-- .../litellm_core_utils/test_core_helpers.py | 11 +++++-- .../test_model_management_endpoints.py | 27 ++++++++++++++++- .../proxy/proxy_server/test_proxy_config.py | 30 +++++++++++++++++++ tests/test_litellm/types/test_router.py | 23 ++++++++++++++ ui/litellm-dashboard/src/lib/http/schema.d.ts | 4 +-- 7 files changed, 101 insertions(+), 16 deletions(-) diff --git a/litellm/litellm_core_utils/core_helpers.py b/litellm/litellm_core_utils/core_helpers.py index 671b2ddce99..bd7a1b8f384 100644 --- a/litellm/litellm_core_utils/core_helpers.py +++ b/litellm/litellm_core_utils/core_helpers.py @@ -5,6 +5,7 @@ from collections.abc import Iterable, Mapping from typing import TYPE_CHECKING, Any, Final, Literal import httpx +from pydantic import TypeAdapter, ValidationError from litellm._logging import verbose_logger from litellm.types.llms.openai import AllMessageValues, OpenAIChatCompletionFinishReason @@ -37,16 +38,16 @@ def safe_divide_seconds(seconds: float, denominator: float, default: float | Non return float(seconds / denominator) +_DROP_PARAMS_BOOL: Final = TypeAdapter(bool) + + def normalize_drop_params(value: object) -> bool | None: if isinstance(value, bool): return value - if isinstance(value, str): - lowered: Final = value.strip().lower() - if lowered == "true": - return True - if lowered == "false": - return False - return None + try: + return _DROP_PARAMS_BOOL.validate_python(value.strip() if isinstance(value, str) else value) + except ValidationError: + return None def safe_divide( diff --git a/litellm/types/router.py b/litellm/types/router.py index e2a04f9da77..47aea6430c3 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -315,7 +315,7 @@ class GenericLiteLLMParams(CredentialLiteLLMParams, CustomPricingLiteLLMParams): timeout: float | str | httpx.Timeout | None = None # if str, pass in as os.environ/ stream_timeout: float | str | None = None # timeout when making stream=True calls, if str, pass in as os.environ/ max_retries: int | None = None - drop_params: bool | None = None + drop_params: bool | str | None = None organization: str | None = None # for openai orgs configurable_clientside_auth_params: CONFIGURABLE_CLIENTSIDE_AUTH_PARAMS = None litellm_credential_name: str | None = None @@ -408,8 +408,9 @@ class GenericLiteLLMParams(CredentialLiteLLMParams, CustomPricingLiteLLMParams): @field_validator("drop_params", mode="before") @classmethod - def coerce_drop_params(cls, value: object) -> bool | None: - return normalize_drop_params(value) + def coerce_drop_params(cls, value: object) -> object: + normalized: Final = normalize_drop_params(value) + return value if normalized is None else normalized def __contains__(self, key) -> bool: # Define custom behavior for the 'in' operator diff --git a/tests/test_litellm/litellm_core_utils/test_core_helpers.py b/tests/test_litellm/litellm_core_utils/test_core_helpers.py index ab43141af23..8797f7c3591 100644 --- a/tests/test_litellm/litellm_core_utils/test_core_helpers.py +++ b/tests/test_litellm/litellm_core_utils/test_core_helpers.py @@ -268,11 +268,16 @@ class TestRedactNestedMatchAndRegexKeys: (" TRUE ", True), ("false", False), ("False", False), + ("yes", True), + ("off", False), + ("1", True), + (1, True), + (0, False), (None, None), - ("yes", None), ("", None), - (1, None), - (0, None), + ("os.environ/DROP_PARAMS", None), + ("v2:gcm:not-a-flag", None), + (2, None), ], ) def test_normalize_drop_params(value, expected): diff --git a/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py index d90338f8480..c02f886fc31 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py @@ -30,7 +30,7 @@ from litellm.proxy.management_endpoints.model_management_endpoints import ( ) from litellm.proxy.utils import PrismaClient from litellm.router import Router -from litellm.types.router import Deployment, LiteLLM_Params, updateDeployment +from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo, updateDeployment, updateLiteLLMParams async def _passthrough_row(update_data): @@ -3070,6 +3070,31 @@ class TestUpdateDBModelBlocked: assert "blocked" not in result +class TestUpdateDBModelKeepsLegacyDropParams: + def test_partial_patch_keeps_encrypted_string_drop_params(self, monkeypatch): + from litellm.proxy.common_utils.encrypt_decrypt_utils import decrypt_value_helper + from litellm.proxy.management_endpoints.model_management_endpoints import update_db_model + + monkeypatch.setenv("LITELLM_SALT_KEY", "sk-1234") + legacy_row = Deployment( + model_name="gpt-5-nano", + litellm_params=LiteLLM_Params( + model="openai/gpt-5-nano", + api_key=encrypt_value_helper(value="sk-old"), + drop_params=encrypt_value_helper(value="true"), + ), + model_info=ModelInfo(id="legacy-row"), + ) + + result = update_db_model( + db_model=legacy_row, + updated_patch=updateDeployment(litellm_params=updateLiteLLMParams(api_key="sk-new")), + ) + + stored = json.loads(result["litellm_params"]) + assert decrypt_value_helper(value=stored["drop_params"], key="drop_params") == "true" + + def _build_db_model_with_pricing(): """Wildcard deployment with custom pricing in litellm_params; Deployment.__init__ mirrors SPECIAL_MODEL_INFO_PARAMS into model_info, so both blobs hold the rate.""" diff --git a/tests/test_litellm/proxy/proxy_server/test_proxy_config.py b/tests/test_litellm/proxy/proxy_server/test_proxy_config.py index 2babfe432f3..a4be9574f89 100644 --- a/tests/test_litellm/proxy/proxy_server/test_proxy_config.py +++ b/tests/test_litellm/proxy/proxy_server/test_proxy_config.py @@ -19,6 +19,7 @@ import pytest import litellm from litellm.proxy._types import CommonProxyErrors +from litellm.proxy.common_utils.encrypt_decrypt_utils import encrypt_value_helper from litellm.proxy.proxy_server import ( ProxyConfig, _is_remote_module_url, @@ -2401,6 +2402,35 @@ def test_ProxyConfig__add_deployment_resolves_env_refs_on_arbitrary_field(monkey assert deployment.litellm_params.some_future_field == "resolved-custom-value" +@pytest.mark.parametrize( + "stored_drop_params", + ["true", "os.environ/DROP_PARAMS_FLAG"], +) +def test_ProxyConfig__add_deployment_turns_stored_drop_params_string_into_bool(monkeypatch, stored_drop_params): + monkeypatch.setenv("LITELLM_SALT_KEY", "sk-1234") + monkeypatch.setenv("DROP_PARAMS_FLAG", "true") + fake_router = MagicMock() + fake_router.upsert_deployment = MagicMock(return_value=True) + monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", fake_router) + pc = ProxyConfig() + db_model = SimpleNamespace( + model_id="model-1", + model_name="gpt-5-nano", + model_info={"id": "model-1"}, + litellm_params={ + "model": encrypt_value_helper(value="openai/gpt-5-nano"), + "drop_params": encrypt_value_helper(value=stored_drop_params), + }, + blocked=False, + ) + + added = pc._add_deployment(db_models=[db_model]) + deployment = fake_router.upsert_deployment.call_args.kwargs["deployment"] + + assert added == 1 + assert deployment.litellm_params.drop_params is True + + # --------------------------------------------------------------------------- # ProxyConfig.decrypt_model_list_from_db # --------------------------------------------------------------------------- diff --git a/tests/test_litellm/types/test_router.py b/tests/test_litellm/types/test_router.py index accd3b32a0d..2f4a29473c3 100644 --- a/tests/test_litellm/types/test_router.py +++ b/tests/test_litellm/types/test_router.py @@ -1,8 +1,10 @@ import pytest +from pydantic import ValidationError from litellm.types.router import ( SPECIAL_MODEL_INFO_PARAMS, Deployment, + GenericLiteLLMParams, LiteLLM_Params, ModelInfo, ) @@ -89,3 +91,24 @@ def test_pricing_strings_are_coerced_to_float(): def test_invalid_pricing_is_rejected(): with pytest.raises(ValueError, match='validation error for ModelInfo'): ModelInfo(id="x", input_cost_per_token="free") + + +@pytest.mark.parametrize( + "value, expected", + [ + (True, True), + ("true", True), + (" False ", False), + ("yes", True), + (None, None), + ("os.environ/DROP_PARAMS", "os.environ/DROP_PARAMS"), + ("v2:gcm:ciphertext-from-a-pre-fix-row", "v2:gcm:ciphertext-from-a-pre-fix-row"), + ], +) +def test_drop_params_coerces_flags_and_keeps_unresolved_strings(value, expected): + assert GenericLiteLLMParams(drop_params=value).drop_params == expected + + +def test_drop_params_rejects_non_flag_non_string_values(): + with pytest.raises(ValidationError): + GenericLiteLLMParams(drop_params=2) diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index f7552bfb397..7b5edf6acfd 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -29397,7 +29397,7 @@ export interface components { /** Default Api Key Tpm Limit */ default_api_key_tpm_limit?: number | null; /** Drop Params */ - drop_params?: boolean | null; + drop_params?: boolean | string | null; /** Gcs Bucket Name */ gcs_bucket_name?: string | null; /** Google Maps Grounding Cost Per Query */ @@ -39570,7 +39570,7 @@ export interface components { /** Default Api Key Tpm Limit */ default_api_key_tpm_limit?: number | null; /** Drop Params */ - drop_params?: boolean | null; + drop_params?: boolean | string | null; /** Gcs Bucket Name */ gcs_bucket_name?: string | null; /** Google Maps Grounding Cost Per Query */ From 1067697c7b471200cd26ee93834f71b09d013415 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 7 Sep 2026 17:11:56 -0700 Subject: [PATCH 15/34] refactor(utils): gate the triton branch on bool(drop_params) like every other provider --- litellm/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/utils.py b/litellm/utils.py index 59db4162410..50282674868 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -4302,7 +4302,7 @@ def get_optional_params( non_default_params=non_default_params, optional_params=optional_params, model=model, - drop_params=drop_params if drop_params is not None else False, + drop_params=bool(drop_params), ) elif custom_llm_provider == "maritalk": From 2f397fa12812afba555dbdeae407597e95968123 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 7 Sep 2026 18:06:29 -0700 Subject: [PATCH 16/34] fix(drop_params): honor string flags in litellm_settings and responses, and fail open on non-flag values --- litellm/proxy/proxy_server.py | 3 ++ litellm/responses/main.py | 5 ++- litellm/types/router.py | 6 ++- .../proxy/proxy_server/test_proxy_config.py | 43 +++++++++++++++++++ .../test_responses_api_request_body.py | 4 +- tests/test_litellm/types/test_router.py | 7 ++- 6 files changed, 59 insertions(+), 9 deletions(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 0915b8dd1b9..76acd414976 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -279,6 +279,7 @@ from litellm.litellm_core_utils.audio_utils.utils import resolve_speech_media_ty from litellm.litellm_core_utils.core_helpers import ( _get_parent_otel_span_from_kwargs, get_litellm_metadata_from_kwargs, + normalize_drop_params, ) from litellm.litellm_core_utils.credential_accessor import CredentialAccessor from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj @@ -5508,6 +5509,8 @@ class ProxyConfig: parse_budget_reset_time(value) setattr(litellm, key, value) + elif key == "drop_params": + litellm.drop_params = bool(normalize_drop_params(value)) else: verbose_proxy_logger.debug( "%s setting litellm.%s=%s%s", diff --git a/litellm/responses/main.py b/litellm/responses/main.py index 5e74b7324b4..52ca6ebb07a 100644 --- a/litellm/responses/main.py +++ b/litellm/responses/main.py @@ -17,6 +17,7 @@ from litellm.completion_extras.litellm_responses_transformation.transformation i from litellm.constants import request_timeout from litellm.integrations.anthropic_cache_control_hook import CARRY_UNMATCHED_MESSAGE_POINTS from litellm.litellm_core_utils.asyncify import run_async_function +from litellm.litellm_core_utils.core_helpers import normalize_drop_params from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.litellm_core_utils.prompt_templates.common_utils import ( update_responses_input_with_model_file_ids, @@ -1253,7 +1254,7 @@ def responses( responses_api_provider_config=responses_api_provider_config, response_api_optional_params=response_api_optional_params, allowed_openai_params=allowed_openai_params, - drop_params=request_drop_params if isinstance(request_drop_params, bool) else None, + drop_params=normalize_drop_params(request_drop_params), ) litellm_logging_obj.update_from_kwargs( @@ -2081,7 +2082,7 @@ def compact_responses( responses_api_provider_config=responses_api_provider_config, response_api_optional_params=response_api_optional_params, allowed_openai_params=None, - drop_params=request_drop_params if isinstance(request_drop_params, bool) else None, + drop_params=normalize_drop_params(request_drop_params), ) # Pre Call logging diff --git a/litellm/types/router.py b/litellm/types/router.py index 47aea6430c3..e8aec027a5e 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -408,9 +408,11 @@ class GenericLiteLLMParams(CredentialLiteLLMParams, CustomPricingLiteLLMParams): @field_validator("drop_params", mode="before") @classmethod - def coerce_drop_params(cls, value: object) -> object: + def coerce_drop_params(cls, value: object) -> bool | str | None: normalized: Final = normalize_drop_params(value) - return value if normalized is None else normalized + if normalized is not None: + return normalized + return value if isinstance(value, str) else None def __contains__(self, key) -> bool: # Define custom behavior for the 'in' operator diff --git a/tests/test_litellm/proxy/proxy_server/test_proxy_config.py b/tests/test_litellm/proxy/proxy_server/test_proxy_config.py index a4be9574f89..c57cd387d13 100644 --- a/tests/test_litellm/proxy/proxy_server/test_proxy_config.py +++ b/tests/test_litellm/proxy/proxy_server/test_proxy_config.py @@ -2431,6 +2431,49 @@ def test_ProxyConfig__add_deployment_turns_stored_drop_params_string_into_bool(m assert deployment.litellm_params.drop_params is True +def test_ProxyConfig__add_deployment_keeps_loading_rows_after_a_non_flag_drop_params(monkeypatch): + monkeypatch.setenv("LITELLM_SALT_KEY", "sk-1234") + fake_router = MagicMock() + fake_router.upsert_deployment = MagicMock(return_value=True) + monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", fake_router) + pc = ProxyConfig() + + def db_model(model_id, drop_params): + return SimpleNamespace( + model_id=model_id, + model_name="gpt-5-nano", + model_info={"id": model_id}, + litellm_params={ + "model": encrypt_value_helper(value="openai/gpt-5-nano"), + "drop_params": encrypt_value_helper(value=drop_params), + }, + blocked=False, + ) + + added = pc._add_deployment(db_models=[db_model("bad-row", 2), db_model("good-after", "true")]) + deployments = [call.kwargs["deployment"] for call in fake_router.upsert_deployment.call_args_list] + + assert added == 2 + assert [d.litellm_params.drop_params for d in deployments] == [None, True] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("configured, expected", [("true", True), ("false", False)]) +async def test_ProxyConfig_load_config_turns_litellm_settings_drop_params_string_into_bool( + tmp_path, monkeypatch, configured, expected +): + f = tmp_path / "c.yaml" + f.write_text(f'model_list: []\nlitellm_settings:\n drop_params: "{configured}"\n') + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", None) + monkeypatch.setattr("litellm.proxy.proxy_server.store_model_in_db", False) + monkeypatch.delenv("LITELLM_CONFIG_BUCKET_NAME", raising=False) + monkeypatch.setattr(litellm, "drop_params", not expected) + + await ProxyConfig().load_config(router=None, config_file_path=str(f)) + + assert litellm.drop_params is expected + + # --------------------------------------------------------------------------- # ProxyConfig.decrypt_model_list_from_db # --------------------------------------------------------------------------- diff --git a/tests/test_litellm/responses/test_responses_api_request_body.py b/tests/test_litellm/responses/test_responses_api_request_body.py index 3e60906ec6d..5fced458208 100644 --- a/tests/test_litellm/responses/test_responses_api_request_body.py +++ b/tests/test_litellm/responses/test_responses_api_request_body.py @@ -246,8 +246,10 @@ async def test_aresponses_keeps_include_obfuscation_in_stream_options(): @pytest.mark.asyncio +@pytest.mark.parametrize("drop_params", [True, "true"]) async def test_aresponses_request_level_drop_params_drops_bedrock_mantle_service_tier( monkeypatch, + drop_params, ): """ Request-level drop_params=True (as the proxy injects for agentic CLIs) must @@ -271,7 +273,7 @@ async def test_aresponses_request_level_drop_params_drops_bedrock_mantle_service aws_region_name="us-east-1", input="hi", service_tier="priority", - drop_params=True, + drop_params=drop_params, ) mock_post.assert_called_once() diff --git a/tests/test_litellm/types/test_router.py b/tests/test_litellm/types/test_router.py index 2f4a29473c3..47fc08167e3 100644 --- a/tests/test_litellm/types/test_router.py +++ b/tests/test_litellm/types/test_router.py @@ -1,5 +1,4 @@ import pytest -from pydantic import ValidationError from litellm.types.router import ( SPECIAL_MODEL_INFO_PARAMS, @@ -109,6 +108,6 @@ def test_drop_params_coerces_flags_and_keeps_unresolved_strings(value, expected) assert GenericLiteLLMParams(drop_params=value).drop_params == expected -def test_drop_params_rejects_non_flag_non_string_values(): - with pytest.raises(ValidationError): - GenericLiteLLMParams(drop_params=2) +@pytest.mark.parametrize("value", [2, 2.5, [], {}]) +def test_drop_params_ignores_non_flag_non_string_values(value): + assert GenericLiteLLMParams(drop_params=value).drop_params is None From b7c2decb7db01b54463d94112f7943032d9309da Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 7 Sep 2026 19:00:47 -0700 Subject: [PATCH 17/34] fix(drop_params): honor string values in litellm_params and the LITELLM_DROP_PARAMS env var get_litellm_params normalizes drop_params once, so a client-body string and router_settings.default_litellm_params reach the anthropic, bedrock, and azure_ai gates as a bool. LITELLM_DROP_PARAMS=false now means off. A value that is neither a flag nor a string logs one warning and counts as unset, both in the deployment validator and in litellm_settings. --- litellm/__init__.py | 3 +- litellm/litellm_core_utils/core_helpers.py | 2 +- .../litellm_core_utils/get_litellm_params.py | 5 +-- litellm/proxy/proxy_server.py | 9 ++++- litellm/types/router.py | 7 +++- .../test_get_litellm_params.py | 8 +++++ .../proxy/proxy_server/test_proxy_config.py | 34 +++++++++++++++++++ .../test_litellm/test_drop_params_env_var.py | 17 ++++++++++ tests/test_litellm/types/test_router.py | 15 ++++++-- 9 files changed, 92 insertions(+), 8 deletions(-) create mode 100644 tests/test_litellm/test_drop_params_env_var.py diff --git a/litellm/__init__.py b/litellm/__init__.py index 62477dd6264..36f143376ff 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -47,6 +47,7 @@ from typing import ( ) from litellm.types.integrations.datadog import DatadogInitParams from litellm.types.integrations.newrelic import NewRelicInitParams +from litellm.litellm_core_utils.core_helpers import normalize_drop_params from litellm._logging import ( set_verbose, _turn_on_debug, @@ -238,7 +239,7 @@ token: Optional[str] = ( ) telemetry = True max_tokens: int = DEFAULT_MAX_TOKENS # OpenAI Defaults -drop_params = bool(os.getenv("LITELLM_DROP_PARAMS", False)) +drop_params = bool(normalize_drop_params(os.getenv("LITELLM_DROP_PARAMS"))) modify_params = bool(os.getenv("LITELLM_MODIFY_PARAMS", False)) use_chat_completions_url_for_anthropic_messages: bool = bool( os.getenv("LITELLM_USE_CHAT_COMPLETIONS_URL_FOR_ANTHROPIC_MESSAGES", False) diff --git a/litellm/litellm_core_utils/core_helpers.py b/litellm/litellm_core_utils/core_helpers.py index bd7a1b8f384..c1f1076c710 100644 --- a/litellm/litellm_core_utils/core_helpers.py +++ b/litellm/litellm_core_utils/core_helpers.py @@ -42,7 +42,7 @@ _DROP_PARAMS_BOOL: Final = TypeAdapter(bool) def normalize_drop_params(value: object) -> bool | None: - if isinstance(value, bool): + if value is None or isinstance(value, bool): return value try: return _DROP_PARAMS_BOOL.validate_python(value.strip() if isinstance(value, str) else value) diff --git a/litellm/litellm_core_utils/get_litellm_params.py b/litellm/litellm_core_utils/get_litellm_params.py index 1fd79db15a6..92b32d32dc0 100644 --- a/litellm/litellm_core_utils/get_litellm_params.py +++ b/litellm/litellm_core_utils/get_litellm_params.py @@ -2,6 +2,7 @@ from collections.abc import Mapping, MutableMapping from types import MappingProxyType from typing import Final +from litellm.litellm_core_utils.core_helpers import normalize_drop_params from litellm.llms.openai.data_residency import infer_openai_data_residency AWS_CREDENTIAL_KWARGS_KEYS: Final = frozenset( @@ -113,7 +114,7 @@ def get_litellm_params( custom_prompt_dict: dict | None = None, litellm_metadata: dict | None = None, disable_add_transform_inline_image_block: bool | None = None, - drop_params: bool | None = None, + drop_params: bool | str | None = None, prompt_id: str | None = None, prompt_variables: dict | None = None, async_call: bool | None = None, @@ -175,7 +176,7 @@ def get_litellm_params( "custom_prompt_dict": custom_prompt_dict, "litellm_metadata": litellm_metadata, "disable_add_transform_inline_image_block": disable_add_transform_inline_image_block, - "drop_params": drop_params, + "drop_params": normalize_drop_params(drop_params), "prompt_id": prompt_id, "prompt_variables": prompt_variables, "async_call": async_call, diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 76acd414976..8a30274cc9f 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -5510,7 +5510,7 @@ class ProxyConfig: parse_budget_reset_time(value) setattr(litellm, key, value) elif key == "drop_params": - litellm.drop_params = bool(normalize_drop_params(value)) + litellm.drop_params = _drop_params_from_litellm_settings(value) else: verbose_proxy_logger.debug( "%s setting litellm.%s=%s%s", @@ -16915,6 +16915,13 @@ def _redact_config_param_value_for_logging(param_name: str | None, param_value: return param_value +def _drop_params_from_litellm_settings(value: object) -> bool: + normalized: Final = normalize_drop_params(value) + if normalized is None and value is not None: + verbose_proxy_logger.warning("litellm_settings.drop_params=%r is not a flag value, treating it as off", value) + return bool(normalized) + + def _redact_general_setting_value(field_name: str, value: JsonValue, is_full_admin: bool) -> JsonValue: if is_full_admin: return value diff --git a/litellm/types/router.py b/litellm/types/router.py index e8aec027a5e..5c9eab30f3d 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -12,6 +12,7 @@ import httpx from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator from typing_extensions import Protocol, ReadOnly, Required, TypedDict, runtime_checkable +from litellm._logging import verbose_logger from litellm._uuid import uuid from litellm.litellm_core_utils.core_helpers import normalize_drop_params @@ -412,7 +413,11 @@ class GenericLiteLLMParams(CredentialLiteLLMParams, CustomPricingLiteLLMParams): normalized: Final = normalize_drop_params(value) if normalized is not None: return normalized - return value if isinstance(value, str) else None + if isinstance(value, str): + return value + if value is not None: + verbose_logger.warning("drop_params=%r is not a flag value, treating it as unset", value) + return None def __contains__(self, key) -> bool: # Define custom behavior for the 'in' operator diff --git a/tests/test_litellm/litellm_core_utils/test_get_litellm_params.py b/tests/test_litellm/litellm_core_utils/test_get_litellm_params.py index fb4cb494bee..f026ff57719 100644 --- a/tests/test_litellm/litellm_core_utils/test_get_litellm_params.py +++ b/tests/test_litellm/litellm_core_utils/test_get_litellm_params.py @@ -215,3 +215,11 @@ class TestMetadataFallsBackToLitellmMetadata: assert result["metadata"] is not litellm_metadata result["metadata"].pop("trace_id") assert litellm_metadata == {"trace_id": "trace-1"} + + +@pytest.mark.parametrize( + "value, expected", + [("true", True), ("false", False), (" TRUE ", True), (True, True), (None, None), ("os.environ/DROP_PARAMS", None)], +) +def test_drop_params_strings_reach_litellm_params_as_flags(value, expected): + assert get_litellm_params(drop_params=value)["drop_params"] is expected diff --git a/tests/test_litellm/proxy/proxy_server/test_proxy_config.py b/tests/test_litellm/proxy/proxy_server/test_proxy_config.py index c57cd387d13..774c63d754a 100644 --- a/tests/test_litellm/proxy/proxy_server/test_proxy_config.py +++ b/tests/test_litellm/proxy/proxy_server/test_proxy_config.py @@ -9,6 +9,7 @@ Pins covered: from __future__ import annotations import json +import logging import os import re from types import SimpleNamespace @@ -2474,6 +2475,39 @@ async def test_ProxyConfig_load_config_turns_litellm_settings_drop_params_string assert litellm.drop_params is expected +@pytest.mark.asyncio +async def test_ProxyConfig_load_config_resolves_a_litellm_settings_drop_params_env_ref(tmp_path, monkeypatch): + f = tmp_path / "c.yaml" + f.write_text("model_list: []\nlitellm_settings:\n drop_params: os.environ/DROP_PARAMS_FROM_ENV\n") + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", None) + monkeypatch.setattr("litellm.proxy.proxy_server.store_model_in_db", False) + monkeypatch.delenv("LITELLM_CONFIG_BUCKET_NAME", raising=False) + monkeypatch.setenv("DROP_PARAMS_FROM_ENV", "true") + monkeypatch.setattr(litellm, "drop_params", False) + + await ProxyConfig().load_config(router=None, config_file_path=str(f)) + + assert litellm.drop_params is True + + +@pytest.mark.asyncio +async def test_ProxyConfig_load_config_warns_and_turns_off_a_non_flag_litellm_settings_drop_params( + tmp_path, monkeypatch, caplog +): + f = tmp_path / "c.yaml" + f.write_text("model_list: []\nlitellm_settings:\n drop_params: ture\n") + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", None) + monkeypatch.setattr("litellm.proxy.proxy_server.store_model_in_db", False) + monkeypatch.delenv("LITELLM_CONFIG_BUCKET_NAME", raising=False) + monkeypatch.setattr(litellm, "drop_params", True) + + with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"): + await ProxyConfig().load_config(router=None, config_file_path=str(f)) + + assert litellm.drop_params is False + assert "litellm_settings.drop_params='ture' is not a flag value, treating it as off" in caplog.text + + # --------------------------------------------------------------------------- # ProxyConfig.decrypt_model_list_from_db # --------------------------------------------------------------------------- diff --git a/tests/test_litellm/test_drop_params_env_var.py b/tests/test_litellm/test_drop_params_env_var.py new file mode 100644 index 00000000000..a1ef3648f95 --- /dev/null +++ b/tests/test_litellm/test_drop_params_env_var.py @@ -0,0 +1,17 @@ +import os +import subprocess +import sys + +import pytest + + +@pytest.mark.parametrize("configured, expected", [("false", "False"), ("true", "True")]) +def test_litellm_drop_params_env_var_is_parsed_as_a_flag(configured, expected): + result = subprocess.run( + [sys.executable, "-c", "import litellm; print(litellm.drop_params)"], + env={**os.environ, "LITELLM_DROP_PARAMS": configured}, + capture_output=True, + text=True, + check=True, + ) + assert result.stdout.strip() == expected diff --git a/tests/test_litellm/types/test_router.py b/tests/test_litellm/types/test_router.py index 47fc08167e3..fd933a9d993 100644 --- a/tests/test_litellm/types/test_router.py +++ b/tests/test_litellm/types/test_router.py @@ -1,3 +1,5 @@ +import logging + import pytest from litellm.types.router import ( @@ -109,5 +111,14 @@ def test_drop_params_coerces_flags_and_keeps_unresolved_strings(value, expected) @pytest.mark.parametrize("value", [2, 2.5, [], {}]) -def test_drop_params_ignores_non_flag_non_string_values(value): - assert GenericLiteLLMParams(drop_params=value).drop_params is None +def test_drop_params_ignores_non_flag_non_string_values_with_a_warning(value, caplog): + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + assert GenericLiteLLMParams(drop_params=value).drop_params is None + assert f"drop_params={value!r} is not a flag value" in caplog.text + + +@pytest.mark.parametrize("value", [True, "true", None, "os.environ/DROP_PARAMS", "v2:gcm:ciphertext-from-a-pre-fix-row"]) +def test_drop_params_flags_and_strings_log_nothing(value, caplog): + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + GenericLiteLLMParams(drop_params=value) + assert caplog.text == "" From d594b9385eb8eae760c809e43ac9174bceadfd46 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 7 Sep 2026 21:05:01 -0700 Subject: [PATCH 18/34] fix(drop_params): warn when a deployment or env drop_params value is not a flag A deployment drop_params string that is not a flag value (a typo like ture) stayed silently off. The router now logs one warning per deployment. LITELLM_DROP_PARAMS and litellm_settings.drop_params share the same helper, so a non-flag value there warns as well instead of flipping silently from on to off --- litellm/__init__.py | 4 +-- litellm/litellm_core_utils/core_helpers.py | 8 +++++ litellm/proxy/proxy_server.py | 11 ++----- litellm/router.py | 6 ++++ .../litellm_core_utils/test_core_helpers.py | 17 ++++++++++ .../test_litellm/test_drop_params_env_var.py | 19 +++++++++-- tests/test_litellm/test_router.py | 33 +++++++++++++++++++ 7 files changed, 84 insertions(+), 14 deletions(-) diff --git a/litellm/__init__.py b/litellm/__init__.py index 36f143376ff..f7d4dce87d2 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -47,7 +47,7 @@ from typing import ( ) from litellm.types.integrations.datadog import DatadogInitParams from litellm.types.integrations.newrelic import NewRelicInitParams -from litellm.litellm_core_utils.core_helpers import normalize_drop_params +from litellm.litellm_core_utils.core_helpers import drop_params_flag from litellm._logging import ( set_verbose, _turn_on_debug, @@ -239,7 +239,7 @@ token: Optional[str] = ( ) telemetry = True max_tokens: int = DEFAULT_MAX_TOKENS # OpenAI Defaults -drop_params = bool(normalize_drop_params(os.getenv("LITELLM_DROP_PARAMS"))) +drop_params = drop_params_flag(os.getenv("LITELLM_DROP_PARAMS"), "LITELLM_DROP_PARAMS", verbose_logger) modify_params = bool(os.getenv("LITELLM_MODIFY_PARAMS", False)) use_chat_completions_url_for_anthropic_messages: bool = bool( os.getenv("LITELLM_USE_CHAT_COMPLETIONS_URL_FOR_ANTHROPIC_MESSAGES", False) diff --git a/litellm/litellm_core_utils/core_helpers.py b/litellm/litellm_core_utils/core_helpers.py index c1f1076c710..a3dfac81cc1 100644 --- a/litellm/litellm_core_utils/core_helpers.py +++ b/litellm/litellm_core_utils/core_helpers.py @@ -1,6 +1,7 @@ # What is this? ## Helper utilities import copy +import logging from collections.abc import Iterable, Mapping from typing import TYPE_CHECKING, Any, Final, Literal @@ -50,6 +51,13 @@ def normalize_drop_params(value: object) -> bool | None: return None +def drop_params_flag(value: object, source: str, logger: logging.Logger) -> bool: + normalized: Final = normalize_drop_params(value) + if normalized is None and value is not None: + logger.warning("%s=%r is not a flag value, treating it as off", source, value) + return bool(normalized) + + def safe_divide( numerator: float, denominator: float, diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 8a30274cc9f..83b99822b4c 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -278,8 +278,8 @@ from litellm.litellm_core_utils.asyncify import asyncify from litellm.litellm_core_utils.audio_utils.utils import resolve_speech_media_type from litellm.litellm_core_utils.core_helpers import ( _get_parent_otel_span_from_kwargs, + drop_params_flag, get_litellm_metadata_from_kwargs, - normalize_drop_params, ) from litellm.litellm_core_utils.credential_accessor import CredentialAccessor from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj @@ -5510,7 +5510,7 @@ class ProxyConfig: parse_budget_reset_time(value) setattr(litellm, key, value) elif key == "drop_params": - litellm.drop_params = _drop_params_from_litellm_settings(value) + litellm.drop_params = drop_params_flag(value, "litellm_settings.drop_params", verbose_proxy_logger) else: verbose_proxy_logger.debug( "%s setting litellm.%s=%s%s", @@ -16915,13 +16915,6 @@ def _redact_config_param_value_for_logging(param_name: str | None, param_value: return param_value -def _drop_params_from_litellm_settings(value: object) -> bool: - normalized: Final = normalize_drop_params(value) - if normalized is None and value is not None: - verbose_proxy_logger.warning("litellm_settings.drop_params=%r is not a flag value, treating it as off", value) - return bool(normalized) - - def _redact_general_setting_value(field_name: str, value: JsonValue, is_full_admin: bool) -> JsonValue: if is_full_admin: return value diff --git a/litellm/router.py b/litellm/router.py index 95cabfad4bd..4bbdba08e13 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -9366,6 +9366,12 @@ class Router: #### VALIDATE MODEL ######## # Check if this is a prompt management model before validating as LLM provider litellm_model: Final = deployment.litellm_params.model + if isinstance(deployment.litellm_params.drop_params, str): + verbose_router_logger.warning( + "model=%s drop_params=%r is not a flag value, treating it as unset", + deployment.model_name, + deployment.litellm_params.drop_params, + ) is_prompt_management_model = False if "/" in litellm_model: diff --git a/tests/test_litellm/litellm_core_utils/test_core_helpers.py b/tests/test_litellm/litellm_core_utils/test_core_helpers.py index 8797f7c3591..e3880175759 100644 --- a/tests/test_litellm/litellm_core_utils/test_core_helpers.py +++ b/tests/test_litellm/litellm_core_utils/test_core_helpers.py @@ -1,9 +1,12 @@ """Tests for litellm_core_utils.core_helpers module.""" +import logging + import pytest from litellm.litellm_core_utils.core_helpers import ( _FINISH_REASON_MAP, + drop_params_flag, get_or_create_metadata_bucket, map_finish_reason, normalize_drop_params, @@ -284,6 +287,20 @@ def test_normalize_drop_params(value, expected): assert normalize_drop_params(value) is expected +@pytest.mark.parametrize("value, expected", [("true", True), ("off", False), (None, False)]) +def test_drop_params_flag_returns_a_bool_without_a_warning(value, expected, caplog): + with caplog.at_level(logging.WARNING, logger="drop-params-test"): + assert drop_params_flag(value, "LITELLM_DROP_PARAMS", logging.getLogger("drop-params-test")) is expected + assert caplog.text == "" + + +@pytest.mark.parametrize("value", ["temperature", "ture", 2]) +def test_drop_params_flag_treats_non_flag_values_as_off_with_a_warning(value, caplog): + with caplog.at_level(logging.WARNING, logger="drop-params-test"): + assert drop_params_flag(value, "LITELLM_DROP_PARAMS", logging.getLogger("drop-params-test")) is False + assert f"LITELLM_DROP_PARAMS={value!r} is not a flag value, treating it as off" in caplog.text + + class TestIsExpectedClientError: def test_status_ranges(self): from litellm.litellm_core_utils.core_helpers import is_expected_client_error diff --git a/tests/test_litellm/test_drop_params_env_var.py b/tests/test_litellm/test_drop_params_env_var.py index a1ef3648f95..339298df3d1 100644 --- a/tests/test_litellm/test_drop_params_env_var.py +++ b/tests/test_litellm/test_drop_params_env_var.py @@ -5,13 +5,26 @@ import sys import pytest -@pytest.mark.parametrize("configured, expected", [("false", "False"), ("true", "True")]) -def test_litellm_drop_params_env_var_is_parsed_as_a_flag(configured, expected): - result = subprocess.run( +def _import_litellm_with(configured: str) -> subprocess.CompletedProcess[str]: + return subprocess.run( [sys.executable, "-c", "import litellm; print(litellm.drop_params)"], env={**os.environ, "LITELLM_DROP_PARAMS": configured}, capture_output=True, text=True, check=True, ) + + +@pytest.mark.parametrize("configured, expected", [("false", "False"), ("true", "True")]) +def test_litellm_drop_params_env_var_is_parsed_as_a_flag(configured, expected): + result = _import_litellm_with(configured) + assert result.stdout.strip() == expected + assert "is not a flag value" not in result.stderr + + +def test_litellm_drop_params_env_var_non_flag_value_is_off_with_a_warning(): + result = _import_litellm_with("temperature") + + assert result.stdout.strip() == "False" + assert "LITELLM_DROP_PARAMS='temperature' is not a flag value, treating it as off" in result.stderr diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index 201a4d84ebe..0c0bad8080d 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -14424,3 +14424,36 @@ async def test_router_deployment_drop_params_string_true_is_honored(monkeypatch) temperature=0.1, ) assert response.choices[0].message.content == "Hello, world!" + + +@pytest.mark.parametrize("value", ["ture", "enabled"]) +def test_router_warns_when_a_deployment_drop_params_string_is_not_a_flag(value, caplog): + with caplog.at_level(logging.WARNING, logger="LiteLLM Router"): + router = Router( + model_list=[ + { + "model_name": "gpt-5-nano", + "litellm_params": {"model": "openai/gpt-5-nano", "api_key": "sk-fake", "drop_params": value}, + } + ] + ) + + deployment = router.get_deployment_by_model_group_name(model_group_name="gpt-5-nano") + assert deployment is not None + assert deployment.litellm_params.drop_params == value + assert f"model=gpt-5-nano drop_params={value!r} is not a flag value, treating it as unset" in caplog.text + + +@pytest.mark.parametrize("value", [True, "true", "off", None]) +def test_router_stays_quiet_when_a_deployment_drop_params_is_a_flag(value, caplog): + with caplog.at_level(logging.WARNING, logger="LiteLLM Router"): + Router( + model_list=[ + { + "model_name": "gpt-5-nano", + "litellm_params": {"model": "openai/gpt-5-nano", "api_key": "sk-fake", "drop_params": value}, + } + ] + ) + + assert "is not a flag value" not in caplog.text From 704013dbb6f83962e4cac97c79ac0dd2109cbabf Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 7 Sep 2026 22:13:23 -0700 Subject: [PATCH 19/34] fix(init): keep non-flag LITELLM_DROP_PARAMS values on with a warning The merge base read the variable by truthiness, so any non-empty value turned the global flag on. Parsing it as a flag made a value such as temperature or enabled silently turn it off, and the only docs for the variable describe it as a list of parameter names, so keep those values on and log a warning that asks for true or false. A blank value stays off without a warning --- litellm/__init__.py | 4 +-- litellm/litellm_core_utils/core_helpers.py | 16 +++++++++++ .../litellm_core_utils/test_core_helpers.py | 28 +++++++++++++++++++ .../test_litellm/test_drop_params_env_var.py | 11 +++++--- 4 files changed, 53 insertions(+), 6 deletions(-) diff --git a/litellm/__init__.py b/litellm/__init__.py index f7d4dce87d2..fc6dc35fe55 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -47,7 +47,7 @@ from typing import ( ) from litellm.types.integrations.datadog import DatadogInitParams from litellm.types.integrations.newrelic import NewRelicInitParams -from litellm.litellm_core_utils.core_helpers import drop_params_flag +from litellm.litellm_core_utils.core_helpers import drop_params_env_flag from litellm._logging import ( set_verbose, _turn_on_debug, @@ -239,7 +239,7 @@ token: Optional[str] = ( ) telemetry = True max_tokens: int = DEFAULT_MAX_TOKENS # OpenAI Defaults -drop_params = drop_params_flag(os.getenv("LITELLM_DROP_PARAMS"), "LITELLM_DROP_PARAMS", verbose_logger) +drop_params = drop_params_env_flag(os.environ, verbose_logger) modify_params = bool(os.getenv("LITELLM_MODIFY_PARAMS", False)) use_chat_completions_url_for_anthropic_messages: bool = bool( os.getenv("LITELLM_USE_CHAT_COMPLETIONS_URL_FOR_ANTHROPIC_MESSAGES", False) diff --git a/litellm/litellm_core_utils/core_helpers.py b/litellm/litellm_core_utils/core_helpers.py index a3dfac81cc1..eacc3e4860a 100644 --- a/litellm/litellm_core_utils/core_helpers.py +++ b/litellm/litellm_core_utils/core_helpers.py @@ -58,6 +58,22 @@ def drop_params_flag(value: object, source: str, logger: logging.Logger) -> bool return bool(normalized) +DROP_PARAMS_ENV_VAR: Final = "LITELLM_DROP_PARAMS" + + +def drop_params_env_flag(environ: Mapping[str, str], logger: logging.Logger) -> bool: + configured: Final = environ.get(DROP_PARAMS_ENV_VAR, "").strip() + if configured == "": + return False + normalized: Final = normalize_drop_params(configured) + if normalized is None: + logger.warning( + "%s=%r is not a flag value, treating it as on. Set it to true or false", DROP_PARAMS_ENV_VAR, configured + ) + return True + return normalized + + def safe_divide( numerator: float, denominator: float, diff --git a/tests/test_litellm/litellm_core_utils/test_core_helpers.py b/tests/test_litellm/litellm_core_utils/test_core_helpers.py index e3880175759..e937be47441 100644 --- a/tests/test_litellm/litellm_core_utils/test_core_helpers.py +++ b/tests/test_litellm/litellm_core_utils/test_core_helpers.py @@ -6,6 +6,7 @@ import pytest from litellm.litellm_core_utils.core_helpers import ( _FINISH_REASON_MAP, + drop_params_env_flag, drop_params_flag, get_or_create_metadata_bucket, map_finish_reason, @@ -301,6 +302,33 @@ def test_drop_params_flag_treats_non_flag_values_as_off_with_a_warning(value, ca assert f"LITELLM_DROP_PARAMS={value!r} is not a flag value, treating it as off" in caplog.text +@pytest.mark.parametrize( + "environ, expected", + [ + ({}, False), + ({"LITELLM_DROP_PARAMS": ""}, False), + ({"LITELLM_DROP_PARAMS": " "}, False), + ({"LITELLM_DROP_PARAMS": "true"}, True), + ({"LITELLM_DROP_PARAMS": " False "}, False), + ({"LITELLM_DROP_PARAMS": "0"}, False), + ], +) +def test_drop_params_env_flag_reads_a_flag_without_a_warning(environ, expected, caplog): + with caplog.at_level(logging.WARNING, logger="drop-params-test"): + assert drop_params_env_flag(environ, logging.getLogger("drop-params-test")) is expected + assert caplog.text == "" + + +@pytest.mark.parametrize("configured", ["temperature", "temperature,top_p", "enabled"]) +def test_drop_params_env_flag_keeps_a_non_flag_value_on_with_a_warning(configured, caplog): + with caplog.at_level(logging.WARNING, logger="drop-params-test"): + assert drop_params_env_flag({"LITELLM_DROP_PARAMS": configured}, logging.getLogger("drop-params-test")) is True + assert ( + f"LITELLM_DROP_PARAMS={configured!r} is not a flag value, treating it as on. Set it to true or false" + in caplog.text + ) + + class TestIsExpectedClientError: def test_status_ranges(self): from litellm.litellm_core_utils.core_helpers import is_expected_client_error diff --git a/tests/test_litellm/test_drop_params_env_var.py b/tests/test_litellm/test_drop_params_env_var.py index 339298df3d1..1e0b7801ef1 100644 --- a/tests/test_litellm/test_drop_params_env_var.py +++ b/tests/test_litellm/test_drop_params_env_var.py @@ -15,7 +15,7 @@ def _import_litellm_with(configured: str) -> subprocess.CompletedProcess[str]: ) -@pytest.mark.parametrize("configured, expected", [("false", "False"), ("true", "True")]) +@pytest.mark.parametrize("configured, expected", [("false", "False"), ("true", "True"), ("", "False")]) def test_litellm_drop_params_env_var_is_parsed_as_a_flag(configured, expected): result = _import_litellm_with(configured) @@ -23,8 +23,11 @@ def test_litellm_drop_params_env_var_is_parsed_as_a_flag(configured, expected): assert "is not a flag value" not in result.stderr -def test_litellm_drop_params_env_var_non_flag_value_is_off_with_a_warning(): +def test_litellm_drop_params_env_var_non_flag_value_stays_on_with_a_warning(): result = _import_litellm_with("temperature") - assert result.stdout.strip() == "False" - assert "LITELLM_DROP_PARAMS='temperature' is not a flag value, treating it as off" in result.stderr + assert result.stdout.strip() == "True" + assert ( + "LITELLM_DROP_PARAMS='temperature' is not a flag value, treating it as on. Set it to true or false" + in result.stderr + ) From 3448175184d8b68dbcf9ec007d62f85651bb0f41 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 8 Sep 2026 11:44:49 -0700 Subject: [PATCH 20/34] fix(responses bridge): keep mid-conversation system messages in input Only the leading run of system messages (before the first non-system message) is joined into the Responses `instructions` field. A system message that arrives after a user, assistant, or tool turn now becomes a system input item at its position, whether its content is a string or a list, so a client that re-sends the same reminder as a string on the next request produces byte-identical input and `instructions` stays stable. Claude Code >= 2.1.237 appends such reminders after every user turn, and folding them into `instructions` made Azure treat every request as a cold prompt (cached_tokens 0 on every request of a session). Fixes #40198 --- .../transformation.py | 9 +- ...responses_transformation_transformation.py | 140 ++++++++++++++++++ 2 files changed, 147 insertions(+), 2 deletions(-) diff --git a/litellm/completion_extras/litellm_responses_transformation/transformation.py b/litellm/completion_extras/litellm_responses_transformation/transformation.py index 87350b5479c..4fe069b0b7d 100644 --- a/litellm/completion_extras/litellm_responses_transformation/transformation.py +++ b/litellm/completion_extras/litellm_responses_transformation/transformation.py @@ -370,7 +370,12 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): and isinstance(tool_call.get("custom"), dict) ) - for msg in messages: + leading_system_count: Final = next( + (index for index, msg in enumerate(messages) if msg.get("role") != "system"), + len(messages), + ) + + for index, msg in enumerate(messages): role = msg.get("role") content = msg.get("content", "") tool_calls = msg.get("tool_calls") @@ -378,7 +383,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): if role == "system": # Extract system message as instructions - if isinstance(content, str): + if isinstance(content, str) and index < leading_system_count: if instructions: # Concatenate multiple system prompts with a space instructions = f"{instructions} {content}" diff --git a/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py b/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py index d4d47b145d1..e4ada0a9b31 100644 --- a/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py +++ b/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py @@ -4147,3 +4147,143 @@ def test_streaming_final_chunk_carries_provider_metadata(): assert chunks[-1]["content_filters"] == content_filters assert "background" not in chunks[-1] assert all("service_tier" not in chunk for chunk in chunks[:-1]) + + +def _system_input_item(text: str) -> dict[str, object]: + return {"type": "message", "role": "system", "content": [{"type": "input_text", "text": text}]} + + +def test_mid_conversation_system_string_stays_in_input_after_a_user_turn(): + handler: Final = LiteLLMResponsesTransformationHandler() + + input_items, instructions = handler.convert_chat_completion_messages_to_responses_api( + [ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "Read the file."}, + {"role": "system", "content": "14982391 tokens left"}, + {"role": "user", "content": "Now summarize it."}, + ] + ) + + assert instructions == "You are a helpful assistant." + assert input_items == [ + {"type": "message", "role": "user", "content": [{"type": "input_text", "text": "Read the file."}]}, + _system_input_item("14982391 tokens left"), + {"type": "message", "role": "user", "content": [{"type": "input_text", "text": "Now summarize it."}]}, + ] + + +def test_leading_system_strings_still_join_instructions_without_a_following_turn(): + handler: Final = LiteLLMResponsesTransformationHandler() + + input_items, instructions = handler.convert_chat_completion_messages_to_responses_api( + [ + {"role": "system", "content": "Be brief."}, + {"role": "system", "content": "Answer in French."}, + ] + ) + + assert instructions == "Be brief. Answer in French." + assert input_items == [] + + +def test_mid_conversation_system_reminder_as_string_and_as_text_block_produce_identical_input_items(): + handler: Final = LiteLLMResponsesTransformationHandler() + reminder: Final = "14982391 tokens left" + + as_string, string_instructions = handler.convert_chat_completion_messages_to_responses_api( + [{"role": "user", "content": "Read the file."}, {"role": "system", "content": reminder}] + ) + as_block, block_instructions = handler.convert_chat_completion_messages_to_responses_api( + [ + {"role": "user", "content": "Read the file."}, + { + "role": "system", + "content": [{"type": "text", "text": reminder, "cache_control": {"type": "ephemeral"}}], + }, + ] + ) + + assert string_instructions is None + assert block_instructions is None + assert json.dumps(as_string) == json.dumps(as_block) + assert as_string[1] == _system_input_item(reminder) + + +def test_claude_code_shaped_history_keeps_a_byte_stable_input_prefix_across_requests(): + handler: Final = LiteLLMResponsesTransformationHandler() + top_level_system: Final = [{"type": "text", "text": "You are Claude Code.", "cache_control": {"type": "ephemeral"}}] + first_reminder: Final = "27k chars of deferred tools" + second_reminder: Final = "14982391 tokens left" + first_request_messages: Final = [ + {"role": "system", "content": top_level_system}, + {"role": "user", "content": "Read inventory.py."}, + { + "role": "system", + "content": [{"type": "text", "text": first_reminder, "cache_control": {"type": "ephemeral"}}], + }, + ] + second_request_messages: Final = [ + {"role": "system", "content": top_level_system}, + {"role": "user", "content": "Read inventory.py."}, + {"role": "system", "content": first_reminder}, + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_1", + "type": "function", + "function": {"name": "Read", "arguments": '{"file_path": "inventory.py"}'}, + } + ], + }, + {"role": "tool", "tool_call_id": "call_1", "content": "ITEMS = []"}, + { + "role": "system", + "content": [{"type": "text", "text": second_reminder, "cache_control": {"type": "ephemeral"}}], + }, + ] + + first_request: Final = handler.transform_request( + model="gpt-5.6-luna", + messages=first_request_messages, + optional_params={}, + litellm_params={}, + headers={}, + litellm_logging_obj=Mock(), + ) + second_request: Final = handler.transform_request( + model="gpt-5.6-luna", + messages=second_request_messages, + optional_params={}, + litellm_params={}, + headers={}, + litellm_logging_obj=Mock(), + ) + + assert "instructions" not in first_request + assert "instructions" not in second_request + assert first_request["input"][0] == _system_input_item("You are Claude Code.") + assert json.dumps(second_request["input"][: len(first_request["input"])]) == json.dumps(first_request["input"]) + assert second_request["input"][len(first_request["input"]) :] == [ + {"type": "function_call", "call_id": "call_1", "name": "Read", "arguments": '{"file_path": "inventory.py"}'}, + {"type": "function_call_output", "call_id": "call_1", "output": [{"type": "input_text", "text": "ITEMS = []"}]}, + _system_input_item(second_reminder), + ] + + +def test_system_string_after_a_developer_message_stays_in_input_in_client_order(): + handler: Final = LiteLLMResponsesTransformationHandler() + + input_items, instructions = handler.convert_chat_completion_messages_to_responses_api( + [ + {"role": "developer", "content": "Always answer in French."}, + {"role": "system", "content": "Be brief."}, + {"role": "user", "content": "Bonjour"}, + ] + ) + + assert instructions is None + assert [item["role"] for item in input_items] == ["developer", "system", "user"] + assert input_items[1] == _system_input_item("Be brief.") From d36e032241490a5a75063425a99da4ccdc85548a Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 8 Sep 2026 13:00:14 -0700 Subject: [PATCH 21/34] fix(proxy): initialize string success/failure callbacks at startup after config load (#38226) * fix(proxy): eagerly initialize string callbacks Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): materialize string callbacks after load_config so later litellm_settings keys are applied Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): drop casts when snapshotting string callbacks so LIT006 stays at base Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: yucheng --- litellm/proxy/utils.py | 11 +++++ tests/test_litellm/proxy/test_proxy_server.py | 41 +++++++++++++++++++ 2 files changed, 52 insertions(+) diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index ddf31cb1d8a..75d7972c621 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -177,6 +177,9 @@ from litellm.types.mcp import ( ) from litellm.types.proxy.policy_engine.pipeline_types import PipelineExecutionResult from litellm.types.utils import LLMResponseTypes, LoggedLiteLLMParams +from litellm.utils import ( + _add_custom_logger_callback_to_specific_event, # pyright: ignore[reportPrivateUsage] # only string-to-logger helper +) if TYPE_CHECKING: from mcp.types import CallToolResult @@ -857,6 +860,14 @@ class ProxyLogging: litellm.logging_callback_manager.add_litellm_async_success_callback(callback) litellm.logging_callback_manager.add_litellm_async_failure_callback(callback) + # Runs after load_config applied every litellm_settings key: logger __init__s read e.g. s3_callback_params + success_callbacks: Final = tuple(cb for cb in litellm.success_callback if isinstance(cb, str)) + failure_callbacks: Final = tuple(cb for cb in litellm.failure_callback if isinstance(cb, str)) + for callback in success_callbacks: + _add_custom_logger_callback_to_specific_event(callback, "success") + for callback in failure_callbacks: + _add_custom_logger_callback_to_specific_event(callback, "failure") + async def update_request_status(self, litellm_call_id: str, status: Literal["success", "fail"]): # only use this if slack alerting is being used if self.alerting is None: diff --git a/tests/test_litellm/proxy/test_proxy_server.py b/tests/test_litellm/proxy/test_proxy_server.py index 937e4f15741..6a901c5e7de 100644 --- a/tests/test_litellm/proxy/test_proxy_server.py +++ b/tests/test_litellm/proxy/test_proxy_server.py @@ -3166,6 +3166,47 @@ async def test_custom_ui_sso_sign_in_handler_config_loading(): os.unlink(config_file_path) +@pytest.mark.asyncio +async def test_startup_initializes_string_callbacks_after_all_litellm_settings_load(tmp_path, monkeypatch): + from litellm.integrations.s3_v2 import S3Logger + from litellm.litellm_core_utils import litellm_logging + from litellm.proxy.proxy_server import ProxyConfig + from litellm.proxy.utils import ProxyLogging + + config_file = tmp_path / "config.yaml" + config_file.write_text( + "model_list: []\n" + "litellm_settings:\n" + " success_callback:\n" + " - s3_v2\n" + " failure_callback:\n" + " - s3_v2\n" + " s3_callback_params:\n" + " s3_bucket_name: ordering-regression-bucket\n" + " s3_region_name: us-west-2\n" + ) + + monkeypatch.setattr(litellm, "success_callback", []) + monkeypatch.setattr(litellm, "_async_success_callback", []) + monkeypatch.setattr(litellm, "failure_callback", []) + monkeypatch.setattr(litellm, "_async_failure_callback", []) + monkeypatch.setattr(litellm, "callbacks", []) + monkeypatch.setattr(litellm, "s3_callback_params", None) + monkeypatch.setattr(litellm_logging, "_in_memory_loggers", []) + + await ProxyConfig().load_config(router=MagicMock(), config_file_path=str(config_file)) + ProxyLogging(user_api_key_cache=MagicMock())._init_litellm_callbacks(llm_router=None) + + success_loggers = [cb for cb in litellm._async_success_callback if isinstance(cb, S3Logger)] + failure_loggers = [cb for cb in litellm._async_failure_callback if isinstance(cb, S3Logger)] + assert len(success_loggers) == 1 + assert len(failure_loggers) == 1 + assert success_loggers[0].s3_bucket_name == "ordering-regression-bucket" + assert success_loggers[0].s3_region_name == "us-west-2" + assert "s3_v2" not in litellm.success_callback + assert "s3_v2" not in litellm.failure_callback + + @pytest.mark.asyncio async def test_load_config_max_budget_env_var_coerced_to_float(tmp_path, monkeypatch): """ From 6a425a5cc59ce67f3860ac04f752cf61340e1028 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 8 Sep 2026 13:08:26 -0700 Subject: [PATCH 22/34] fix(responses): record spend for native Responses API WebSocket sessions (#38856) * fix(responses): record spend for native Responses API WebSocket sessions Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(responses): bill usage from response.incomplete WebSocket turns A turn cut short by max_output_tokens ends in response.incomplete, which OpenAI bills but the processor only read response.completed, so those sessions still logged zero spend Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(responses): hoist websocket usage test imports to module scope Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(responses): price websocket sessions through the standard cost path The realtime completion_cost branch skips cost_discount_config and cost_margin_config, so a native Responses WebSocket session was priced differently from the same usage over HTTP /v1/responses. Drop the explicit widening so the LiteLLMRealtimeStreamLoggingObject built by normalize_logging_result flows through the generic usage path, and pin WS == HTTP cost under a 50% provider discount in the regression test Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * ci: rerun proxy-infra after flaky test_check_migration process tree test Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: yucheng --- litellm/cost_calculator.py | 40 ++++++++ litellm/litellm_core_utils/litellm_logging.py | 12 +++ litellm/types/utils.py | 1 + .../test_litellm_logging.py | 92 ++++++++++++++++++- 4 files changed, 144 insertions(+), 1 deletion(-) diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 9a9d2ceda03..c9c7df4d75e 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -2414,6 +2414,46 @@ class RealtimeAPITokenUsageProcessor(BaseTokenUsageProcessor): ) +_RESPONSES_WS_BILLABLE_EVENT_TYPES: Final = frozenset({"response.completed", "response.incomplete"}) + + +class _ResponsesWsEventResponse(BaseModel): + usage: Mapping[str, object] | None = None + + +class _ResponsesWsEvent(BaseModel): + type: str = "" + response: _ResponsesWsEventResponse | None = None + + +class ResponsesWebSocketTokenUsageProcessor(BaseTokenUsageProcessor): + @staticmethod + def collect_usage_from_responses_ws_results( + results: Sequence[Mapping[str, object]], + ) -> tuple[Usage, ...]: + events: Final = tuple(_ResponsesWsEvent.model_validate(result) for result in results) + return tuple( + ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage( # pyright: ignore[reportPrivateUsage] # same shared transform the realtime processor uses + event.response.usage + ) + for event in events + if event.type in _RESPONSES_WS_BILLABLE_EVENT_TYPES + and event.response is not None + and event.response.usage is not None + ) + + @staticmethod + def collect_and_combine_usage_from_responses_ws_results( + results: Sequence[Mapping[str, object]], + ) -> Usage: + collected_usage_objects: Final = ResponsesWebSocketTokenUsageProcessor.collect_usage_from_responses_ws_results( + results + ) + return ResponsesWebSocketTokenUsageProcessor.combine_usage_objects( + list(collected_usage_objects) # mutable-ok: combine_usage_objects requires a list parameter + ) + + _TRANSCRIPTION_COMPLETED_EVENT_TYPE: Final = "conversation.item.input_audio_transcription.completed" diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index ca2cca5360f..e3c94866985 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -48,6 +48,7 @@ from litellm.constants import ( ) from litellm.cost_calculator import ( RealtimeAPITokenUsageProcessor, + ResponsesWebSocketTokenUsageProcessor, _select_model_name_for_cost_calc, ) from litellm.exceptions import ( @@ -2028,6 +2029,17 @@ class Logging(LiteLLMLoggingBaseClass): results=result, ) + elif self.call_type == CallTypes.aresponses_websocket.value and isinstance(result, list): # pyright: ignore[reportUnknownMemberType] # Logging.call_type is untyped + combined_ws_usage: Final = ( + ResponsesWebSocketTokenUsageProcessor.collect_and_combine_usage_from_responses_ws_results( + results=result # pyright: ignore[reportUnknownArgumentType] # raw event dicts from the WS stream + ) + ) + logging_result = LiteLLMRealtimeStreamLoggingObject( + usage=combined_ws_usage, + results=result, # pyright: ignore[reportUnknownArgumentType] # raw event dicts from the WS stream + ) + elif ( self.call_type == CallTypes.llm_passthrough_route.value or self.call_type == CallTypes.allm_passthrough_route.value diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 9b5fb08a45f..40a6d77482f 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -580,6 +580,7 @@ CallTypesLiteral = Literal[ "search", "asearch", "_arealtime", + "_aresponses_websocket", "create_batch", "acreate_batch", "create_file", diff --git a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py index 0fdca755685..b6366bc803e 100644 --- a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py +++ b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py @@ -22,7 +22,13 @@ from litellm.litellm_core_utils.litellm_logging import ( _get_status_fields, set_callbacks, ) -from litellm.types.utils import ModelResponse, TextCompletionResponse +from litellm.types.llms.openai import ResponseAPIUsage, ResponsesAPIResponse +from litellm.types.utils import ( + CallTypes, + LiteLLMRealtimeStreamLoggingObject, + ModelResponse, + TextCompletionResponse, +) @pytest.fixture @@ -6393,6 +6399,90 @@ async def test_prompt_hook_injection_marker_recorded_for_every_surface(logging_o assert pre_choice["metadata"]["litellm_gateway_injected_cache"] == "" +def _responses_ws_logging_obj() -> LitellmLogging: + return LitellmLogging( + model="gpt-4o", + messages=[], + stream=False, + call_type=CallTypes.aresponses_websocket.value, + start_time=time.time(), + litellm_call_id="responses-ws-usage-test", + function_id="responses-ws-usage-test", + ) + + +def test_normalize_logging_result_extracts_usage_for_responses_websocket(monkeypatch): + """LIT-6512: native /v1/responses WebSocket sessions logged $0 spend because the usage + carried by stored response.completed events was never extracted. The session must cost + exactly what the same usage costs over HTTP /v1/responses, discounts included.""" + monkeypatch.setattr(litellm, "cost_discount_config", {"openai": 0.5}) + logging_obj = _responses_ws_logging_obj() + events = [ + {"type": "response.created", "response": {}}, + { + "type": "response.completed", + "response": {"usage": {"input_tokens": 100, "output_tokens": 40, "total_tokens": 140}}, + }, + { + "type": "response.completed", + "response": {"usage": {"input_tokens": 60, "output_tokens": 10, "total_tokens": 70}}, + }, + ] + + normalized = logging_obj.normalize_logging_result(result=events) + + assert isinstance(normalized, LiteLLMRealtimeStreamLoggingObject) + assert normalized.usage.prompt_tokens == 160 + assert normalized.usage.completion_tokens == 50 + + ws_cost = litellm.completion_cost( + completion_response=normalized, + model="gpt-4o", + call_type=CallTypes.aresponses_websocket.value, + custom_llm_provider="openai", + ) + http_cost = litellm.completion_cost( + completion_response=ResponsesAPIResponse( + id="resp-6512", + created_at=1700000000, + output=[], + usage=ResponseAPIUsage(input_tokens=160, output_tokens=50, total_tokens=210), + ), + model="gpt-4o", + call_type=CallTypes.aresponses.value, + custom_llm_provider="openai", + ) + assert ws_cost > 0 + assert ws_cost == http_cost + + +def test_normalize_logging_result_bills_incomplete_responses_websocket_turns(): + """LIT-6512: a turn cut short by max_output_tokens ends in response.incomplete, which + OpenAI bills, so its usage counts toward the session like a completed turn.""" + events = [ + { + "type": "response.created", + "response": {"usage": {"input_tokens": 999, "output_tokens": 999, "total_tokens": 1998}}, + }, + { + "type": "response.incomplete", + "response": {"usage": {"input_tokens": 15, "output_tokens": 16, "total_tokens": 31}}, + }, + { + "type": "response.completed", + "response": {"usage": {"input_tokens": 40, "output_tokens": 4, "total_tokens": 44}}, + }, + {"type": "response.failed", "response": {"usage": None}}, + ] + + normalized = _responses_ws_logging_obj().normalize_logging_result(result=events) + + assert isinstance(normalized, LiteLLMRealtimeStreamLoggingObject) + assert normalized.usage.prompt_tokens == 55 + assert normalized.usage.completion_tokens == 20 + assert normalized.usage.total_tokens == 75 + + def test_get_standard_logging_object_payload_reads_overhead_from_logging_obj_for_dict_results(logging_obj): """LIT-5466: /v1/messages returns a plain dict with no _hidden_params, so the overhead recorded on the logging object must reach hidden_params.litellm_overhead_time_ms (SpendLogs).""" From 9d164fa34150ac7dc0551d9efec16cdb27848c5c Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Tue, 8 Sep 2026 13:24:15 -0700 Subject: [PATCH 23/34] bump: litellm-proxy-extras 0.4.94 -> 0.4.95, litellm 1.101.0 -> 1.102.0 --- litellm-proxy-extras/pyproject.toml | 4 ++-- pyproject.toml | 6 +++--- uv.lock | 6 +++--- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/litellm-proxy-extras/pyproject.toml b/litellm-proxy-extras/pyproject.toml index 82d31fec373..91b4e4a7ba1 100644 --- a/litellm-proxy-extras/pyproject.toml +++ b/litellm-proxy-extras/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "litellm-proxy-extras" -version = "0.4.94" +version = "0.4.95" description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package." readme = "README.md" requires-python = ">=3.9" @@ -26,7 +26,7 @@ required-version = ">=0.10.9" module-root = "" [tool.commitizen] -version = "0.4.94" +version = "0.4.95" version_files = [ "pyproject.toml:^version", "../pyproject.toml:litellm-proxy-extras==", diff --git a/pyproject.toml b/pyproject.toml index af35c77d259..706b14b8a04 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "litellm" -version = "1.101.0" +version = "1.102.0" description = "Library to easily interface with LLM API providers" readme = "README.md" requires-python = ">=3.10, <3.15" @@ -67,7 +67,7 @@ proxy = [ "azure-identity>=1.25.2,<2.0", "azure-storage-blob>=12.28.0,<13.0", "mcp>=1.28.1,<2.0", - "litellm-proxy-extras==0.4.94", + "litellm-proxy-extras==0.4.95", "litellm-enterprise==0.1.65", "RestrictedPython>=8.5,<9.0", "rich>=13.9.4,<14.0", @@ -328,7 +328,7 @@ members = ["enterprise", "litellm-proxy-extras"] profile = "black" [tool.commitizen] -version = "1.101.0" +version = "1.102.0" version_files = [ "pyproject.toml:^version", ] diff --git a/uv.lock b/uv.lock index 89205cd9527..12dc2c69d23 100644 --- a/uv.lock +++ b/uv.lock @@ -10,7 +10,7 @@ resolution-markers = [ ] [options] -exclude-newer = "2026-09-02T16:58:34.594994Z" +exclude-newer = "2026-09-05T20:24:07.535116Z" exclude-newer-span = "P3D" [manifest] @@ -4358,7 +4358,7 @@ wheels = [ [[package]] name = "litellm" -version = "1.101.0" +version = "1.102.0" source = { editable = "." } dependencies = [ { name = "aiohttp" }, @@ -4776,7 +4776,7 @@ source = { editable = "enterprise" } [[package]] name = "litellm-proxy-extras" -version = "0.4.94" +version = "0.4.95" source = { editable = "litellm-proxy-extras" } [[package]] From 4a3a78c256bab366597a76d44930b323bce6335b Mon Sep 17 00:00:00 2001 From: tin-berri Date: Tue, 8 Sep 2026 13:28:57 -0700 Subject: [PATCH 24/34] feat(complexity_router): rebalance heuristic weights in the dashboard and grade custom dimensions by match count (#40205) The Advanced scoring editor now lists built-in and custom dimensions together. Editing any weight holds it and rescales the others proportionally so the vector totals 1.00, and Save stores those explicit values. The backend scores exactly what is stored, with no runtime normalization, so routers nobody edits keep their weights. CustomDimension gains an opt-in scoring_mode. match_count scores 0, 0.5 or 1 by distinct matcher hits; the default stays binary. The tuning fingerprint omits a binary scoring_mode, so routers written before this change keep their recorded baseline and the upgrade does not consume the free heuristic-v1 tuning slot. --- .../complexity_router/README.md | 12 +- .../complexity_router/complexity_router.py | 38 ++++- .../complexity_router/config.py | 13 +- .../auto_router_tuning_baseline.py | 12 +- .../router_strategy/test_complexity_router.py | 148 +++++++++++++----- .../test_auto_router_tuning_baseline.py | 73 ++++++++- .../add_model/ClassificationMethodConfig.tsx | 5 +- .../add_model/ComplexityRouterConfig.test.tsx | 6 +- .../add_model/ComplexityRouterConfig.tsx | 6 + .../add_model/CustomDimensionRows.tsx | 136 ++++++++++++++++ ...euristicScoringConfig.integration.test.tsx | 90 +++++++++++ .../add_model/HeuristicScoringConfig.test.tsx | 14 +- .../add_model/HeuristicScoringConfig.tsx | 85 ++++++++-- .../add_model/add_auto_router_tab.tsx | 4 + .../build_complexity_router_config.test.ts | 47 +++++- .../build_complexity_router_config.ts | 17 +- .../add_model/custom_dimensions.test.ts | 41 +++++ .../components/add_model/custom_dimensions.ts | 54 +++++++ .../add_model/heuristic_scoring_knobs.test.ts | 119 ++++++++++++++ .../add_model/heuristic_scoring_knobs.ts | 97 +++++++++++- .../src/components/add_model/tier_rows.ts | 1 + ...d_updated_complexity_router_config.test.ts | 49 ++++++ .../edit_auto_router_modal.tsx | 17 +- .../src/lib/autorouter_presets.test.ts | 28 ++++ .../src/lib/autorouter_presets.ts | 12 ++ ui/litellm-dashboard/src/lib/http/schema.d.ts | 9 +- 26 files changed, 1047 insertions(+), 86 deletions(-) create mode 100644 ui/litellm-dashboard/src/components/add_model/CustomDimensionRows.tsx create mode 100644 ui/litellm-dashboard/src/components/add_model/HeuristicScoringConfig.integration.test.tsx create mode 100644 ui/litellm-dashboard/src/components/add_model/custom_dimensions.test.ts create mode 100644 ui/litellm-dashboard/src/components/add_model/custom_dimensions.ts diff --git a/litellm/router_strategy/complexity_router/README.md b/litellm/router_strategy/complexity_router/README.md index 88ed374dd3f..d605b43e42a 100644 --- a/litellm/router_strategy/complexity_router/README.md +++ b/litellm/router_strategy/complexity_router/README.md @@ -207,17 +207,27 @@ custom_dimensions: - name: sqlMigration weight: 0.7 patterns: ['\b(create|alter|drop)\s{1,4}table\b'] + - name: dataPipeline + weight: 0.4 + scoring_mode: match_count + keywords: [airflow, dbt, snowflake] ``` Each dimension contributes its weight once when any matcher hits the current ask. Repeated matches do not increase it. The built-in score and tier boundaries are unchanged, and the total score is not renormalized. Keywords use the existing case-insensitive word-boundary and CJK rules. Regexes search the first 2048 characters case-insensitively and compile during configuration validation and router initialization, never per request +`scoring_mode` is optional and defaults to `binary`, the behavior above. `match_count` grades the dimension by how many distinct matchers hit: none scores 0 and emits no signal, one scores half the weight, two or more score the full weight. Repeated occurrences of one matcher never raise the count, keywords are distinct case-insensitively, patterns are distinct by source, and a keyword and a pattern are always distinct from each other. Matching stops as soon as the selected mode's maximum is reached, so a binary dimension still stops at its first hit. Existing configurations without the field keep binary scoring and the same tuning fingerprint, so the field only counts as a tuning change when set to `match_count` + +### Weights through the API versus the dashboard + +The API and YAML store exactly the weights written. A `dimension_weights` map and inline custom weights are read literally, missing recognized built-in names score zero, and nothing renormalizes the vector, so a total other than 1 is legal and scores accordingly. The dashboard's heuristic scoring editor is the one place that rebalances: editing one weight there holds it and redistributes the remainder across the other active dimensions in the draft, then Save sends the resulting explicit values, which the backend stores and scores as written. Opening a router, applying a preset, editing matchers, changing `scoring_mode`, or saving unrelated fields never normalizes existing weights + Only `heuristic`, `heuristic_first` and `hybrid` accept custom dimensions. Each name must be a unique ASCII identifier starting with a letter, at most 64 characters, and cannot reuse a built-in dimension name or a key in `dimension_weights`. Set its weight inline, greater than zero and at most one Patterns are checked at configuration time against a grammar whose worst case stays a few milliseconds on 2048 characters. Every quantifier needs an explicit upper bound of at most 64 and must repeat a single character or character class, so `\s{1,4}` is accepted while `\s+`, `(a|aa){0,12}` and `(?:ab){0,64}` are refused. Backreferences, lookarounds, atomic groups and possessive quantifiers are refused as well. Each pattern is then costed: alternation branches and repeat lengths multiply the ways the engine can retry, and every later piece of the pattern is charged once per path that can reach it, so `a?a?a?a?a?a?a?a?` followed by a long fixed tail is refused even though each quantifier is small. The budget is 2048 work units per pattern and 8192 across the router. An invalid or over-budget pattern fails the write with a message naming the pattern and the rule it broke Limits are 16 dimensions, 32 combined keywords/patterns per dimension, 256 characters per matcher and 4096 matcher characters per dimension. Matching runs inline on the request path with no timeout and no worker thread, because the grammar is what bounds the cost. These are routing hints, not security enforcement rules -The existing heuristic-v1 tuning quota covers custom dimensions and their weights: one changed router without an auto-router license, unlimited with the entitlement. Omitting `custom_dimensions` preserves existing scoring. Routing decisions and spend logs include signals such as `custom (sqlMigration)` without recording the configured pattern or matched text. The field is configured through YAML or the model API; this change adds no dashboard editor +The existing heuristic-v1 tuning quota covers custom dimensions, their weights and their scoring mode: one changed router without an auto-router license, unlimited with the entitlement. Omitting `custom_dimensions` preserves existing scoring. Routing decisions and spend logs include signals such as `custom (sqlMigration)` without recording the configured pattern or matched text. The field is configured through YAML, the model API, or the dashboard's heuristic scoring editor ## Usage diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py index c8644f52c57..ae1f82fc40b 100644 --- a/litellm/router_strategy/complexity_router/complexity_router.py +++ b/litellm/router_strategy/complexity_router/complexity_router.py @@ -20,7 +20,7 @@ import random import re import time from collections.abc import Callable, Iterator, Mapping, Sequence -from itertools import accumulate, islice, takewhile +from itertools import accumulate, chain, islice, takewhile from threading import Lock from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Literal, NamedTuple, cast @@ -82,6 +82,7 @@ from .config import ( ClassificationRubric, ComplexityRouterConfig, ComplexityTier, + CustomDimension, TierDefinition, ) from .stall_detector import detect_stalled_task @@ -879,6 +880,15 @@ class DimensionScore: self.signal = signal +class _CustomDimensionMatchers(NamedTuple): + """One custom dimension's distinct matchers and the number of hits that saturates its score.""" + + dimension: CustomDimension + keywords: tuple[str, ...] + patterns: tuple[re.Pattern[str], ...] + saturation: int + + class KeywordOverride(NamedTuple): """A keyword_tier_rules match: the winning tier and, on the lexical path, the keyword that fired.""" @@ -1121,7 +1131,12 @@ class ComplexityRouter(CustomLogger): ) self.simple_keywords = self.config.simple_keywords or DEFAULT_SIMPLE_KEYWORDS self._custom_dimensions = tuple( - (dimension, tuple(re.compile(pattern, re.IGNORECASE) for pattern in dimension.patterns)) + _CustomDimensionMatchers( + dimension, + tuple(dict.fromkeys(keyword.lower() for keyword in dimension.keywords)), + tuple(re.compile(pattern, re.IGNORECASE) for pattern in dict.fromkeys(dimension.patterns)), + 2 if dimension.scoring_mode == "match_count" else 1, + ) for dimension in self.config.custom_dimensions ) if self.config.has_custom_tiers: @@ -1325,15 +1340,26 @@ class ComplexityRouter(CustomLogger): score: Final = score_high if match_count >= high_threshold else score_low return DimensionScore(name, score, f"{signal_label} ({detail})"), match_count + def _count_custom_hits(self, matchers: _CustomDimensionMatchers, user_text: str, scanned: str) -> int: + hits: Final = chain( + (self._keyword_matches(user_text, keyword) for keyword in matchers.keywords), + (pattern.search(scanned) is not None for pattern in matchers.patterns), + ) + return sum(islice((1 for hit in hits if hit), matchers.saturation)) + def _score_custom_dimensions(self, prompt: str, user_text: str) -> tuple[tuple[DimensionScore, float], ...]: if not self._custom_dimensions: return () scanned: Final = prompt[:CUSTOM_PATTERN_SCAN_CHARS] return tuple( - (DimensionScore(dimension.name, 1.0, f"custom ({dimension.name})"), dimension.weight) - for dimension, patterns in self._custom_dimensions - if any(self._keyword_matches(user_text, keyword) for keyword in dimension.keywords) - or any(pattern.search(scanned) is not None for pattern in patterns) + ( + DimensionScore( + matchers.dimension.name, hits / matchers.saturation, f"custom ({matchers.dimension.name})" + ), + matchers.dimension.weight, + ) + for matchers in self._custom_dimensions + if (hits := self._count_custom_hits(matchers, user_text, scanned)) ) def _score_multi_step(self, text: str) -> DimensionScore: diff --git a/litellm/router_strategy/complexity_router/config.py b/litellm/router_strategy/complexity_router/config.py index 3ec9f9b5394..37375d9727d 100644 --- a/litellm/router_strategy/complexity_router/config.py +++ b/litellm/router_strategy/complexity_router/config.py @@ -667,6 +667,14 @@ class CustomDimension(BaseModel): weight: float = Field(gt=0, le=1, allow_inf_nan=False) keywords: tuple[Annotated[str, Field(min_length=1, max_length=256)], ...] = Field(default=(), max_length=32) patterns: tuple[Annotated[str, Field(min_length=1, max_length=256)], ...] = Field(default=(), max_length=32) + scoring_mode: Literal["binary", "match_count"] = Field( + default="binary", + description=( + "'binary' scores 1 when any matcher hits. 'match_count' scores 0.5 when one distinct matcher hits and 1 " + "when two or more do; repeated occurrences of one matcher never raise it. Keywords are distinct " + "case-insensitively, patterns by source, and a keyword and a pattern are always distinct from each other." + ), + ) @model_validator(mode="after") def _validate_matchers(self) -> "CustomDimension": @@ -794,8 +802,9 @@ class ComplexityRouterConfig(BaseModel): default=(), max_length=16, description=( - "Named binary dimensions added to the heuristic-v1 score. Each contributes its inline weight once " - "when any keyword matches the current ask or a case-insensitive regex matches its first 2048 characters. " + "Named dimensions added to the heuristic-v1 score. Each contributes its inline weight once " + "when any keyword matches the current ask or a case-insensitive regex matches its first 2048 characters; " + "scoring_mode 'match_count' instead grades half weight for one distinct matcher and full for two or more. " "Regex quantifiers repeat one character or class at most 64 times. Unbounded quantifiers, repeated groups, " "backreferences and lookarounds are rejected. Conservative work limits include alternation paths, " "repeat lengths and subsequent matching: 2048 units per pattern, 8192 across the router. " diff --git a/litellm/router_utils/auto_router_tuning_baseline.py b/litellm/router_utils/auto_router_tuning_baseline.py index 74f7b82389a..9699ab886b9 100644 --- a/litellm/router_utils/auto_router_tuning_baseline.py +++ b/litellm/router_utils/auto_router_tuning_baseline.py @@ -52,7 +52,17 @@ def tuning_fingerprint(complexity_router_config: object) -> str | None: supplied: Final = ((_TUNING_FIELD_SET - frozenset(("tier_model_configs",))) & frozenset(raw)) | ( frozenset(("tier_model_configs",)) if validated.tier_model_configs else frozenset() ) - payload: Final = validated.model_dump(mode="json", include=supplied) + payload: Final = validated.model_dump( + mode="json", + include=supplied, + exclude={ + "custom_dimensions": { + index: {"scoring_mode"} + for index, dimension in enumerate(validated.custom_dimensions) + if dimension.scoring_mode == "binary" + } + }, + ) return hashlib.sha256(json.dumps(payload, sort_keys=True, separators=(",", ":")).encode()).hexdigest() diff --git a/tests/test_litellm/router_strategy/test_complexity_router.py b/tests/test_litellm/router_strategy/test_complexity_router.py index 5b1d8562abd..b0f2a65bb9b 100644 --- a/tests/test_litellm/router_strategy/test_complexity_router.py +++ b/tests/test_litellm/router_strategy/test_complexity_router.py @@ -826,6 +826,8 @@ class TestCustomDimensions: pytest.param({"keywords": ["x"] * 32, "patterns": ["y"]}, {}, id="combined-matcher-count"), pytest.param({"keywords": ["x" * 256] * 17}, {}, id="matcher-character-budget"), pytest.param({"unknown": True}, {}, id="extra-field"), + pytest.param({"scoring_mode": "graded"}, {}, id="unknown-scoring-mode"), + pytest.param({"scoring_mode": None}, {}, id="null-scoring-mode"), ], ) def test_custom_dimension_invalid_configuration_rejected( @@ -878,43 +880,125 @@ class TestCustomDimensions: ) @pytest.mark.asyncio - @pytest.mark.parametrize("current_ask", ("Hello!", "orbitmesh")) + @pytest.mark.parametrize("scoring_mode", ("binary", "match_count")) + @pytest.mark.parametrize("current_ask", ("Hello!", "orbitmesh", "orbitmesh fluxgate")) async def test_custom_dimensions_public_hook_scores_only_current_ask( - self, mock_router_instance: MagicMock, current_ask: str + self, mock_router_instance: MagicMock, current_ask: str, scoring_mode: str ) -> None: router: Final = ComplexityRouter( "test-router", mock_router_instance, { "tiers": {"SIMPLE": "cheap", "MEDIUM": "mid", "COMPLEX": "strong", "REASONING": "top"}, - "custom_dimensions": [{"name": "internalFrameworks", "weight": 0.7, "keywords": ["orbitmesh"]}], + "dimension_weights": {}, + "custom_dimensions": [ + { + "name": "internalFrameworks", + "weight": 0.8, + "keywords": ["orbitmesh", "fluxgate"], + "scoring_mode": scoring_mode, + } + ], }, ) result: Final = await router.async_pre_routing_hook( model="test-router", request_kwargs={}, messages=[ - {"role": "system", "content": "orbitmesh"}, - {"role": "user", "content": "orbitmesh"}, - {"role": "assistant", "content": "orbitmesh is ready"}, + {"role": "system", "content": "orbitmesh fluxgate"}, + {"role": "user", "content": "orbitmesh fluxgate"}, + {"role": "assistant", "content": "orbitmesh fluxgate is ready"}, {"role": "user", "content": current_ask}, - {"role": "tool", "tool_call_id": "previous", "content": "orbitmesh"}, + {"role": "tool", "tool_call_id": "previous", "content": "orbitmesh fluxgate"}, ], ) assert result is not None assert result.routing_decision is not None - assert ("custom (internalFrameworks)" in result.routing_decision["signals"]) is (current_ask == "orbitmesh") - assert result.model == ("top" if current_ask == "orbitmesh" else "cheap") + expected_score: Final = ( + 0.0 + if current_ask == "Hello!" + else 0.4 + if scoring_mode == "match_count" and current_ask == "orbitmesh" + else 0.8 + ) + assert result.routing_decision["score"] == expected_score + assert ("custom (internalFrameworks)" in result.routing_decision["signals"]) is (expected_score > 0) + assert result.model == ("cheap" if expected_score == 0 else "strong" if expected_score == 0.4 else "top") assert "orbitmesh" not in " ".join(result.routing_decision["signals"]) - def test_custom_patterns_scan_only_the_first_2048_characters(self, mock_router_instance: MagicMock) -> None: + @pytest.mark.parametrize("scoring_mode", ("binary", "match_count")) + def test_custom_patterns_scan_only_the_first_2048_characters( + self, mock_router_instance: MagicMock, scoring_mode: str + ) -> None: router: Final = ComplexityRouter( "test-router", mock_router_instance, - {"custom_dimensions": [{"name": "late", "weight": 0.7, "patterns": [r"zzz{1,3}"]}]}, + { + "custom_dimensions": [ + { + "name": "late", + "weight": 0.7, + "patterns": [r"zzz{1,3}", r"yyy{1,3}"], + "scoring_mode": scoring_mode, + } + ] + }, ) + baseline: Final = ComplexityRouter("test-router", mock_router_instance) assert "custom (late)" in router.classify("a" * 2040 + " zzz")[2] assert "custom (late)" not in router.classify("a" * 2048 + " zzz")[2] + second_hit_past_the_bound: Final = "yyy " + "a" * 2044 + " zzz" + contribution: Final = ( + router.classify(second_hit_past_the_bound)[1] - baseline.classify(second_hit_past_the_bound)[1] + ) + assert contribution == pytest.approx(0.7 if scoring_mode == "binary" else 0.35) + + @pytest.mark.parametrize( + "prompt,expected_score", + [ + pytest.param("Hello!", 0.0, id="no-hit"), + pytest.param("orbitmesh orbitmesh ORBITMESH again", 0.5, id="one-keyword-repeated"), + pytest.param("create table a; CREATE TABLE b; create table c", 0.5, id="one-pattern-repeated"), + pytest.param("orbitmesh and fluxgate", 1.0, id="two-keywords"), + pytest.param("orbitmesh then create table t", 1.0, id="keyword-plus-pattern"), + pytest.param("create table a; alter table b", 1.0, id="two-patterns"), + pytest.param("orbitmesh fluxgate create table a alter table b", 1.0, id="all-matchers"), + ], + ) + def test_match_count_grades_distinct_matchers( + self, mock_router_instance: MagicMock, prompt: str, expected_score: float + ) -> None: + dimension: Final = { + "name": "graded", + "weight": 0.6, + "keywords": ["orbitmesh", "ORBITMESH", "fluxgate"], + "patterns": [r"\bcreate\s{1,4}table\b", r"\bcreate\s{1,4}table\b", r"\balter\s{1,4}table\b"], + } + baseline: Final = ComplexityRouter("test-router", mock_router_instance) + binary: Final = ComplexityRouter("test-router", mock_router_instance, {"custom_dimensions": [dimension]}) + graded: Final = ComplexityRouter( + "test-router", + mock_router_instance, + {"custom_dimensions": [{**dimension, "scoring_mode": "match_count"}]}, + ) + _, baseline_score, baseline_signals = baseline.classify(prompt) + _, binary_score, binary_signals = binary.classify(prompt) + _, graded_score, graded_signals = graded.classify(prompt) + assert graded_score == pytest.approx(baseline_score + 0.6 * expected_score) + assert binary_score == pytest.approx(baseline_score + (0.6 if expected_score else 0.0)) + expected_signals: Final = [*baseline_signals, *(["custom (graded)"] if expected_score else [])] + assert graded_signals == expected_signals + assert binary_signals == expected_signals + + def test_scoring_mode_round_trips_and_defaults_to_binary(self) -> None: + dimension: Final = {"name": "graded", "weight": 0.6, "keywords": ["orbitmesh"]} + legacy: Final = ComplexityRouterConfig.model_validate({"custom_dimensions": [dimension]}) + graded: Final = ComplexityRouterConfig.model_validate( + {"custom_dimensions": [{**dimension, "scoring_mode": "match_count"}]} + ) + assert legacy.custom_dimensions[0].scoring_mode == "binary" + assert graded.model_dump(mode="json")["custom_dimensions"][0]["scoring_mode"] == "match_count" + assert ComplexityRouterConfig.model_validate(graded.model_dump(mode="json")) == graded def test_custom_dimensions_router_wide_regex_work_is_capped(self) -> None: heavy: Final = {"weight": 0.5, "patterns": ["a?" * 8 + "z"]} @@ -1511,6 +1595,7 @@ class TestRouterComplexityDeploymentMethods: def test_the_shipped_rubric_and_default_prompt_stay_free(self) -> None: """Only an operator-written prompt is gated: picking a shipped rubric preset, or writing no prompt at all, leaves a router unmetered, so several of them register under a ceiling of one.""" + def rubric(model_name: str, model_id: str, preset: str | None) -> dict[str, object]: llm_config: dict[str, object] = {"model": "gpt-4o-mini"} if preset is not None: @@ -1647,6 +1732,7 @@ class TestRouterComplexityDeploymentMethods: def test_renaming_built_in_tiers_is_not_a_custom_tier_set(self) -> None: """tier_labels renames the built-in ladder without defining one, so it stays ungated: two such routers register under a ceiling of one.""" + def labeled(model_name: str, model_id: str) -> dict[str, object]: row = self._router_row(model_name, model_id, "heuristic") row["litellm_params"]["complexity_router_config"]["tier_labels"] = {"SIMPLE": "Cheap", "MEDIUM": "Standard"} @@ -2520,9 +2606,7 @@ class TestLLMClassifier: assert outcome.classifier_cost == pytest.approx(1.35e-05) @pytest.mark.asyncio - async def test_aclassify_timeout_does_not_inherit_router_retries_or_fallbacks( - self, llm_classifier_config - ): + async def test_aclassify_timeout_does_not_inherit_router_retries_or_fallbacks(self, llm_classifier_config): real_router = Router( model_list=[ { @@ -2565,9 +2649,7 @@ class TestLLMClassifier: assert real_router.total_calls["openai/mock-backup-classifier"] == 0 @pytest.mark.asyncio - async def test_aclassify_enforces_total_classifier_deadline( - self, mock_router_instance, llm_classifier_config - ): + async def test_aclassify_enforces_total_classifier_deadline(self, mock_router_instance, llm_classifier_config): cancelled = asyncio.Event() async def slow_classifier(**_kwargs: object) -> None: @@ -12414,9 +12496,7 @@ class TestTierHealthFailover: llm_provider="", ) filtered = (*cooling, *blocked, *excluded) - healthy = [ - {"model_name": model, "model_info": {"id": i}} for i in ids_by_model[model] if i not in filtered - ] + healthy = [{"model_name": model, "model_info": {"id": i}} for i in ids_by_model[model] if i not in filtered] if not healthy: raise RouterRateLimitError( model=model, cooldown_time=60.0, enable_pre_call_checks=False, cooldown_list=[] @@ -12845,9 +12925,7 @@ class TestTierHealthFailover: assert all(probed is not request_kwargs for probed in router.litellm_router_instance.probed_kwargs) @pytest.mark.asyncio - async def test_a_peer_whose_every_deployment_is_over_its_rpm_is_not_a_failover_target( - self, mock_router_instance - ): + async def test_a_peer_whose_every_deployment_is_over_its_rpm_is_not_a_failover_target(self, mock_router_instance): """RPM exhaustion is its own verdict from the owner (RouterRateLimitErrorBasic). A peer in that state would be rejected downstream, so it cannot be the substitute.""" from litellm.types.router import RouterRateLimitErrorBasic @@ -12880,9 +12958,7 @@ class TestTierHealthFailover: assert {r.model for r in results} == {"live-c"} @pytest.mark.asyncio - async def test_the_probe_forwards_input_so_window_checks_run_on_input_only_surfaces( - self, mock_router_instance - ): + async def test_the_probe_forwards_input_so_window_checks_run_on_input_only_surfaces(self, mock_router_instance): """The Responses API carries its prompt as `input`, never as messages. The owner only runs its context-window pre-call check when one of them is present, so dropping `input` would silently skip window filtering on that whole surface.""" @@ -12908,9 +12984,7 @@ class TestTierHealthFailover: ), "the eligibility probe must forward `input` to the owner" @pytest.mark.asyncio - async def test_a_group_the_router_has_no_deployment_for_is_not_a_failover_target( - self, mock_router_instance - ): + async def test_a_group_the_router_has_no_deployment_for_is_not_a_failover_target(self, mock_router_instance): """The owner answers an unconfigured group with BadRequestError. Reading that as live would both skip failover off it and let it be chosen as a substitute.""" router = self._router( @@ -13099,9 +13173,7 @@ class TestClassifierVision: routed as default_fallback on text the request never contained. """ router = self._router(mock_router_instance, vision={"enabled": True}) - response = await router.async_pre_routing_hook( - model="m", request_kwargs={}, messages=self._turn(IMG_PART) - ) + response = await router.async_pre_routing_hook(model="m", request_kwargs={}, messages=self._turn(IMG_PART)) assert response.routing_decision["cause"] == "llm_classifier" assert response.model == "t-complex" assert [block["type"] for block in self._classifier_user_content(mock_router_instance)] == [ @@ -13112,9 +13184,7 @@ class TestClassifierVision: @pytest.mark.asyncio async def test_image_only_turn_still_falls_back_when_vision_is_off(self, mock_router_instance): router = self._router(mock_router_instance, vision={"enabled": False}) - response = await router.async_pre_routing_hook( - model="m", request_kwargs={}, messages=self._turn(IMG_PART) - ) + response = await router.async_pre_routing_hook(model="m", request_kwargs={}, messages=self._turn(IMG_PART)) assert response.routing_decision["cause"] == "default_fallback" mock_router_instance.acompletion.assert_not_awaited() @@ -13184,9 +13254,7 @@ class TestClassifierVision: makes the image the only variable; a margin loose enough to leave the score undecided would pass whether or not the guard exists. """ - router = self._router( - mock_router_instance, vision={"enabled": True}, classifier_type=classifier_type, **extra - ) + router = self._router(mock_router_instance, vision={"enabled": True}, classifier_type=classifier_type, **extra) response = await router.async_pre_routing_hook( model="m", request_kwargs={}, messages=self._turn({"type": "text", "text": "what is this"}, IMG_PART) ) @@ -13200,9 +13268,7 @@ class TestClassifierVision: self, mock_router_instance, classifier_type, extra, short_circuit_cause ): """The negative class: same router, same text, no image, and the scorer still decides.""" - router = self._router( - mock_router_instance, vision={"enabled": True}, classifier_type=classifier_type, **extra - ) + router = self._router(mock_router_instance, vision={"enabled": True}, classifier_type=classifier_type, **extra) response = await router.async_pre_routing_hook( model="m", request_kwargs={}, messages=[{"role": "user", "content": "what is this"}] ) diff --git a/tests/test_litellm/router_utils/test_auto_router_tuning_baseline.py b/tests/test_litellm/router_utils/test_auto_router_tuning_baseline.py index f686a62db76..75c115cd3ab 100644 --- a/tests/test_litellm/router_utils/test_auto_router_tuning_baseline.py +++ b/tests/test_litellm/router_utils/test_auto_router_tuning_baseline.py @@ -7,6 +7,7 @@ from typing import Final import pytest +from litellm.router_strategy.complexity_router.config import ComplexityRouterConfig from litellm.router_utils.auto_router_tuning_baseline import ( DEFAULT_TUNING_FINGERPRINT, HEURISTIC_V1_TUNING_FIELDS, @@ -21,6 +22,33 @@ from litellm.router_utils.auto_router_tuning_baseline import ( _TIERS = {"SIMPLE": "cheap", "MEDIUM": "mid", "COMPLEX": "strong", "REASONING": "top"} _ALT_TIERS = {**_TIERS, "COMPLEX": "other-strong"} +_KEYWORD_DIMENSION: Final = {"name": "internalFrameworks", "weight": 0.7, "keywords": ["orbitmesh"]} +_HISTORICAL_FINGERPRINTS: Final = ( + ({}, "44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a"), + ( + {"custom_dimensions": [_KEYWORD_DIMENSION]}, + "b5c3c3f3be6341a8a16148d68d9067e03f94ed01a0bbfcde7955763042744372", + ), + ( + {"custom_dimensions": [{"name": "sqlDdl", "weight": 0.4, "patterns": [r"\bCREATE\s{1,4}TABLE\b"]}]}, + "814ce0017fc7f60a160b262f658d910e9bdf784e6139a4ba4f1e2657aa203950", + ), + ( + { + "tiers": _TIERS, + "dimension_weights": {"codePresence": 0.3}, + "custom_dimensions": [ + { + "name": "internalFrameworks", + "weight": 0.2, + "keywords": ["orbitmesh", "fluxgate"], + "patterns": [r"\bALTER\s{1,4}TABLE\b"], + } + ], + }, + "38970dc9224e265ab38c89674563d8d0537822591f9239b45251db6f5ca6cc39", + ), +) def _router( @@ -77,6 +105,27 @@ class TestTuningFingerprint: def test_explicit_empty_tier_model_configs_follow_omission(self) -> None: assert tuning_fingerprint({"tier_model_configs": {}}) == DEFAULT_TUNING_FINGERPRINT + @pytest.mark.parametrize(("config", "fingerprint"), _HISTORICAL_FINGERPRINTS) + def test_fingerprints_recorded_before_scoring_mode_existed_are_preserved( + self, config: Mapping[str, object], fingerprint: str + ) -> None: + """Literal hashes captured from the merged implementation at 9bc9104102, before CustomDimension.scoring_mode.""" + assert tuning_fingerprint(config) == fingerprint + + def test_binary_scoring_mode_hashes_like_its_absence(self) -> None: + historical: Final = tuning_fingerprint({"custom_dimensions": [_KEYWORD_DIMENSION]}) + explicit: Final = tuning_fingerprint({"custom_dimensions": [{**_KEYWORD_DIMENSION, "scoring_mode": "binary"}]}) + reserialized: Final = ComplexityRouterConfig.model_validate( + {"custom_dimensions": [_KEYWORD_DIMENSION]} + ).model_dump(mode="json", include={"custom_dimensions"}) + assert reserialized["custom_dimensions"][0]["scoring_mode"] == "binary" + assert reserialized["custom_dimensions"][0]["patterns"] == [] + assert historical == explicit == tuning_fingerprint(reserialized) + assert ( + tuning_fingerprint({"custom_dimensions": [{**_KEYWORD_DIMENSION, "scoring_mode": "match_count"}]}) + != historical + ) + def test_tier_model_overrides_change_the_fingerprint(self) -> None: plain = tuning_fingerprint({"tiers": {"SIMPLE": "x"}}) with_override = tuning_fingerprint( @@ -218,15 +267,18 @@ class TestQuota: is None ) - def test_custom_dimension_add_edit_and_revert_share_one_quota_slot(self) -> None: + @pytest.mark.parametrize( + "edit", + [ + pytest.param({"weight": 0.9}, id="weight"), + pytest.param({"scoring_mode": "match_count"}, id="scoring-mode"), + ], + ) + def test_custom_dimension_add_edit_and_revert_share_one_quota_slot(self, edit: Mapping[str, object]) -> None: baselines: Final = snapshot_tuning_baselines(()) original: Final = _router("a", {}) - config: Final = { - "custom_dimensions": [{"name": "internalFrameworks", "weight": 0.7, "keywords": ["orbitmesh"]}] - } - edited_config: Final = { - "custom_dimensions": [{"name": "internalFrameworks", "weight": 0.9, "keywords": ["orbitmesh"]}] - } + config: Final = {"custom_dimensions": [_KEYWORD_DIMENSION]} + edited_config: Final = {"custom_dimensions": [{**_KEYWORD_DIMENSION, **edit}]} added: Final = _router("a", config) edited: Final = _router("a", edited_config) second: Final = _router("b", config) @@ -240,6 +292,13 @@ class TestQuota: assert mutable_tuned_identities((original,), baselines) == frozenset() assert tuning_quota_violation(candidate=second, others=(original,), baselines=baselines, limit=1) is None + def test_graded_dimension_recorded_at_snapshot_is_its_own_baseline(self) -> None: + graded: Final = _router("a", {"custom_dimensions": [{**_KEYWORD_DIMENSION, "scoring_mode": "match_count"}]}) + baselines: Final = snapshot_tuning_baselines((graded,)) + assert mutable_tuned_identities((graded,), baselines) == frozenset() + reverted_to_binary: Final = _router("a", {"custom_dimensions": [_KEYWORD_DIMENSION]}) + assert mutable_tuned_identities((reverted_to_binary,), baselines) == {router_identity(graded)} + def test_violation_message_names_the_limit_and_remedy(self) -> None: message = tuning_limit_violation(held=2, limit=1) assert message is not None diff --git a/ui/litellm-dashboard/src/components/add_model/ClassificationMethodConfig.tsx b/ui/litellm-dashboard/src/components/add_model/ClassificationMethodConfig.tsx index 15cfff01766..111eeebe5a3 100644 --- a/ui/litellm-dashboard/src/components/add_model/ClassificationMethodConfig.tsx +++ b/ui/litellm-dashboard/src/components/add_model/ClassificationMethodConfig.tsx @@ -44,8 +44,9 @@ import { } from "./ComplexityRouterConfig"; const DEFAULT_SCORING_EXPLANATION = - "The router scores each request across 7 dimensions: token count, code presence, reasoning markers, technical " + - "terms, simple indicators, multi-step patterns, and question complexity. The weighted score determines the tier:"; + "The router scores each request across 7 built-in dimensions: token count, code presence, reasoning markers, technical " + + "terms, simple indicators, multi-step patterns, and question complexity, plus any custom dimensions you add. " + + "The weighted score determines the tier:"; const HEURISTIC_V2_EXPLANATION = "The router estimates success probability for all four tiers with the bundled calibrated model, then selects " + diff --git a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.test.tsx b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.test.tsx index 2970e14b335..0c12cc0ba1a 100644 --- a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.test.tsx +++ b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.test.tsx @@ -1521,14 +1521,16 @@ describe("ComplexityRouterConfig tier editing", () => { renderWithProviders(); fireEvent.click(screen.getByText("Advanced: Classification Method")); expect(screen.queryByText("How Classification Works")).not.toBeInTheDocument(); - expect(screen.queryByText("scores each request across 7 dimensions", { exact: false })).not.toBeInTheDocument(); + expect( + screen.queryByText("scores each request across 7 built-in dimensions", { exact: false }), + ).not.toBeInTheDocument(); }); it("keeps the scorer card on a built-in router, whose tiers the score still decides", () => { renderWithProviders(); fireEvent.click(screen.getByText("Advanced: Classification Method")); expect(screen.getByText("How Classification Works")).toBeInTheDocument(); - expect(screen.getByText("scores each request across 7 dimensions", { exact: false })).toBeInTheDocument(); + expect(screen.getByText("scores each request across 7 built-in dimensions", { exact: false })).toBeInTheDocument(); }); it("says why a custom row is blocked instead of only reddening its border", () => { diff --git a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx index d7df6ce33bb..de80e714e66 100644 --- a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx +++ b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx @@ -50,6 +50,7 @@ import EscalationKeywords from "./EscalationKeywords"; import KeywordTierRules, { KeywordTierRule } from "./KeywordTierRules"; import SemanticKeywordMatching from "./SemanticKeywordMatching"; import { type DimensionWeights, type TierBoundaries, type TokenThresholds } from "./heuristic_scoring_knobs"; +import { type CustomDimensionRow } from "./custom_dimensions"; import CompressionControls from "./CompressionControls"; import { type AutoRouterCompressionState, DEFAULT_AUTO_ROUTER_COMPRESSION } from "./buildAutoRouterCompression"; @@ -432,6 +433,11 @@ export interface ComplexityRouterConfigValue { tier_boundaries?: TierBoundaries; token_thresholds?: TokenThresholds; dimension_weights?: DimensionWeights; + /** + * Operator-added scoring dimensions, each carrying its own inline weight. Undefined means the router has + * none and keeps the key out of the payload; an empty array is a real "the last row was removed" state. + */ + custom_dimensions?: CustomDimensionRow[]; /** * Score floor the reasoning-marker override must clear. Undefined keeps the key out of the payload, so the * floor tracks tier_boundaries.simple_medium; an explicit 0 is a real floor that promotes on the markers alone. diff --git a/ui/litellm-dashboard/src/components/add_model/CustomDimensionRows.tsx b/ui/litellm-dashboard/src/components/add_model/CustomDimensionRows.tsx new file mode 100644 index 00000000000..34d8370aafb --- /dev/null +++ b/ui/litellm-dashboard/src/components/add_model/CustomDimensionRows.tsx @@ -0,0 +1,136 @@ +import { Trash2 } from "lucide-react"; +import { useState } from "react"; +import { Button } from "@/components/ui/button"; +import { Input } from "@/components/ui/input"; +import { Label } from "@/components/ui/label"; +import { Textarea } from "@/components/ui/textarea"; +import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"; +import { Slider } from "@/components/ui/slider"; +import type { CustomDimensionRow } from "./custom_dimensions"; + +const SCORING_MODES = [ + { value: "binary", label: "Binary" }, + { value: "match_count", label: "Match count" }, +] as const; + +interface Props { + rows: CustomDimensionRow[]; + disabled: boolean; + onChange: (rows: CustomDimensionRow[]) => void; + onWeight: (id: string, weight: number) => void; + onAdd: () => void; + onRemove: (id: string) => void; +} + +export default function CustomDimensionRows({ rows, disabled, onChange, onWeight, onAdd, onRemove }: Props) { + const [draft, setDraft] = useState<{ id: string; raw: string } | null>(null); + const update = (id: string, patch: Partial) => + onChange(rows.map((row) => (row.id === id ? { ...row, ...patch } : row))); + const editWeight = (id: string, raw: string) => { + setDraft({ id, raw }); + if (raw.trim() && Number.isFinite(Number(raw))) onWeight(id, Number(raw)); + }; + return ( +
+ {rows.map((row, index) => ( +
+ Custom dimension {index + 1} +
+ +
+
+ + update(row.id, { name: event.target.value })} + /> +
+
+ + onWeight(row.id, Array.isArray(value) ? value[0] : value)} + /> + setDraft(null)} + onChange={(event) => editWeight(row.id, event.target.value)} + /> +
+
+ {(["keywords", "patterns"] as const).map((field) => ( +
+ +