From 498c4254ead47876814b8a4f9b2fce485b943a37 Mon Sep 17 00:00:00 2001 From: Eric84626 Date: Sun, 7 Dec 2025 20:53:51 +0800 Subject: [PATCH 01/97] fix: Return 403 exception when calling GET responses api --- litellm/proxy/auth/auth_checks.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/litellm/proxy/auth/auth_checks.py b/litellm/proxy/auth/auth_checks.py index fc79a4d3591..309bd577606 100644 --- a/litellm/proxy/auth/auth_checks.py +++ b/litellm/proxy/auth/auth_checks.py @@ -402,13 +402,14 @@ def _allowed_routes_check(user_route: str, allowed_routes: list) -> bool: - user_route: str - the route the user is trying to call - allowed_routes: List[str|LiteLLMRoutes] - the list of allowed routes for the user. """ + from starlette.routing import compile_path for allowed_route in allowed_routes: - if ( - allowed_route in LiteLLMRoutes.__members__ - and user_route in LiteLLMRoutes[allowed_route].value - ): - return True + if allowed_route in LiteLLMRoutes.__members__: + for template in LiteLLMRoutes[allowed_route].value: + regex, _, _ = compile_path(template) + if regex.match(user_route): + return True elif allowed_route == user_route: return True return False From 4ab58619ad56d93eb4add67bfd6064c50f449fa1 Mon Sep 17 00:00:00 2001 From: Eric84626 Date: Sun, 14 Dec 2025 17:55:13 +0800 Subject: [PATCH 02/97] fix: added new step into rotate master key function for processing credentials table --- .../proxy/credential_endpoints/endpoints.py | 9 ++--- .../key_management_endpoints.py | 34 +++++++++++++++++++ 2 files changed, 39 insertions(+), 4 deletions(-) diff --git a/litellm/proxy/credential_endpoints/endpoints.py b/litellm/proxy/credential_endpoints/endpoints.py index 647abb73648..9f228bb1184 100644 --- a/litellm/proxy/credential_endpoints/endpoints.py +++ b/litellm/proxy/credential_endpoints/endpoints.py @@ -21,11 +21,11 @@ router = APIRouter() class CredentialHelperUtils: @staticmethod - def encrypt_credential_values(credential: CredentialItem) -> CredentialItem: + def encrypt_credential_values(credential: CredentialItem, new_encryption_key: Optional[str] = None) -> CredentialItem: """Encrypt values in credential.credential_values and add to DB""" encrypted_credential_values = {} for key, value in (credential.credential_values or {}).items(): - encrypted_credential_values[key] = encrypt_value_helper(value) + encrypted_credential_values[key] = encrypt_value_helper(value, new_encryption_key) # Return a new object to avoid mutating the caller's credential, which # is kept in memory and should remain unencrypted. @@ -246,7 +246,7 @@ async def delete_credential( def update_db_credential( - db_credential: CredentialItem, updated_patch: CredentialItem + db_credential: CredentialItem, updated_patch: CredentialItem, new_encryption_key: Optional[str] = None ) -> CredentialItem: """ Update a credential in the DB. @@ -258,7 +258,8 @@ def update_db_credential( ) encrypted_credential = CredentialHelperUtils.encrypt_credential_values( - updated_patch + updated_patch, + new_encryption_key, ) # update model name if encrypted_credential.credential_name: diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index da44bda791d..8ea3122ce01 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -2539,6 +2539,40 @@ async def _rotate_master_key( new_master_key=new_master_key, ) + # 5. process credentials table + try: + credentials = await prisma_client.db.litellm_credentialstable.find_many() + except Exception: + credentials = None + if credentials: + from litellm.proxy.credential_endpoints.endpoints import update_db_credential + + for cred in credentials: + try: + decrypted_cred = proxy_config.decrypt_credentials(cred) + encrypted_cred = update_db_credential( + db_credential=cred, + updated_patch=decrypted_cred, + new_encryption_key=new_master_key, + ) + credential_object_jsonified = jsonify_object(encrypted_cred.model_dump()) + await prisma_client.db.litellm_credentialstable.update( + where={"credential_name": cred.credential_name}, + data={ + **credential_object_jsonified, + "updated_by": user_api_key_dict.user_id, + }, + ) + except Exception as e: + verbose_proxy_logger.error( + f"Failed to re-encrypt credential {cred.credential_name}: {str(e)}" + ) + # Continue with next credential instead of failing entire rotation + continue + verbose_proxy_logger.debug( + f"Successfully re-encrypted {len(credentials)} credentials with new master key" + ) + def get_new_token(data: Optional[RegenerateKeyRequest]) -> str: if data and data.new_key is not None: From f4f5ea85dfe7eff7130724b1d7331354468f5ce8 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Thu, 18 Dec 2025 14:42:41 +0530 Subject: [PATCH 03/97] Add redisvl in requirements.txt --- requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index c36f94f0752..2fb6c52cfd3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,7 +7,8 @@ starlette==0.49.1 # starlette fastapi dep backoff==2.2.1 # server dep pyyaml==6.0.2 # server dep uvicorn==0.31.1 # server dep -gunicorn==23.0.0 # server dep +gunicorn==23.0.0 # server depredisvl +redisvl==0.4.1 # redis semantic cache fastuuid==0.13.5 # for uuid4 uvloop==0.21.0 # uvicorn dep, gives us much better performance under load boto3==1.36.0 # aws bedrock/sagemaker calls From 7ddab06bedba0d6151309c308a38cbfa13588dff Mon Sep 17 00:00:00 2001 From: Eric84626 Date: Sat, 20 Dec 2025 11:35:20 +0800 Subject: [PATCH 04/97] fix: fixed the issue of handling root paths when processing Discovery protected resource metadata and authorization server metadata URLs. --- .../mcp_server/discoverable_endpoints.py | 25 ++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py b/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py index ffa17a5b7c4..4b6020f582b 100644 --- a/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py +++ b/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py @@ -15,6 +15,7 @@ from litellm.proxy.common_utils.encrypt_decrypt_utils import ( ) from litellm.proxy.common_utils.http_parsing_utils import _read_request_body from litellm.types.mcp_server.mcp_server_manager import MCPServer +from litellm.proxy.utils import get_server_root_path router = APIRouter( tags=["mcp"], @@ -381,7 +382,18 @@ async def callback(code: str, state: str): # ------------------------------ # Optional .well-known endpoints for MCP + OAuth discovery # ------------------------------ -@router.get("/.well-known/oauth-protected-resource/{mcp_server_name}/mcp") +""" + Per SEP-985, the client MUST: + 1. Try resource_metadata from WWW-Authenticate header (if present) + 2. Fall back to path-based well-known URI: /.well-known/oauth-protected-resource/{path} + ( + If the resource identifier value contains a path or query component, any terminating slash (/) + following the host component MUST be removed before inserting /.well-known/ and the well-known + URI path suffix between the host component and the path(include root path) and/or query components. + https://datatracker.ietf.org/doc/html/rfc9728#section-3.1) + 3. Fall back to root-based well-known URI: /.well-known/oauth-protected-resource +""" +@router.get(f"/.well-known/oauth-protected-resource{'' if get_server_root_path() == '/' else get_server_root_path()}/{{mcp_server_name}}/mcp") @router.get("/.well-known/oauth-protected-resource") async def oauth_protected_resource_mcp( request: Request, mcp_server_name: Optional[str] = None @@ -403,8 +415,15 @@ async def oauth_protected_resource_mcp( ), # this is what Claude will call } - -@router.get("/.well-known/oauth-authorization-server/{mcp_server_name}") +""" + https://datatracker.ietf.org/doc/html/rfc8414#section-3.1 + RFC 8414: Path-aware OAuth discovery + If the issuer identifier value contains a path component, any + terminating "/" MUST be removed before inserting "/.well-known/" and + the well-known URI suffix between the host component and the path(include root path) + component. +""" +@router.get(f"/.well-known/oauth-authorization-server{'' if get_server_root_path() == '/' else get_server_root_path()}/{{mcp_server_name}}") @router.get("/.well-known/oauth-authorization-server") async def oauth_authorization_server_mcp( request: Request, mcp_server_name: Optional[str] = None From 3a2ab6b0d12be8863d2a7604a1f3e8ab5721b521 Mon Sep 17 00:00:00 2001 From: Eric84626 Date: Sat, 20 Dec 2025 11:55:12 +0800 Subject: [PATCH 05/97] fix: added additional grant type into oauth_authorization_server response for fixing mcp auth register bad request issue --- .../proxy/_experimental/mcp_server/discoverable_endpoints.py | 2 +- .../_experimental/mcp_server/test_discoverable_endpoints.py | 3 ++- ui/litellm-dashboard/src/hooks/useMcpOAuthFlow.tsx | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py b/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py index ffa17a5b7c4..5433196dfe3 100644 --- a/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py +++ b/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py @@ -428,7 +428,7 @@ async def oauth_authorization_server_mcp( "authorization_endpoint": authorization_endpoint, "token_endpoint": token_endpoint, "response_types_supported": ["code"], - "grant_types_supported": ["authorization_code"], + "grant_types_supported": ["authorization_code", "refresh_token"], "code_challenge_methods_supported": ["S256"], "token_endpoint_auth_methods_supported": ["client_secret_post"], # Claude expects a registration endpoint, even if we just fake it diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py index 6df9abd3fee..30f3d55f028 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py @@ -354,7 +354,7 @@ async def test_register_client_remote_registration_success(): request_payload = { "client_name": "Litellm Proxy", - "grant_types": ["authorization_code"], + "grant_types": ["authorization_code", "refresh_token"], "response_types": ["code"], "token_endpoint_auth_method": "client_secret_post", } @@ -603,6 +603,7 @@ async def test_oauth_authorization_server_respects_x_forwarded_proto(): assert response["authorization_endpoint"].startswith("https://litellm.example.com/") assert response["token_endpoint"].startswith("https://litellm.example.com/") assert response["registration_endpoint"].startswith("https://litellm.example.com/") + assert response["grant_types_supported"] == ["authorization_code", "refresh_token"] @pytest.mark.asyncio diff --git a/ui/litellm-dashboard/src/hooks/useMcpOAuthFlow.tsx b/ui/litellm-dashboard/src/hooks/useMcpOAuthFlow.tsx index 9600c962564..a62d8baa75b 100644 --- a/ui/litellm-dashboard/src/hooks/useMcpOAuthFlow.tsx +++ b/ui/litellm-dashboard/src/hooks/useMcpOAuthFlow.tsx @@ -136,7 +136,7 @@ export const useMcpOAuthFlow = ({ if (!hasPreconfiguredCredentials) { const registration = await registerMcpOAuthClient(accessToken, serverId, { client_name: temporaryPayload.alias || temporaryPayload.server_name || serverId, - grant_types: ["authorization_code"], + grant_types: ["authorization_code", "refresh_token"], response_types: ["code"], token_endpoint_auth_method: temporaryPayload.credentials && temporaryPayload.credentials.client_secret ? "client_secret_post" : "none", From 684fba42eaaf6a4d47795e56fd668b8d46b01525 Mon Sep 17 00:00:00 2001 From: Eric84626 Date: Sat, 20 Dec 2025 13:22:29 +0800 Subject: [PATCH 06/97] fix: added RFC RECOMMENDED property(scopes_supported) to protected resource and authorization server metadata --- .../mcp_server/discoverable_endpoints.py | 17 +++++- .../mcp_server/test_discoverable_endpoints.py | 54 ++++++++++++++++++- 2 files changed, 68 insertions(+), 3 deletions(-) diff --git a/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py b/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py index d6fe3f2b9cf..ded591a8f53 100644 --- a/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py +++ b/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py @@ -398,8 +398,14 @@ async def callback(code: str, state: str): async def oauth_protected_resource_mcp( request: Request, mcp_server_name: Optional[str] = None ): + from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + global_mcp_server_manager, + ) # Get the correct base URL considering X-Forwarded-* headers request_base_url = get_request_base_url(request) + mcp_server: Optional[MCPServer] = None + if mcp_server_name: + mcp_server = global_mcp_server_manager.get_mcp_server_by_name(mcp_server_name) return { "authorization_servers": [ ( @@ -413,6 +419,7 @@ async def oauth_protected_resource_mcp( if mcp_server_name else f"{request_base_url}/mcp" ), # this is what Claude will call + "scopes_supported": mcp_server.scopes if mcp_server else [], } """ @@ -428,6 +435,9 @@ async def oauth_protected_resource_mcp( async def oauth_authorization_server_mcp( request: Request, mcp_server_name: Optional[str] = None ): + from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + global_mcp_server_manager, + ) # Get the correct base URL considering X-Forwarded-* headers request_base_url = get_request_base_url(request) @@ -442,16 +452,21 @@ async def oauth_authorization_server_mcp( else f"{request_base_url}/token" ) + mcp_server: Optional[MCPServer] = None + if mcp_server_name: + mcp_server = global_mcp_server_manager.get_mcp_server_by_name(mcp_server_name) + return { "issuer": request_base_url, # point to your proxy "authorization_endpoint": authorization_endpoint, "token_endpoint": token_endpoint, "response_types_supported": ["code"], + "scopes_supported": mcp_server.scopes if mcp_server else [], "grant_types_supported": ["authorization_code", "refresh_token"], "code_challenge_methods_supported": ["S256"], "token_endpoint_auth_methods_supported": ["client_secret_post"], # Claude expects a registration endpoint, even if we just fake it - "registration_endpoint": f"{request_base_url}/{mcp_server_name}/register", + "registration_endpoint": f"{request_base_url}/{mcp_server_name}/register" if mcp_server_name else f"{request_base_url}/register", } diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py index 30f3d55f028..4c5723b8284 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py @@ -556,9 +556,33 @@ async def test_oauth_protected_resource_respects_x_forwarded_proto(): from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( oauth_protected_resource_mcp, ) + from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + global_mcp_server_manager, + ) + from litellm.types.mcp import MCPAuth + from litellm.types.mcp_server.mcp_server_manager import MCPServer + from litellm.proxy._types import MCPTransport from fastapi import Request except ImportError: pytest.skip("MCP discoverable endpoints not available") + # Clear registry + global_mcp_server_manager.registry.clear() + + # Create mock OAuth2 server + oauth2_server = MCPServer( + server_id="test_oauth_server", + name="test_oauth", + server_name="test_oauth", + alias="test_oauth", + transport=MCPTransport.http, + auth_type=MCPAuth.oauth2, + client_id="test_client_id", + client_secret="test_client_secret", + authorization_url="https://provider.com/oauth/authorize", + token_url="https://provider.com/oauth/token", + scopes=["read", "write"], + ) + global_mcp_server_manager.registry[oauth2_server.server_id] = oauth2_server # Mock request with http base_url but X-Forwarded-Proto: https mock_request = MagicMock(spec=Request) @@ -568,13 +592,14 @@ async def test_oauth_protected_resource_respects_x_forwarded_proto(): # Call the endpoint response = await oauth_protected_resource_mcp( request=mock_request, - mcp_server_name="test_server", + mcp_server_name="test_oauth", ) # Verify response uses HTTPS URLs assert response["authorization_servers"][0].startswith( "https://litellm.example.com/" ) + assert response["scopes_supported"] == oauth2_server.scopes @pytest.mark.asyncio @@ -584,9 +609,33 @@ async def test_oauth_authorization_server_respects_x_forwarded_proto(): from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( oauth_authorization_server_mcp, ) + from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + global_mcp_server_manager, + ) + from litellm.types.mcp import MCPAuth + from litellm.types.mcp_server.mcp_server_manager import MCPServer + from litellm.proxy._types import MCPTransport from fastapi import Request except ImportError: pytest.skip("MCP discoverable endpoints not available") + # Clear registry + global_mcp_server_manager.registry.clear() + + # Create mock OAuth2 server + oauth2_server = MCPServer( + server_id="test_oauth_server", + name="test_oauth", + server_name="test_oauth", + alias="test_oauth", + transport=MCPTransport.http, + auth_type=MCPAuth.oauth2, + client_id="test_client_id", + client_secret="test_client_secret", + authorization_url="https://provider.com/oauth/authorize", + token_url="https://provider.com/oauth/token", + scopes=["read", "write"], + ) + global_mcp_server_manager.registry[oauth2_server.server_id] = oauth2_server # Mock request with http base_url but X-Forwarded-Proto: https mock_request = MagicMock(spec=Request) @@ -596,7 +645,7 @@ async def test_oauth_authorization_server_respects_x_forwarded_proto(): # Call the endpoint response = await oauth_authorization_server_mcp( request=mock_request, - mcp_server_name="test_server", + mcp_server_name="test_oauth", ) # Verify response uses HTTPS URLs @@ -604,6 +653,7 @@ async def test_oauth_authorization_server_respects_x_forwarded_proto(): assert response["token_endpoint"].startswith("https://litellm.example.com/") assert response["registration_endpoint"].startswith("https://litellm.example.com/") assert response["grant_types_supported"] == ["authorization_code", "refresh_token"] + assert response["scopes_supported"] == oauth2_server.scopes @pytest.mark.asyncio From 0306f02e74d7fff462fb727639a60e5ae11d64e4 Mon Sep 17 00:00:00 2001 From: Eric84626 Date: Sat, 20 Dec 2025 14:13:18 +0800 Subject: [PATCH 07/97] fix: removed initialize the tool name to MCP server name mapping(oauth2) on startup for avoiding 401 error --- .../mcp_server/mcp_server_manager.py | 3 +++ .../mcp_server/test_mcp_server_manager.py | 19 +++++++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py index 8c9d8630457..c2215efe9d0 100644 --- a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py +++ b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py @@ -1913,6 +1913,9 @@ class MCPServerManager: Note: This now handles prefixed tool names """ for server in self.get_registry().values(): + if server.auth_type == MCPAuth.oauth2: + # Skip OAuth2 servers for now as they may require user-specific tokens + continue tools = await self._get_tools_from_server(server) for tool in tools: # The tool.name here is already prefixed from _get_tools_from_server diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py index 7a6e5ad17f6..c0ded9c728c 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py @@ -536,7 +536,26 @@ class TestMCPServerManager: assert ( server.registration_url == "https://discovered.example.com/register" ) + @pytest.mark.asyncio + async def test_config_oauth_initialize_tool_name_to_mcp_server_name_mapping(self): + manager = MCPServerManager() + config = { + "example": { + "url": "https://example.com/mcp", + "transport": MCPTransport.http, + "auth_type": MCPAuth.oauth2, + "scopes": ["config"], + "authorization_url": "https://config.example.com/auth", + } + } + + await manager.load_servers_from_config(config) + + # Initialize the tool mapping + await manager._initialize_tool_name_to_mcp_server_name_mapping() + assert manager.tool_name_to_mcp_server_name_mapping == {} + @pytest.mark.asyncio async def test_list_tools_handles_missing_server_alias(self): """Test that list_tools handles servers without alias gracefully""" From 36a369a747fccedb87e12cf26996aebfc221f57e Mon Sep 17 00:00:00 2001 From: Eric84626 Date: Sat, 20 Dec 2025 14:28:52 +0800 Subject: [PATCH 08/97] fix: upgraded mcp sdk depency version for fixing ClosedResourceError --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index f222acc46e6..cb12a658814 100644 --- a/requirements.txt +++ b/requirements.txt @@ -20,7 +20,7 @@ google-cloud-aiplatform==1.47.0 # for vertex ai calls google-cloud-iam==2.19.1 # for GCP IAM Redis authentication google-genai==1.22.0 anthropic[vertex]==0.54.0 -mcp==1.21.2 ; python_version >= "3.10" # for MCP server +mcp==1.25.0 ; python_version >= "3.10" # for MCP server google-generativeai==0.5.0 # for vertex ai calls async_generator==1.10.0 # for async ollama calls langfuse==2.59.7 # for langfuse self-hosted logging From 78693bb9d065924c2b4fd26962aeedc8eb84f208 Mon Sep 17 00:00:00 2001 From: mangabits <1457532+mangabits@users.noreply.github.com> Date: Fri, 19 Dec 2025 18:06:02 -0800 Subject: [PATCH 09/97] Use already configured opentelemetry providers Users that instrument using opentelemetry-instrument can now setup exporters as per their environment. --- litellm/integrations/opentelemetry.py | 152 ++++++++++++------ .../test_opentelemetry_unit_tests.py | 25 --- .../integrations/test_opentelemetry.py | 92 +++++++++++ 3 files changed, 192 insertions(+), 77 deletions(-) diff --git a/litellm/integrations/opentelemetry.py b/litellm/integrations/opentelemetry.py index 12e60bc25bb..c874c799d5e 100644 --- a/litellm/integrations/opentelemetry.py +++ b/litellm/integrations/opentelemetry.py @@ -196,50 +196,87 @@ class OpenTelemetry(CustomLogger): litellm.service_callback.append(self) setattr(proxy_server, "open_telemetry_logger", self) + def _get_or_create_provider( + self, + provider, + provider_name: str, + get_existing_provider_fn, + sdk_provider_class, + create_new_provider_fn, + set_provider_fn, + ): + """ + Generic helper to get or create an OpenTelemetry provider (Tracer, Meter, or Logger). + + Args: + provider: The provider instance passed to the init function (can be None) + provider_name: Name for logging (e.g., "TracerProvider") + get_existing_provider_fn: Function to get the existing global provider + sdk_provider_class: The SDK provider class to check for (e.g., TracerProvider from SDK) + create_new_provider_fn: Function to create a new provider instance + set_provider_fn: Function to set the provider globally + + Returns: + The provider to use (either existing, new, or explicitly provided) + """ + if provider is None: + # Check if a provider is already set globally + try: + existing_provider = get_existing_provider_fn() + + # If a real SDK provider exists (set by another SDK like Langfuse), use it + # This uses a positive check for SDK providers instead of a negative check for proxy providers + if isinstance(existing_provider, sdk_provider_class): + verbose_logger.debug( + "OpenTelemetry: Using existing %s: %s", + provider_name, + type(existing_provider).__name__, + ) + provider = existing_provider + # Don't call set_provider to preserve existing context + else: + # Default proxy provider or unknown type, create our own + verbose_logger.debug("OpenTelemetry: Creating new %s", provider_name) + provider = create_new_provider_fn() + set_provider_fn(provider) + except Exception as e: + # Fallback: create a new provider if something goes wrong + verbose_logger.debug( + "OpenTelemetry: Exception checking existing %s, creating new one: %s", + provider_name, + str(e), + ) + provider = create_new_provider_fn() + set_provider_fn(provider) + else: + # Provider explicitly provided (e.g., for testing) + # Do NOT call set_provider_fn - the caller is responsible for managing global state + # If they want it to be global, they've already set it before passing it to us + verbose_logger.debug( + "OpenTelemetry: Using provided TracerProvider: %s", + type(provider).__name__, + ) + + return provider + def _init_tracing(self, tracer_provider): from opentelemetry import trace from opentelemetry.sdk.trace import TracerProvider from opentelemetry.trace import SpanKind - # use provided tracer or create a new one - if tracer_provider is None: - # Check if a TracerProvider is already set globally (e.g., by Langfuse SDK) - try: - from opentelemetry.trace import ProxyTracerProvider + def create_tracer_provider(): + provider = TracerProvider(resource=_get_litellm_resource()) + provider.add_span_processor(self._get_span_processor()) + return provider - existing_provider = trace.get_tracer_provider() - - # If an actual provider exists (not the default proxy), use it - if not isinstance(existing_provider, ProxyTracerProvider): - verbose_logger.debug( - "OpenTelemetry: Using existing TracerProvider: %s", - type(existing_provider).__name__, - ) - tracer_provider = existing_provider - # Don't call set_tracer_provider to preserve existing context - else: - # No real provider exists yet, create our own - verbose_logger.debug("OpenTelemetry: Creating new TracerProvider") - tracer_provider = TracerProvider(resource=_get_litellm_resource()) - tracer_provider.add_span_processor(self._get_span_processor()) - trace.set_tracer_provider(tracer_provider) - except Exception as e: - # Fallback: create a new provider if something goes wrong - verbose_logger.debug( - "OpenTelemetry: Exception checking existing provider, creating new one: %s", - str(e), - ) - tracer_provider = TracerProvider(resource=_get_litellm_resource()) - tracer_provider.add_span_processor(self._get_span_processor()) - trace.set_tracer_provider(tracer_provider) - else: - # Tracer provider explicitly provided (e.g., for testing) - # Do NOT call set_tracer_provider - the caller is responsible for managing global state - # If they want it to be global, they've already set it before passing it to us - verbose_logger.debug( - "OpenTelemetry: Using provided TracerProvider: %s", - type(tracer_provider).__name__, - ) + tracer_provider = self._get_or_create_provider( + provider=tracer_provider, + provider_name="TracerProvider", + get_existing_provider_fn=trace.get_tracer_provider, + sdk_provider_class=TracerProvider, + create_new_provider_fn=create_tracer_provider, + set_provider_fn=trace.set_tracer_provider, + ) # Grab our tracer from the TracerProvider (not from global context) # This ensures we use the provided TracerProvider (e.g., for testing) @@ -259,8 +296,7 @@ class OpenTelemetry(CustomLogger): from opentelemetry import metrics from opentelemetry.sdk.metrics import Histogram, MeterProvider - # Only create OTLP infrastructure if no custom meter provider is provided - if meter_provider is None: + def create_meter_provider(): from opentelemetry.exporter.otlp.proto.grpc.metric_exporter import ( OTLPMetricExporter, ) @@ -281,15 +317,20 @@ class OpenTelemetry(CustomLogger): _metric_exporter, export_interval_millis=10000 ) - meter_provider = MeterProvider( + return MeterProvider( metric_readers=[_metric_reader], resource=_get_litellm_resource() ) - meter = meter_provider.get_meter(__name__) - else: - # Use the provided meter provider as-is, without creating additional OTLP infrastructure - meter = meter_provider.get_meter(__name__) - metrics.set_meter_provider(meter_provider) + meter_provider = self._get_or_create_provider( + provider=meter_provider, + provider_name="MeterProvider", + get_existing_provider_fn=metrics.get_meter_provider, + sdk_provider_class=MeterProvider, + create_new_provider_fn=create_meter_provider, + set_provider_fn=metrics.set_meter_provider, + ) + + meter = meter_provider.get_meter(__name__) self._operation_duration_histogram = meter.create_histogram( name="gen_ai.client.operation.duration", # Replace with semconv constant in otel 1.38 @@ -327,22 +368,29 @@ class OpenTelemetry(CustomLogger): if not self.config.enable_events: return - from opentelemetry._logs import set_logger_provider + from opentelemetry._logs import get_logger_provider, set_logger_provider from opentelemetry.sdk._logs import LoggerProvider as OTLoggerProvider from opentelemetry.sdk._logs.export import BatchLogRecordProcessor - # set up log pipeline - if logger_provider is None: + def create_logger_provider(): litellm_resource = _get_litellm_resource() - logger_provider = OTLoggerProvider(resource=litellm_resource) + provider = OTLoggerProvider(resource=litellm_resource) # Only add OTLP exporter if we created the logger provider ourselves log_exporter = self._get_log_exporter() if log_exporter: - logger_provider.add_log_record_processor( + provider.add_log_record_processor( BatchLogRecordProcessor(log_exporter) # type: ignore[arg-type] ) + return provider - set_logger_provider(logger_provider) + logger_provider = self._get_or_create_provider( + provider=logger_provider, + provider_name="LoggerProvider", + get_existing_provider_fn=get_logger_provider, + sdk_provider_class=OTLoggerProvider, + create_new_provider_fn=create_logger_provider, + set_provider_fn=set_logger_provider, + ) def log_success_event(self, kwargs, response_obj, start_time, end_time): self._handle_success(kwargs, response_obj, start_time, end_time) diff --git a/tests/logging_callback_tests/test_opentelemetry_unit_tests.py b/tests/logging_callback_tests/test_opentelemetry_unit_tests.py index 3d0682d9033..04f8abe64de 100644 --- a/tests/logging_callback_tests/test_opentelemetry_unit_tests.py +++ b/tests/logging_callback_tests/test_opentelemetry_unit_tests.py @@ -65,31 +65,6 @@ class TestOpentelemetryUnitTests(BaseLoggingCallbackTest): # External spans should only be closed by their creators parent_otel_span.end.assert_not_called() - def test_init_tracing_respects_existing_tracer_provider(self): - """ - Unit test: _init_tracing() should respect existing TracerProvider. - - When a TracerProvider already exists (e.g., set by Langfuse SDK), - LiteLLM should use it instead of creating a new one. - """ - from opentelemetry import trace - from opentelemetry.sdk.trace import TracerProvider - from litellm.integrations.opentelemetry import OpenTelemetry - - # Setup: Create and set an existing TracerProvider - tracer_provider = TracerProvider() - trace.set_tracer_provider(tracer_provider) - existing_provider = trace.get_tracer_provider() - - # Act: Initialize OpenTelemetry integration (should detect existing provider) - otel_integration = OpenTelemetry() - - # Assert: The existing provider should still be active - current_provider = trace.get_tracer_provider() - assert current_provider is existing_provider, ( - "Existing TracerProvider should be respected and not overridden" - ) - def test_get_span_context_detects_active_span(self): """ Unit test: _get_span_context() should auto-detect active spans from global context. diff --git a/tests/test_litellm/integrations/test_opentelemetry.py b/tests/test_litellm/integrations/test_opentelemetry.py index 5d648f601f6..bd7d4bf2179 100644 --- a/tests/test_litellm/integrations/test_opentelemetry.py +++ b/tests/test_litellm/integrations/test_opentelemetry.py @@ -172,6 +172,98 @@ class TestOpenTelemetryCostBreakdown(unittest.TestCase): assert ("gen_ai.cost.original_cost", 0.004) not in call_args_list +class TestOpenTelemetryProviderInitialization(unittest.TestCase): + """Test suite for verifying provider initialization respects existing providers""" + + def test_init_tracing_respects_existing_tracer_provider(self): + """ + Unit test: _init_tracing() should respect existing TracerProvider. + + When a TracerProvider already exists (e.g., set by Langfuse SDK), + LiteLLM should use it instead of creating a new one. + """ + from opentelemetry import trace + from opentelemetry.sdk.trace import TracerProvider + + # Setup: Create and set an existing TracerProvider + tracer_provider = TracerProvider() + trace.set_tracer_provider(tracer_provider) + existing_provider = trace.get_tracer_provider() + + # Act: Initialize OpenTelemetry integration (should detect existing provider) + otel_integration = OpenTelemetry() + + # Assert: The existing provider should still be active + current_provider = trace.get_tracer_provider() + assert current_provider is existing_provider, ( + "Existing TracerProvider should be respected and not overridden" + ) + + def test_init_metrics_respects_existing_meter_provider(self): + """ + Unit test: _init_metrics() should respect existing MeterProvider. + + When a MeterProvider already exists (e.g., set by Langfuse SDK), + LiteLLM should use it instead of creating a new one. + """ + from opentelemetry import metrics + from opentelemetry.sdk.metrics import MeterProvider + + # Setup: Enable metrics for this test + os.environ["LITELLM_OTEL_INTEGRATION_ENABLE_METRICS"] = "true" + + try: + # Create and set an existing MeterProvider + meter_provider = MeterProvider() + metrics.set_meter_provider(meter_provider) + existing_provider = metrics.get_meter_provider() + + # Act: Initialize OpenTelemetry integration (should detect existing provider) + config = OpenTelemetryConfig.from_env() + otel_integration = OpenTelemetry(config=config) + + # Assert: The existing provider should still be active + current_provider = metrics.get_meter_provider() + assert current_provider is existing_provider, ( + "Existing MeterProvider should be respected and not overridden" + ) + finally: + # Cleanup + os.environ.pop("LITELLM_OTEL_INTEGRATION_ENABLE_METRICS", None) + + def test_init_logs_respects_existing_logger_provider(self): + """ + Unit test: _init_logs() should respect existing LoggerProvider. + + When a LoggerProvider already exists (e.g., set by Langfuse SDK), + LiteLLM should use it instead of creating a new one. + """ + from opentelemetry._logs import get_logger_provider, set_logger_provider + from opentelemetry.sdk._logs import LoggerProvider as OTLoggerProvider + + # Setup: Enable events for this test + os.environ["LITELLM_OTEL_INTEGRATION_ENABLE_EVENTS"] = "true" + + try: + # Create and set an existing LoggerProvider + logger_provider = OTLoggerProvider() + set_logger_provider(logger_provider) + existing_provider = get_logger_provider() + + # Act: Initialize OpenTelemetry integration (should detect existing provider) + config = OpenTelemetryConfig.from_env() + otel_integration = OpenTelemetry(config=config) + + # Assert: The existing provider should still be active + current_provider = get_logger_provider() + assert current_provider is existing_provider, ( + "Existing LoggerProvider should be respected and not overridden" + ) + finally: + # Cleanup + os.environ.pop("LITELLM_OTEL_INTEGRATION_ENABLE_EVENTS", None) + + class TestOpenTelemetry(unittest.TestCase): POLL_INTERVAL = 0.05 POLL_TIMEOUT = 2.0 From 9ca043f1d85020e779bb2d907683ce816534d02c Mon Sep 17 00:00:00 2001 From: mangabits <1457532+mangabits@users.noreply.github.com> Date: Mon, 29 Dec 2025 17:21:28 -0800 Subject: [PATCH 10/97] Handle all protocols for all telemetry --- litellm/integrations/opentelemetry.py | 107 ++++++++++++++++++++------ 1 file changed, 82 insertions(+), 25 deletions(-) diff --git a/litellm/integrations/opentelemetry.py b/litellm/integrations/opentelemetry.py index c874c799d5e..b356e41c152 100644 --- a/litellm/integrations/opentelemetry.py +++ b/litellm/integrations/opentelemetry.py @@ -294,32 +294,18 @@ class OpenTelemetry(CustomLogger): return from opentelemetry import metrics - from opentelemetry.sdk.metrics import Histogram, MeterProvider + from opentelemetry.sdk.metrics import MeterProvider def create_meter_provider(): - from opentelemetry.exporter.otlp.proto.grpc.metric_exporter import ( - OTLPMetricExporter, - ) - from opentelemetry.sdk.metrics.export import ( - AggregationTemporality, - PeriodicExportingMetricReader, - ) - - normalized_endpoint = self._normalize_otel_endpoint( - self.config.endpoint, "metrics" - ) - _metric_exporter = OTLPMetricExporter( - endpoint=normalized_endpoint, - headers=OpenTelemetry._get_headers_dictionary(self.config.headers), - preferred_temporality={Histogram: AggregationTemporality.DELTA}, - ) - _metric_reader = PeriodicExportingMetricReader( - _metric_exporter, export_interval_millis=10000 - ) - - return MeterProvider( - metric_readers=[_metric_reader], resource=_get_litellm_resource() + metric_reader = self._get_metric_reader() + if metric_reader: + return MeterProvider( + metric_readers=[metric_reader], resource=_get_litellm_resource() + ) + verbose_logger.warning( + "OpenTelemetry: No metric reader created. Metrics will not be exported." ) + return MeterProvider(resource=_get_litellm_resource()) meter_provider = self._get_or_create_provider( provider=meter_provider, @@ -383,7 +369,7 @@ class OpenTelemetry(CustomLogger): ) return provider - logger_provider = self._get_or_create_provider( + self._get_or_create_provider( provider=logger_provider, provider_name="LoggerProvider", get_existing_provider_fn=get_logger_provider, @@ -992,6 +978,15 @@ class OpenTelemetry(CustomLogger): if not self.config.enable_events: return + # NOTE: Semantic logs (gen_ai.content.prompt/completion events) have compatibility issues + # with OTEL SDK >= 1.39.0 due to breaking changes in PR #4676: + # - LogRecord moved from opentelemetry.sdk._logs to opentelemetry.sdk._logs._internal + # - LogRecord constructor no longer accepts 'resource' parameter (now inherited from LoggerProvider) + # - LogData class was removed entirely + # These logs work correctly in OTEL SDK < 1.39.0 but may fail in >= 1.39.0. + # See: https://github.com/open-telemetry/opentelemetry-python/pull/4676 + # TODO: Refactor to use the proper OTEL Logs API instead of directly creating SDK LogRecords + from opentelemetry._logs import SeverityNumber, get_logger, get_logger_provider from opentelemetry.sdk._logs import LogRecord as SdkLogRecord @@ -1855,7 +1850,8 @@ class OpenTelemetry(CustomLogger): ) return self.OTEL_EXPORTER - if self.OTEL_EXPORTER == "console": + otel_logs_exporter = os.getenv("OTEL_LOGS_EXPORTER") + if self.OTEL_EXPORTER == "console" or otel_logs_exporter == "console": from opentelemetry.sdk._logs.export import ConsoleLogExporter verbose_logger.debug( @@ -1902,6 +1898,67 @@ class OpenTelemetry(CustomLogger): return ConsoleLogExporter() + def _get_metric_reader(self): + """ + Get the appropriate metric reader based on the configuration. + """ + from opentelemetry.sdk.metrics import Histogram + from opentelemetry.sdk.metrics.export import ( + AggregationTemporality, + ConsoleMetricExporter, + PeriodicExportingMetricReader, + ) + + verbose_logger.debug( + "OpenTelemetry Logger, initializing metric reader\nself.OTEL_EXPORTER: %s\nself.OTEL_ENDPOINT: %s\nself.OTEL_HEADERS: %s", + self.OTEL_EXPORTER, + self.OTEL_ENDPOINT, + self.OTEL_HEADERS, + ) + + _split_otel_headers = OpenTelemetry._get_headers_dictionary(self.OTEL_HEADERS) + normalized_endpoint = self._normalize_otel_endpoint(self.OTEL_ENDPOINT, "metrics") + + if self.OTEL_EXPORTER == "console": + exporter = ConsoleMetricExporter() + return PeriodicExportingMetricReader(exporter, export_interval_millis=5000) + + elif ( + self.OTEL_EXPORTER == "otlp_http" + or self.OTEL_EXPORTER == "http/protobuf" + or self.OTEL_EXPORTER == "http/json" + ): + from opentelemetry.exporter.otlp.proto.http.metric_exporter import ( + OTLPMetricExporter, + ) + + exporter = OTLPMetricExporter( + endpoint=normalized_endpoint, + headers=_split_otel_headers, + preferred_temporality={Histogram: AggregationTemporality.DELTA}, + ) + return PeriodicExportingMetricReader(exporter, export_interval_millis=5000) + + elif self.OTEL_EXPORTER == "otlp_grpc" or self.OTEL_EXPORTER == "grpc": + from opentelemetry.exporter.otlp.proto.grpc.metric_exporter import ( + OTLPMetricExporter, + ) + + exporter = OTLPMetricExporter( + endpoint=normalized_endpoint, + headers=_split_otel_headers, + preferred_temporality={Histogram: AggregationTemporality.DELTA}, + ) + return PeriodicExportingMetricReader(exporter, export_interval_millis=5000) + + else: + verbose_logger.warning( + "OpenTelemetry: Unknown metric exporter '%s', defaulting to console. Supported: console, otlp_http, otlp_grpc", + self.OTEL_EXPORTER, + ) + exporter = ConsoleMetricExporter() + return PeriodicExportingMetricReader(exporter, export_interval_millis=5000) + def _normalize_otel_endpoint( self, endpoint: Optional[str], signal_type: str ) -> Optional[str]: From 4a1bc90e61a47512e7355ace5f246e89672f2cdd Mon Sep 17 00:00:00 2001 From: mangabits <1457532+mangabits@users.noreply.github.com> Date: Tue, 30 Dec 2025 18:11:54 -0800 Subject: [PATCH 11/97] Add more tests --- .../integrations/test_opentelemetry.py | 87 ++++++++++--------- 1 file changed, 44 insertions(+), 43 deletions(-) diff --git a/tests/test_litellm/integrations/test_opentelemetry.py b/tests/test_litellm/integrations/test_opentelemetry.py index bd7d4bf2179..6c17570e135 100644 --- a/tests/test_litellm/integrations/test_opentelemetry.py +++ b/tests/test_litellm/integrations/test_opentelemetry.py @@ -199,6 +199,7 @@ class TestOpenTelemetryProviderInitialization(unittest.TestCase): "Existing TracerProvider should be respected and not overridden" ) + @patch.dict(os.environ, {"LITELLM_OTEL_INTEGRATION_ENABLE_METRICS": "true"}, clear=True) def test_init_metrics_respects_existing_meter_provider(self): """ Unit test: _init_metrics() should respect existing MeterProvider. @@ -209,28 +210,22 @@ class TestOpenTelemetryProviderInitialization(unittest.TestCase): from opentelemetry import metrics from opentelemetry.sdk.metrics import MeterProvider - # Setup: Enable metrics for this test - os.environ["LITELLM_OTEL_INTEGRATION_ENABLE_METRICS"] = "true" + # Create and set an existing MeterProvider + meter_provider = MeterProvider() + metrics.set_meter_provider(meter_provider) + existing_provider = metrics.get_meter_provider() - try: - # Create and set an existing MeterProvider - meter_provider = MeterProvider() - metrics.set_meter_provider(meter_provider) - existing_provider = metrics.get_meter_provider() + # Act: Initialize OpenTelemetry integration (should detect existing provider) + config = OpenTelemetryConfig.from_env() + otel_integration = OpenTelemetry(config=config) - # Act: Initialize OpenTelemetry integration (should detect existing provider) - config = OpenTelemetryConfig.from_env() - otel_integration = OpenTelemetry(config=config) - - # Assert: The existing provider should still be active - current_provider = metrics.get_meter_provider() - assert current_provider is existing_provider, ( - "Existing MeterProvider should be respected and not overridden" - ) - finally: - # Cleanup - os.environ.pop("LITELLM_OTEL_INTEGRATION_ENABLE_METRICS", None) + # Assert: The existing provider should still be active + current_provider = metrics.get_meter_provider() + assert current_provider is existing_provider, ( + "Existing MeterProvider should be respected and not overridden" + ) + @patch.dict(os.environ, {"LITELLM_OTEL_INTEGRATION_ENABLE_EVENTS": "true"}, clear=True) def test_init_logs_respects_existing_logger_provider(self): """ Unit test: _init_logs() should respect existing LoggerProvider. @@ -241,27 +236,20 @@ class TestOpenTelemetryProviderInitialization(unittest.TestCase): from opentelemetry._logs import get_logger_provider, set_logger_provider from opentelemetry.sdk._logs import LoggerProvider as OTLoggerProvider - # Setup: Enable events for this test - os.environ["LITELLM_OTEL_INTEGRATION_ENABLE_EVENTS"] = "true" + # Create and set an existing LoggerProvider + logger_provider = OTLoggerProvider() + set_logger_provider(logger_provider) + existing_provider = get_logger_provider() - try: - # Create and set an existing LoggerProvider - logger_provider = OTLoggerProvider() - set_logger_provider(logger_provider) - existing_provider = get_logger_provider() + # Act: Initialize OpenTelemetry integration (should detect existing provider) + config = OpenTelemetryConfig.from_env() + otel_integration = OpenTelemetry(config=config) - # Act: Initialize OpenTelemetry integration (should detect existing provider) - config = OpenTelemetryConfig.from_env() - otel_integration = OpenTelemetry(config=config) - - # Assert: The existing provider should still be active - current_provider = get_logger_provider() - assert current_provider is existing_provider, ( - "Existing LoggerProvider should be respected and not overridden" - ) - finally: - # Cleanup - os.environ.pop("LITELLM_OTEL_INTEGRATION_ENABLE_EVENTS", None) + # Assert: The existing provider should still be active + current_provider = get_logger_provider() + assert current_provider is existing_provider, ( + "Existing LoggerProvider should be respected and not overridden" + ) class TestOpenTelemetry(unittest.TestCase): @@ -712,7 +700,6 @@ class TestOpenTelemetry(unittest.TestCase): self.assertEqual(attributes.get("extra.attr"), "extra-value") - def test_handle_success_spans_only(self): # make sure neither events nor metrics is on os.environ.pop("LITELLM_OTEL_INTEGRATION_ENABLE_EVENTS", None) @@ -779,11 +766,8 @@ class TestOpenTelemetry(unittest.TestCase): logs = log_exporter.get_finished_logs() self.assertFalse(logs, "Did not expect any logs") + @patch.dict(os.environ, {"LITELLM_OTEL_INTEGRATION_ENABLE_METRICS": "true"}, clear=True) def test_handle_success_spans_and_metrics(self): - # only metrics on - os.environ.pop("LITELLM_OTEL_INTEGRATION_ENABLE_EVENTS", None) - os.environ["LITELLM_OTEL_INTEGRATION_ENABLE_METRICS"] = "true" - # ─── build in‐memory OTEL providers/exporters ───────────────────────────── span_exporter = InMemorySpanExporter() tracer_provider = TracerProvider() @@ -1412,6 +1396,23 @@ class TestOpenTelemetryProtocolSelection(unittest.TestCase): ) self.assertEqual(normalized, "http://collector:4317/v1/logs") + def test_get_metric_reader_uses_http_exporter_for_http_protobuf(self): + """Test that http/protobuf protocol uses OTLPMetricExporterHTTP""" + from opentelemetry.exporter.otlp.proto.http.metric_exporter import ( + OTLPMetricExporter, + ) + from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader + + config = OpenTelemetryConfig( + exporter="http/protobuf", endpoint="http://collector:4318" + ) + otel = OpenTelemetry(config=config) + + reader = otel._get_metric_reader() + + self.assertIsInstance(reader, PeriodicExportingMetricReader) + self.assertIsInstance(reader._exporter, OTLPMetricExporter) + class TestOpenTelemetryExternalSpan(unittest.TestCase): """ From 42d4aab3e7046a2b6545c14dcd08e2b56953a289 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Mon, 5 Jan 2026 11:21:34 +0530 Subject: [PATCH 12/97] Add mapping for reasoning effort to summary of responses API --- .../transformation.py | 14 +++---- ...odel_prices_and_context_window_backup.json | 42 +++++++++++++++++++ ...responses_transformation_transformation.py | 40 ++++++++++++++++++ 3 files changed, 89 insertions(+), 7 deletions(-) diff --git a/litellm/completion_extras/litellm_responses_transformation/transformation.py b/litellm/completion_extras/litellm_responses_transformation/transformation.py index 55a8e665bbd..9511537f72b 100644 --- a/litellm/completion_extras/litellm_responses_transformation/transformation.py +++ b/litellm/completion_extras/litellm_responses_transformation/transformation.py @@ -691,19 +691,19 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): if isinstance(reasoning_effort, dict): return Reasoning(**reasoning_effort) # type: ignore[typeddict-item] - # If string is passed, map without summary (default) + # If string is passed, map with summary="concise" if reasoning_effort == "none": - return Reasoning(effort="none") # type: ignore + return Reasoning(effort="none", summary="concise") # type: ignore elif reasoning_effort == "high": - return Reasoning(effort="high") + return Reasoning(effort="high", summary="concise") elif reasoning_effort == "xhigh": - return Reasoning(effort="xhigh") # type: ignore[typeddict-item] + return Reasoning(effort="xhigh", summary="concise") # type: ignore[typeddict-item] elif reasoning_effort == "medium": - return Reasoning(effort="medium") + return Reasoning(effort="medium", summary="concise") elif reasoning_effort == "low": - return Reasoning(effort="low") + return Reasoning(effort="low", summary="concise") elif reasoning_effort == "minimal": - return Reasoning(effort="minimal") + return Reasoning(effort="minimal", summary="concise") return None def _transform_response_format_to_text_format( diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index d32adf54b5e..81b4469f24c 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -11640,6 +11640,7 @@ "supports_tool_choice": true }, "gemini-1.5-flash": { + "deprecation_date": "2025-09-29", "input_cost_per_audio_per_second": 2e-06, "input_cost_per_audio_per_second_above_128k_tokens": 4e-06, "input_cost_per_character": 1.875e-08, @@ -11744,6 +11745,7 @@ "supports_vision": true }, "gemini-1.5-flash-exp-0827": { + "deprecation_date": "2025-09-29", "input_cost_per_audio_per_second": 2e-06, "input_cost_per_audio_per_second_above_128k_tokens": 4e-06, "input_cost_per_character": 1.875e-08, @@ -11778,6 +11780,7 @@ "supports_vision": true }, "gemini-1.5-flash-preview-0514": { + "deprecation_date": "2025-09-29", "input_cost_per_audio_per_second": 2e-06, "input_cost_per_audio_per_second_above_128k_tokens": 4e-06, "input_cost_per_character": 1.875e-08, @@ -11811,6 +11814,7 @@ "supports_vision": true }, "gemini-1.5-pro": { + "deprecation_date": "2025-09-29", "input_cost_per_audio_per_second": 3.125e-05, "input_cost_per_audio_per_second_above_128k_tokens": 6.25e-05, "input_cost_per_character": 3.125e-07, @@ -11898,6 +11902,7 @@ "supports_vision": true }, "gemini-1.5-pro-preview-0215": { + "deprecation_date": "2025-09-29", "input_cost_per_audio_per_second": 3.125e-05, "input_cost_per_audio_per_second_above_128k_tokens": 6.25e-05, "input_cost_per_character": 3.125e-07, @@ -11925,6 +11930,7 @@ "supports_tool_choice": true }, "gemini-1.5-pro-preview-0409": { + "deprecation_date": "2025-09-29", "input_cost_per_audio_per_second": 3.125e-05, "input_cost_per_audio_per_second_above_128k_tokens": 6.25e-05, "input_cost_per_character": 3.125e-07, @@ -11951,6 +11957,7 @@ "supports_tool_choice": true }, "gemini-1.5-pro-preview-0514": { + "deprecation_date": "2025-09-29", "input_cost_per_audio_per_second": 3.125e-05, "input_cost_per_audio_per_second_above_128k_tokens": 6.25e-05, "input_cost_per_character": 3.125e-07, @@ -12222,6 +12229,7 @@ "tpm": 250000 }, "gemini-2.0-flash-preview-image-generation": { + "deprecation_date": "2025-11-14", "cache_read_input_token_cost": 2.5e-08, "input_cost_per_audio_token": 7e-07, "input_cost_per_token": 1e-07, @@ -12260,6 +12268,7 @@ "supports_web_search": true }, "gemini-2.0-flash-thinking-exp": { + "deprecation_date": "2025-12-02", "cache_read_input_token_cost": 0.0, "input_cost_per_audio_per_second": 0, "input_cost_per_audio_per_second_above_128k_tokens": 0, @@ -12308,6 +12317,7 @@ "supports_web_search": true }, "gemini-2.0-flash-thinking-exp-01-21": { + "deprecation_date": "2025-12-02", "cache_read_input_token_cost": 0.0, "input_cost_per_audio_per_second": 0, "input_cost_per_audio_per_second_above_128k_tokens": 0, @@ -12494,6 +12504,7 @@ "tpm": 8000000 }, "gemini-2.5-flash-image-preview": { + "deprecation_date": "2026-01-15", "cache_read_input_token_cost": 7.5e-08, "input_cost_per_audio_token": 1e-06, "input_cost_per_token": 3e-07, @@ -12804,6 +12815,7 @@ "tpm": 8000000 }, "gemini-2.5-flash-lite-preview-06-17": { + "deprecation_date": "2025-11-18", "cache_read_input_token_cost": 2.5e-08, "input_cost_per_audio_token": 5e-07, "input_cost_per_token": 1e-07, @@ -12893,6 +12905,7 @@ "supports_web_search": true }, "gemini-2.5-flash-preview-05-20": { + "deprecation_date": "2025-11-18", "cache_read_input_token_cost": 7.5e-08, "input_cost_per_audio_token": 1e-06, "input_cost_per_token": 3e-07, @@ -13164,6 +13177,7 @@ "supports_web_search": true }, "gemini-2.5-pro-preview-03-25": { + "deprecation_date": "2025-12-02", "cache_read_input_token_cost": 3.125e-07, "input_cost_per_audio_token": 1.25e-06, "input_cost_per_token": 1.25e-06, @@ -13209,6 +13223,7 @@ "supports_web_search": true }, "gemini-2.5-pro-preview-05-06": { + "deprecation_date": "2025-12-02", "cache_read_input_token_cost": 3.125e-07, "input_cost_per_audio_token": 1.25e-06, "input_cost_per_token": 1.25e-06, @@ -13424,6 +13439,7 @@ "tpm": 10000000 }, "gemini/gemini-1.5-flash": { + "deprecation_date": "2025-09-29", "input_cost_per_token": 7.5e-08, "input_cost_per_token_above_128k_tokens": 1.5e-07, "litellm_provider": "gemini", @@ -13507,6 +13523,7 @@ "tpm": 4000000 }, "gemini/gemini-1.5-flash-8b": { + "deprecation_date": "2025-09-29", "input_cost_per_token": 0, "input_cost_per_token_above_128k_tokens": 0, "litellm_provider": "gemini", @@ -13533,6 +13550,7 @@ "tpm": 4000000 }, "gemini/gemini-1.5-flash-8b-exp-0827": { + "deprecation_date": "2025-09-29", "input_cost_per_token": 0, "input_cost_per_token_above_128k_tokens": 0, "litellm_provider": "gemini", @@ -13558,6 +13576,7 @@ "tpm": 4000000 }, "gemini/gemini-1.5-flash-8b-exp-0924": { + "deprecation_date": "2025-09-29", "input_cost_per_token": 0, "input_cost_per_token_above_128k_tokens": 0, "litellm_provider": "gemini", @@ -13584,6 +13603,7 @@ "tpm": 4000000 }, "gemini/gemini-1.5-flash-exp-0827": { + "deprecation_date": "2025-09-29", "input_cost_per_token": 0, "input_cost_per_token_above_128k_tokens": 0, "litellm_provider": "gemini", @@ -13609,6 +13629,7 @@ "tpm": 4000000 }, "gemini/gemini-1.5-flash-latest": { + "deprecation_date": "2025-09-29", "input_cost_per_token": 7.5e-08, "input_cost_per_token_above_128k_tokens": 1.5e-07, "litellm_provider": "gemini", @@ -13635,6 +13656,7 @@ "tpm": 4000000 }, "gemini/gemini-1.5-pro": { + "deprecation_date": "2025-09-29", "input_cost_per_token": 3.5e-06, "input_cost_per_token_above_128k_tokens": 7e-06, "litellm_provider": "gemini", @@ -13696,6 +13718,7 @@ "tpm": 4000000 }, "gemini/gemini-1.5-pro-exp-0801": { + "deprecation_date": "2025-09-29", "input_cost_per_token": 3.5e-06, "input_cost_per_token_above_128k_tokens": 7e-06, "litellm_provider": "gemini", @@ -13715,6 +13738,7 @@ "tpm": 4000000 }, "gemini/gemini-1.5-pro-exp-0827": { + "deprecation_date": "2025-09-29", "input_cost_per_token": 0, "input_cost_per_token_above_128k_tokens": 0, "litellm_provider": "gemini", @@ -13734,6 +13758,7 @@ "tpm": 4000000 }, "gemini/gemini-1.5-pro-latest": { + "deprecation_date": "2025-09-29", "input_cost_per_token": 3.5e-06, "input_cost_per_token_above_128k_tokens": 7e-06, "litellm_provider": "gemini", @@ -13916,6 +13941,7 @@ "tpm": 4000000 }, "gemini/gemini-2.0-flash-lite-preview-02-05": { + "deprecation_date": "2025-12-02", "cache_read_input_token_cost": 1.875e-08, "input_cost_per_audio_token": 7.5e-08, "input_cost_per_token": 7.5e-08, @@ -13953,6 +13979,7 @@ "tpm": 10000000 }, "gemini/gemini-2.0-flash-live-001": { + "deprecation_date": "2025-12-09", "cache_read_input_token_cost": 7.5e-08, "input_cost_per_audio_token": 2.1e-06, "input_cost_per_image": 2.1e-06, @@ -14001,6 +14028,7 @@ "tpm": 250000 }, "gemini/gemini-2.0-flash-preview-image-generation": { + "deprecation_date": "2025-11-14", "cache_read_input_token_cost": 2.5e-08, "input_cost_per_audio_token": 7e-07, "input_cost_per_token": 1e-07, @@ -14040,6 +14068,7 @@ "tpm": 10000000 }, "gemini/gemini-2.0-flash-thinking-exp": { + "deprecation_date": "2025-12-02", "cache_read_input_token_cost": 0.0, "input_cost_per_audio_per_second": 0, "input_cost_per_audio_per_second_above_128k_tokens": 0, @@ -14089,6 +14118,7 @@ "tpm": 4000000 }, "gemini/gemini-2.0-flash-thinking-exp-01-21": { + "deprecation_date": "2025-12-02", "cache_read_input_token_cost": 0.0, "input_cost_per_audio_per_second": 0, "input_cost_per_audio_per_second_above_128k_tokens": 0, @@ -14277,6 +14307,7 @@ "tpm": 8000000 }, "gemini/gemini-2.5-flash-image-preview": { + "deprecation_date": "2026-01-15", "cache_read_input_token_cost": 7.5e-08, "input_cost_per_audio_token": 1e-06, "input_cost_per_token": 3e-07, @@ -14597,6 +14628,7 @@ "tpm": 250000 }, "gemini/gemini-2.5-flash-lite-preview-06-17": { + "deprecation_date": "2025-11-18", "cache_read_input_token_cost": 2.5e-08, "input_cost_per_audio_token": 5e-07, "input_cost_per_token": 1e-07, @@ -14688,6 +14720,7 @@ "tpm": 250000 }, "gemini/gemini-2.5-flash-preview-05-20": { + "deprecation_date": "2025-11-18", "cache_read_input_token_cost": 7.5e-08, "input_cost_per_audio_token": 1e-06, "input_cost_per_token": 3e-07, @@ -15034,6 +15067,7 @@ "tpm": 250000 }, "gemini/gemini-2.5-pro-preview-03-25": { + "deprecation_date": "2025-12-02", "cache_read_input_token_cost": 3.125e-07, "input_cost_per_audio_token": 7e-07, "input_cost_per_token": 1.25e-06, @@ -15074,6 +15108,7 @@ "tpm": 10000000 }, "gemini/gemini-2.5-pro-preview-05-06": { + "deprecation_date": "2025-12-02", "cache_read_input_token_cost": 3.125e-07, "input_cost_per_audio_token": 7e-07, "input_cost_per_token": 1.25e-06, @@ -15349,6 +15384,7 @@ "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing" }, "gemini/imagen-3.0-generate-002": { + "deprecation_date": "2025-11-10", "litellm_provider": "gemini", "mode": "image_generation", "output_cost_per_image": 0.04, @@ -15415,6 +15451,7 @@ ] }, "gemini/veo-3.0-fast-generate-preview": { + "deprecation_date": "2025-11-12", "litellm_provider": "gemini", "max_input_tokens": 1024, "max_tokens": 1024, @@ -15429,6 +15466,7 @@ ] }, "gemini/veo-3.0-generate-preview": { + "deprecation_date": "2025-11-12", "litellm_provider": "gemini", "max_input_tokens": 1024, "max_tokens": 1024, @@ -25126,6 +25164,7 @@ "source": "https://docs.mistral.ai/capabilities/code_generation/" }, "text-embedding-004": { + "deprecation_date": "2026-01-14", "input_cost_per_character": 2.5e-08, "input_cost_per_token": 1e-07, "litellm_provider": "vertex_ai-embedding-models", @@ -27896,6 +27935,7 @@ "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing" }, "vertex_ai/imagen-3.0-generate-002": { + "deprecation_date": "2025-11-10", "litellm_provider": "vertex_ai-image-models", "mode": "image_generation", "output_cost_per_image": 0.04, @@ -28406,6 +28446,7 @@ ] }, "vertex_ai/veo-3.0-fast-generate-preview": { + "deprecation_date": "2025-11-12", "litellm_provider": "vertex_ai-video-models", "max_input_tokens": 1024, "max_tokens": 1024, @@ -28420,6 +28461,7 @@ ] }, "vertex_ai/veo-3.0-generate-preview": { + "deprecation_date": "2025-11-12", "litellm_provider": "vertex_ai-video-models", "max_input_tokens": 1024, "max_tokens": 1024, diff --git a/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py b/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py index 4320c932f41..77ac0742161 100644 --- a/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py +++ b/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py @@ -1007,3 +1007,43 @@ def test_multiple_tool_calls_in_single_choice(): assert tool_calls[2]["function"]["name"] == "get_horoscope" print("✓ Multiple tool calls are correctly grouped in a single choice") + + +def test_map_reasoning_effort_adds_summary_detailed(): + """ + Test that _map_reasoning_effort adds summary="detailed" when user provides reasoning_effort as a string. + + This ensures that when users pass reasoning_effort in the completions API for OpenAI responses/models, + the transformation automatically includes summary="detailed" in the reasoning parameter. + """ + from litellm.completion_extras.litellm_responses_transformation.transformation import ( + LiteLLMResponsesTransformationHandler, + ) + + handler = LiteLLMResponsesTransformationHandler() + + # Test all string effort levels + effort_levels = ["none", "low", "medium", "high", "xhigh", "minimal"] + + for effort in effort_levels: + result = handler._map_reasoning_effort(effort) + + assert result is not None, f"Result should not be None for effort={effort}" + assert result["effort"] == effort, f"Effort should be {effort}" + assert result["summary"] == "concise", f"Summary should be 'detailed' for effort={effort}" + + print(f"✓ reasoning_effort='{effort}' correctly maps to effort='{effort}', summary='detailed'") + + # Test that dict input is passed through as-is (no modification) + dict_input = {"effort": "high", "summary": "custom_summary"} + result_dict = handler._map_reasoning_effort(dict_input) + assert result_dict["effort"] == "high" + assert result_dict["summary"] == "custom_summary" + print("✓ Dict input is passed through without modification") + + # Test that None/unknown values return None + result_unknown = handler._map_reasoning_effort("unknown_value") + assert result_unknown is None + print("✓ Unknown reasoning_effort values return None") + + print("✓ All reasoning_effort string values correctly map to summary='detailed'") From 0f8e4364d609c5e7d11d08108de975e89ae0821f Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Mon, 5 Jan 2026 11:24:35 +0530 Subject: [PATCH 13/97] Replace summary param as detailed --- .../transformation.py | 12 ++++++------ ...itellm_responses_transformation_transformation.py | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/litellm/completion_extras/litellm_responses_transformation/transformation.py b/litellm/completion_extras/litellm_responses_transformation/transformation.py index 9511537f72b..3fb69f97fdf 100644 --- a/litellm/completion_extras/litellm_responses_transformation/transformation.py +++ b/litellm/completion_extras/litellm_responses_transformation/transformation.py @@ -693,17 +693,17 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): # If string is passed, map with summary="concise" if reasoning_effort == "none": - return Reasoning(effort="none", summary="concise") # type: ignore + return Reasoning(effort="none", summary="detailed") # type: ignore elif reasoning_effort == "high": - return Reasoning(effort="high", summary="concise") + return Reasoning(effort="high", summary="detailed") elif reasoning_effort == "xhigh": - return Reasoning(effort="xhigh", summary="concise") # type: ignore[typeddict-item] + return Reasoning(effort="xhigh", summary="detailed") # type: ignore[typeddict-item] elif reasoning_effort == "medium": - return Reasoning(effort="medium", summary="concise") + return Reasoning(effort="medium", summary="detailed") elif reasoning_effort == "low": - return Reasoning(effort="low", summary="concise") + return Reasoning(effort="low", summary="detailed") elif reasoning_effort == "minimal": - return Reasoning(effort="minimal", summary="concise") + return Reasoning(effort="minimal", summary="detailed") return None def _transform_response_format_to_text_format( diff --git a/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py b/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py index 77ac0742161..6490352c39b 100644 --- a/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py +++ b/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py @@ -1030,7 +1030,7 @@ def test_map_reasoning_effort_adds_summary_detailed(): assert result is not None, f"Result should not be None for effort={effort}" assert result["effort"] == effort, f"Effort should be {effort}" - assert result["summary"] == "concise", f"Summary should be 'detailed' for effort={effort}" + assert result["summary"] == "detailed", f"Summary should be 'detailed' for effort={effort}" print(f"✓ reasoning_effort='{effort}' correctly maps to effort='{effort}', summary='detailed'") From 02a41a5c13bfbe89d8d0672e27c0e768e57b7727 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Mon, 5 Jan 2026 11:40:40 +0530 Subject: [PATCH 14/97] fix the comment --- .../litellm_responses_transformation/transformation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/completion_extras/litellm_responses_transformation/transformation.py b/litellm/completion_extras/litellm_responses_transformation/transformation.py index 3fb69f97fdf..5b206317b29 100644 --- a/litellm/completion_extras/litellm_responses_transformation/transformation.py +++ b/litellm/completion_extras/litellm_responses_transformation/transformation.py @@ -691,7 +691,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): if isinstance(reasoning_effort, dict): return Reasoning(**reasoning_effort) # type: ignore[typeddict-item] - # If string is passed, map with summary="concise" + # If string is passed, map with summary="detailed" if reasoning_effort == "none": return Reasoning(effort="none", summary="detailed") # type: ignore elif reasoning_effort == "high": From 0730a74ddabc25f7a94fc986b4e370c1d7b99130 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Mon, 5 Jan 2026 12:25:13 +0530 Subject: [PATCH 15/97] fix: auth header for custom api base in generate Content request --- .../gemini/google_genai/transformation.py | 4 +- .../test_google_genai_transformation.py | 81 +++++++++++++++++++ 2 files changed, 84 insertions(+), 1 deletion(-) diff --git a/litellm/llms/gemini/google_genai/transformation.py b/litellm/llms/gemini/google_genai/transformation.py index d3b55b13ad4..3474c8abe34 100644 --- a/litellm/llms/gemini/google_genai/transformation.py +++ b/litellm/llms/gemini/google_genai/transformation.py @@ -153,7 +153,9 @@ class GoogleGenAIConfig(BaseGoogleGenAIGenerateContentConfig, VertexLLM): gemini_api_key = api_key or self._get_google_ai_studio_api_key( dict(litellm_params or {}) ) - if gemini_api_key is not None: + if isinstance(gemini_api_key, dict): + default_headers.update(gemini_api_key) + elif gemini_api_key is not None: default_headers[self.XGOOGLE_API_KEY] = gemini_api_key if headers is not None: default_headers.update(headers) diff --git a/tests/test_litellm/google_genai/test_google_genai_transformation.py b/tests/test_litellm/google_genai/test_google_genai_transformation.py index 0d5468cf275..8943d198dc1 100644 --- a/tests/test_litellm/google_genai/test_google_genai_transformation.py +++ b/tests/test_litellm/google_genai/test_google_genai_transformation.py @@ -366,3 +366,84 @@ def test_transform_generate_content_request_system_instruction_with_tools(): assert "tools" in result, "tools should be in request body" assert result["tools"] == tools assert result["model"] == "gemini-3-flash-preview" + + +def test_validate_environment_with_dict_api_key(): + """ + Test that validate_environment correctly handles api_key as a dict. + + This happens when using custom api_base with Gemini - the auth_header + is returned as {"x-goog-api-key": "sk-test"} and should be merged into + headers instead of being set as a header value. + + Regression test for: https://github.com/BerriAI/litellm/issues/xxxxx + """ + config = GoogleGenAIConfig() + + # Simulate the case where auth_header is a dict (custom api_base scenario) + auth_header_dict = {"x-goog-api-key": "sk-test-key-123"} + + result = config.validate_environment( + api_key=auth_header_dict, + headers=None, + model="gemini-2.5-pro", + litellm_params={} + ) + + # The dict should be merged into headers, not set as a value + assert "x-goog-api-key" in result, "x-goog-api-key should be in headers" + assert result["x-goog-api-key"] == "sk-test-key-123", "API key should be the string value, not a dict" + assert isinstance(result["x-goog-api-key"], str), "Header value should be a string, not a dict" + assert "Content-Type" in result, "Content-Type should be in headers" + assert result["Content-Type"] == "application/json" + + +def test_validate_environment_with_string_api_key(): + """ + Test that validate_environment correctly handles api_key as a string. + + This is the normal case when using standard Gemini API. + """ + config = GoogleGenAIConfig() + + # Normal case: api_key is a string + api_key_string = "sk-test-key-456" + + result = config.validate_environment( + api_key=api_key_string, + headers=None, + model="gemini-2.5-pro", + litellm_params={} + ) + + # The string should be set as the header value + assert "x-goog-api-key" in result, "x-goog-api-key should be in headers" + assert result["x-goog-api-key"] == "sk-test-key-456", "API key should match input" + assert isinstance(result["x-goog-api-key"], str), "Header value should be a string" + assert "Content-Type" in result, "Content-Type should be in headers" + + +def test_validate_environment_with_extra_headers(): + """ + Test that validate_environment correctly merges extra headers with dict api_key. + """ + config = GoogleGenAIConfig() + + # Custom api_base scenario with additional headers + auth_header_dict = {"x-goog-api-key": "sk-test-key-789"} + extra_headers = {"X-Custom-Header": "custom-value"} + + result = config.validate_environment( + api_key=auth_header_dict, + headers=extra_headers, + model="gemini-2.5-pro", + litellm_params={} + ) + + # Both the auth dict and extra headers should be merged + assert "x-goog-api-key" in result, "x-goog-api-key should be in headers" + assert result["x-goog-api-key"] == "sk-test-key-789", "API key should be correctly set" + assert isinstance(result["x-goog-api-key"], str), "Header value should be a string" + assert "X-Custom-Header" in result, "Extra headers should be merged" + assert result["X-Custom-Header"] == "custom-value" + assert "Content-Type" in result From 6337ea876bd03def56c5d51fc77e2ec8ed93b557 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Mon, 5 Jan 2026 12:57:30 +0530 Subject: [PATCH 16/97] fix: Add custom llm provider to get_llm_provider when sent via UI --- litellm/main.py | 12 +++++- .../litellm_utils_tests/test_health_check.py | 37 +++++++++++++++++++ 2 files changed, 47 insertions(+), 2 deletions(-) diff --git a/litellm/main.py b/litellm/main.py index a0f3461b45c..f4f27eb5841 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -110,7 +110,6 @@ from litellm.types.utils import ( RawRequestTypedDict, StreamingChoices, ) - from litellm.utils import ( Choices, CustomStreamWrapper, @@ -6656,7 +6655,16 @@ async def ahealth_check( if model in litellm.model_cost and mode is None: mode = litellm.model_cost[model].get("mode") - model, custom_llm_provider, _, _ = get_llm_provider(model=model) + custom_llm_provider_from_params = model_params.get("custom_llm_provider", None) + api_base_from_params = model_params.get("api_base", None) + api_key_from_params = model_params.get("api_key", None) + + model, custom_llm_provider, _, _ = get_llm_provider( + model=model, + custom_llm_provider=custom_llm_provider_from_params, + api_base=api_base_from_params, + api_key=api_key_from_params, + ) if model in litellm.model_cost and mode is None: mode = litellm.model_cost[model].get("mode") diff --git a/tests/litellm_utils_tests/test_health_check.py b/tests/litellm_utils_tests/test_health_check.py index 98b1a353793..986f8cf4912 100644 --- a/tests/litellm_utils_tests/test_health_check.py +++ b/tests/litellm_utils_tests/test_health_check.py @@ -637,3 +637,40 @@ async def test_image_generation_health_check_prompt(monkeypatch): assert len(health_check_calls) == 1 assert health_check_calls[0]["prompt"] == override_prompt + + +@pytest.mark.asyncio +async def test_health_check_with_custom_llm_provider(): + """ + Test that ahealth_check correctly uses custom_llm_provider from model_params. + + This test verifies the fix for the issue where the UI's "Test connect" button + failed with "LLM Provider NOT provided" error for OpenAI-compatible self-hosted + providers, even when a provider was selected in the dropdown. + + The fix ensures that when custom_llm_provider is passed in model_params, + it's properly forwarded to get_llm_provider() to identify the correct provider. + """ + from unittest.mock import MagicMock + + # Mock the completion call to avoid making real API calls + mock_response = MagicMock() + mock_response._hidden_params = {"headers": {"x-ratelimit-remaining-tokens": "1000"}} + + with patch("litellm.acompletion", return_value=mock_response): + # Test with a custom model name that wouldn't be recognized without custom_llm_provider + response = await litellm.ahealth_check( + model_params={ + "model": "deepseek-r1-distill-qwen-1.5B-q4", + "custom_llm_provider": "openai", + "api_base": "https://example.com/v1", + "api_key": "fake-key", + }, + mode="chat", + ) + + print(f"response: {response}") + + # Should succeed without "LLM Provider NOT provided" error + assert "error" not in response + assert isinstance(response, dict) From 28c7659d3da19119340a9301816fef1070f7048a Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Mon, 5 Jan 2026 16:06:10 +0530 Subject: [PATCH 17/97] Potential fix for code scanning alert no. 3954: Clear-text logging of sensitive information Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com> --- tests/litellm_utils_tests/test_health_check.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/litellm_utils_tests/test_health_check.py b/tests/litellm_utils_tests/test_health_check.py index 986f8cf4912..19882bbe4be 100644 --- a/tests/litellm_utils_tests/test_health_check.py +++ b/tests/litellm_utils_tests/test_health_check.py @@ -669,8 +669,6 @@ async def test_health_check_with_custom_llm_provider(): mode="chat", ) - print(f"response: {response}") - # Should succeed without "LLM Provider NOT provided" error assert "error" not in response assert isinstance(response, dict) From d590c820c212e3bfebc1c37f152d44131dbbb128 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Mon, 5 Jan 2026 16:32:21 +0530 Subject: [PATCH 18/97] Fix: TypeError: Object of type function is not JSON serializable --- litellm/utils.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/litellm/utils.py b/litellm/utils.py index df0b2317123..fd1ee48850a 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -2990,6 +2990,8 @@ def get_optional_params_embeddings( # noqa: PLR0915 drop_params = passed_params.pop("drop_params", None) additional_drop_params = passed_params.pop("additional_drop_params", None) + # Remove function objects from passed_params to avoid JSON serialization errors + passed_params.pop("get_supported_openai_params", None) def _check_valid_arg(supported_params: Optional[list]): if supported_params is None: From b24156079fb6b8fb95640946f2208d522bc4a638 Mon Sep 17 00:00:00 2001 From: Matt Greathouse Date: Mon, 5 Jan 2026 06:07:23 -0500 Subject: [PATCH 19/97] Add gemini-3-flash on openrouter (#18247) --- model_prices_and_context_window.json | 47 ++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 81b4469f24c..e823dd5dc6b 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -22605,6 +22605,53 @@ "supports_vision": true, "supports_web_search": true }, + "openrouter/google/gemini-3-flash-preview": { + "cache_read_input_token_cost": 5e-08, + "input_cost_per_audio_token": 1e-06, + "input_cost_per_token": 5e-07, + "litellm_provider": "openrouter", + "max_audio_length_hours": 8.4, + "max_audio_per_prompt": 1, + "max_images_per_prompt": 3000, + "max_input_tokens": 1048576, + "max_output_tokens": 65535, + "max_pdf_size_mb": 30, + "max_tokens": 65535, + "max_video_length": 1, + "max_videos_per_prompt": 10, + "mode": "chat", + "output_cost_per_reasoning_token": 3e-06, + "output_cost_per_token": 3e-06, + "rpm": 2000, + "source": "https://ai.google.dev/pricing/gemini-3", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/completions", + "/v1/batch" + ], + "supported_modalities": [ + "text", + "image", + "audio", + "video" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_output": false, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_url_context": true, + "supports_vision": true, + "supports_web_search": true, + "tpm": 800000 + }, "openrouter/google/gemini-pro-1.5": { "input_cost_per_image": 0.00265, "input_cost_per_token": 2.5e-06, From 840617de73b95a9ab5d22a54f92d7cb6de8640fe Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Mon, 5 Jan 2026 16:41:24 +0530 Subject: [PATCH 20/97] fix: tests.test_litellm.interactions.test_gemini_interactions.TestGeminiInteractions --- .../gemini/interactions/transformation.py | 24 ++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/litellm/llms/gemini/interactions/transformation.py b/litellm/llms/gemini/interactions/transformation.py index d21775eb236..769d70473b9 100644 --- a/litellm/llms/gemini/interactions/transformation.py +++ b/litellm/llms/gemini/interactions/transformation.py @@ -139,7 +139,7 @@ class GoogleAIStudioInteractionsConfig(BaseInteractionsAPIConfig): raw_response: httpx.Response, logging_obj: LiteLLMLoggingObj, ) -> InteractionsAPIResponse: - """Parse response - it already matches our response type.""" + """Parse response and transform to OpenAI-compatible format.""" try: logging_obj.post_call( original_response=raw_response.text, @@ -155,6 +155,18 @@ class GoogleAIStudioInteractionsConfig(BaseInteractionsAPIConfig): verbose_logger.debug("Google AI Interactions response: %s", raw_json) + # Transform usage to OpenAI format + if "usage" in raw_json and raw_json["usage"]: + usage = raw_json["usage"] + # Map Google's field names to OpenAI's field names + if "total_input_tokens" in usage: + usage["input_tokens"] = usage["total_input_tokens"] + if "total_output_tokens" in usage: + usage["output_tokens"] = usage["total_output_tokens"] + if "total_tokens" in usage: + # Keep total_tokens as is - it's the same in both formats + pass + response = InteractionsAPIResponse(**raw_json) response._hidden_params["headers"] = dict(raw_response.headers) response._hidden_params["additional_headers"] = process_response_headers(dict(raw_response.headers)) @@ -200,6 +212,16 @@ class GoogleAIStudioInteractionsConfig(BaseInteractionsAPIConfig): status_code=raw_response.status_code, headers=dict(raw_response.headers), ) + + # Transform usage to OpenAI format + if "usage" in raw_json and raw_json["usage"]: + usage = raw_json["usage"] + # Map Google's field names to OpenAI's field names + if "total_input_tokens" in usage: + usage["input_tokens"] = usage["total_input_tokens"] + if "total_output_tokens" in usage: + usage["output_tokens"] = usage["total_output_tokens"] + response = InteractionsAPIResponse(**raw_json) response._hidden_params["headers"] = dict(raw_response.headers) return response From 7f4dba2a01f1db8a27eb8b58f97efb3374c55496 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 5 Jan 2026 16:51:23 +0530 Subject: [PATCH 21/97] [Feat] Add Cost Estimator for AI Gateway (#18643) * add estimate_cost endpoint * TestCostEstimateEndpoint * fix estimate_cost * add /cost/estimate to spend tracking routes * fix code QA checks * fixes endpoint --- ...odel_prices_and_context_window_backup.json | 42 +++++ litellm/proxy/_types.py | 44 +++++ .../cost_tracking_settings.py | 168 +++++++++++++++++- .../test_cost_estimate_endpoint.py | 75 ++++++++ 4 files changed, 328 insertions(+), 1 deletion(-) create mode 100644 tests/litellm/proxy/management_endpoints/test_cost_estimate_endpoint.py diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index d32adf54b5e..81b4469f24c 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -11640,6 +11640,7 @@ "supports_tool_choice": true }, "gemini-1.5-flash": { + "deprecation_date": "2025-09-29", "input_cost_per_audio_per_second": 2e-06, "input_cost_per_audio_per_second_above_128k_tokens": 4e-06, "input_cost_per_character": 1.875e-08, @@ -11744,6 +11745,7 @@ "supports_vision": true }, "gemini-1.5-flash-exp-0827": { + "deprecation_date": "2025-09-29", "input_cost_per_audio_per_second": 2e-06, "input_cost_per_audio_per_second_above_128k_tokens": 4e-06, "input_cost_per_character": 1.875e-08, @@ -11778,6 +11780,7 @@ "supports_vision": true }, "gemini-1.5-flash-preview-0514": { + "deprecation_date": "2025-09-29", "input_cost_per_audio_per_second": 2e-06, "input_cost_per_audio_per_second_above_128k_tokens": 4e-06, "input_cost_per_character": 1.875e-08, @@ -11811,6 +11814,7 @@ "supports_vision": true }, "gemini-1.5-pro": { + "deprecation_date": "2025-09-29", "input_cost_per_audio_per_second": 3.125e-05, "input_cost_per_audio_per_second_above_128k_tokens": 6.25e-05, "input_cost_per_character": 3.125e-07, @@ -11898,6 +11902,7 @@ "supports_vision": true }, "gemini-1.5-pro-preview-0215": { + "deprecation_date": "2025-09-29", "input_cost_per_audio_per_second": 3.125e-05, "input_cost_per_audio_per_second_above_128k_tokens": 6.25e-05, "input_cost_per_character": 3.125e-07, @@ -11925,6 +11930,7 @@ "supports_tool_choice": true }, "gemini-1.5-pro-preview-0409": { + "deprecation_date": "2025-09-29", "input_cost_per_audio_per_second": 3.125e-05, "input_cost_per_audio_per_second_above_128k_tokens": 6.25e-05, "input_cost_per_character": 3.125e-07, @@ -11951,6 +11957,7 @@ "supports_tool_choice": true }, "gemini-1.5-pro-preview-0514": { + "deprecation_date": "2025-09-29", "input_cost_per_audio_per_second": 3.125e-05, "input_cost_per_audio_per_second_above_128k_tokens": 6.25e-05, "input_cost_per_character": 3.125e-07, @@ -12222,6 +12229,7 @@ "tpm": 250000 }, "gemini-2.0-flash-preview-image-generation": { + "deprecation_date": "2025-11-14", "cache_read_input_token_cost": 2.5e-08, "input_cost_per_audio_token": 7e-07, "input_cost_per_token": 1e-07, @@ -12260,6 +12268,7 @@ "supports_web_search": true }, "gemini-2.0-flash-thinking-exp": { + "deprecation_date": "2025-12-02", "cache_read_input_token_cost": 0.0, "input_cost_per_audio_per_second": 0, "input_cost_per_audio_per_second_above_128k_tokens": 0, @@ -12308,6 +12317,7 @@ "supports_web_search": true }, "gemini-2.0-flash-thinking-exp-01-21": { + "deprecation_date": "2025-12-02", "cache_read_input_token_cost": 0.0, "input_cost_per_audio_per_second": 0, "input_cost_per_audio_per_second_above_128k_tokens": 0, @@ -12494,6 +12504,7 @@ "tpm": 8000000 }, "gemini-2.5-flash-image-preview": { + "deprecation_date": "2026-01-15", "cache_read_input_token_cost": 7.5e-08, "input_cost_per_audio_token": 1e-06, "input_cost_per_token": 3e-07, @@ -12804,6 +12815,7 @@ "tpm": 8000000 }, "gemini-2.5-flash-lite-preview-06-17": { + "deprecation_date": "2025-11-18", "cache_read_input_token_cost": 2.5e-08, "input_cost_per_audio_token": 5e-07, "input_cost_per_token": 1e-07, @@ -12893,6 +12905,7 @@ "supports_web_search": true }, "gemini-2.5-flash-preview-05-20": { + "deprecation_date": "2025-11-18", "cache_read_input_token_cost": 7.5e-08, "input_cost_per_audio_token": 1e-06, "input_cost_per_token": 3e-07, @@ -13164,6 +13177,7 @@ "supports_web_search": true }, "gemini-2.5-pro-preview-03-25": { + "deprecation_date": "2025-12-02", "cache_read_input_token_cost": 3.125e-07, "input_cost_per_audio_token": 1.25e-06, "input_cost_per_token": 1.25e-06, @@ -13209,6 +13223,7 @@ "supports_web_search": true }, "gemini-2.5-pro-preview-05-06": { + "deprecation_date": "2025-12-02", "cache_read_input_token_cost": 3.125e-07, "input_cost_per_audio_token": 1.25e-06, "input_cost_per_token": 1.25e-06, @@ -13424,6 +13439,7 @@ "tpm": 10000000 }, "gemini/gemini-1.5-flash": { + "deprecation_date": "2025-09-29", "input_cost_per_token": 7.5e-08, "input_cost_per_token_above_128k_tokens": 1.5e-07, "litellm_provider": "gemini", @@ -13507,6 +13523,7 @@ "tpm": 4000000 }, "gemini/gemini-1.5-flash-8b": { + "deprecation_date": "2025-09-29", "input_cost_per_token": 0, "input_cost_per_token_above_128k_tokens": 0, "litellm_provider": "gemini", @@ -13533,6 +13550,7 @@ "tpm": 4000000 }, "gemini/gemini-1.5-flash-8b-exp-0827": { + "deprecation_date": "2025-09-29", "input_cost_per_token": 0, "input_cost_per_token_above_128k_tokens": 0, "litellm_provider": "gemini", @@ -13558,6 +13576,7 @@ "tpm": 4000000 }, "gemini/gemini-1.5-flash-8b-exp-0924": { + "deprecation_date": "2025-09-29", "input_cost_per_token": 0, "input_cost_per_token_above_128k_tokens": 0, "litellm_provider": "gemini", @@ -13584,6 +13603,7 @@ "tpm": 4000000 }, "gemini/gemini-1.5-flash-exp-0827": { + "deprecation_date": "2025-09-29", "input_cost_per_token": 0, "input_cost_per_token_above_128k_tokens": 0, "litellm_provider": "gemini", @@ -13609,6 +13629,7 @@ "tpm": 4000000 }, "gemini/gemini-1.5-flash-latest": { + "deprecation_date": "2025-09-29", "input_cost_per_token": 7.5e-08, "input_cost_per_token_above_128k_tokens": 1.5e-07, "litellm_provider": "gemini", @@ -13635,6 +13656,7 @@ "tpm": 4000000 }, "gemini/gemini-1.5-pro": { + "deprecation_date": "2025-09-29", "input_cost_per_token": 3.5e-06, "input_cost_per_token_above_128k_tokens": 7e-06, "litellm_provider": "gemini", @@ -13696,6 +13718,7 @@ "tpm": 4000000 }, "gemini/gemini-1.5-pro-exp-0801": { + "deprecation_date": "2025-09-29", "input_cost_per_token": 3.5e-06, "input_cost_per_token_above_128k_tokens": 7e-06, "litellm_provider": "gemini", @@ -13715,6 +13738,7 @@ "tpm": 4000000 }, "gemini/gemini-1.5-pro-exp-0827": { + "deprecation_date": "2025-09-29", "input_cost_per_token": 0, "input_cost_per_token_above_128k_tokens": 0, "litellm_provider": "gemini", @@ -13734,6 +13758,7 @@ "tpm": 4000000 }, "gemini/gemini-1.5-pro-latest": { + "deprecation_date": "2025-09-29", "input_cost_per_token": 3.5e-06, "input_cost_per_token_above_128k_tokens": 7e-06, "litellm_provider": "gemini", @@ -13916,6 +13941,7 @@ "tpm": 4000000 }, "gemini/gemini-2.0-flash-lite-preview-02-05": { + "deprecation_date": "2025-12-02", "cache_read_input_token_cost": 1.875e-08, "input_cost_per_audio_token": 7.5e-08, "input_cost_per_token": 7.5e-08, @@ -13953,6 +13979,7 @@ "tpm": 10000000 }, "gemini/gemini-2.0-flash-live-001": { + "deprecation_date": "2025-12-09", "cache_read_input_token_cost": 7.5e-08, "input_cost_per_audio_token": 2.1e-06, "input_cost_per_image": 2.1e-06, @@ -14001,6 +14028,7 @@ "tpm": 250000 }, "gemini/gemini-2.0-flash-preview-image-generation": { + "deprecation_date": "2025-11-14", "cache_read_input_token_cost": 2.5e-08, "input_cost_per_audio_token": 7e-07, "input_cost_per_token": 1e-07, @@ -14040,6 +14068,7 @@ "tpm": 10000000 }, "gemini/gemini-2.0-flash-thinking-exp": { + "deprecation_date": "2025-12-02", "cache_read_input_token_cost": 0.0, "input_cost_per_audio_per_second": 0, "input_cost_per_audio_per_second_above_128k_tokens": 0, @@ -14089,6 +14118,7 @@ "tpm": 4000000 }, "gemini/gemini-2.0-flash-thinking-exp-01-21": { + "deprecation_date": "2025-12-02", "cache_read_input_token_cost": 0.0, "input_cost_per_audio_per_second": 0, "input_cost_per_audio_per_second_above_128k_tokens": 0, @@ -14277,6 +14307,7 @@ "tpm": 8000000 }, "gemini/gemini-2.5-flash-image-preview": { + "deprecation_date": "2026-01-15", "cache_read_input_token_cost": 7.5e-08, "input_cost_per_audio_token": 1e-06, "input_cost_per_token": 3e-07, @@ -14597,6 +14628,7 @@ "tpm": 250000 }, "gemini/gemini-2.5-flash-lite-preview-06-17": { + "deprecation_date": "2025-11-18", "cache_read_input_token_cost": 2.5e-08, "input_cost_per_audio_token": 5e-07, "input_cost_per_token": 1e-07, @@ -14688,6 +14720,7 @@ "tpm": 250000 }, "gemini/gemini-2.5-flash-preview-05-20": { + "deprecation_date": "2025-11-18", "cache_read_input_token_cost": 7.5e-08, "input_cost_per_audio_token": 1e-06, "input_cost_per_token": 3e-07, @@ -15034,6 +15067,7 @@ "tpm": 250000 }, "gemini/gemini-2.5-pro-preview-03-25": { + "deprecation_date": "2025-12-02", "cache_read_input_token_cost": 3.125e-07, "input_cost_per_audio_token": 7e-07, "input_cost_per_token": 1.25e-06, @@ -15074,6 +15108,7 @@ "tpm": 10000000 }, "gemini/gemini-2.5-pro-preview-05-06": { + "deprecation_date": "2025-12-02", "cache_read_input_token_cost": 3.125e-07, "input_cost_per_audio_token": 7e-07, "input_cost_per_token": 1.25e-06, @@ -15349,6 +15384,7 @@ "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing" }, "gemini/imagen-3.0-generate-002": { + "deprecation_date": "2025-11-10", "litellm_provider": "gemini", "mode": "image_generation", "output_cost_per_image": 0.04, @@ -15415,6 +15451,7 @@ ] }, "gemini/veo-3.0-fast-generate-preview": { + "deprecation_date": "2025-11-12", "litellm_provider": "gemini", "max_input_tokens": 1024, "max_tokens": 1024, @@ -15429,6 +15466,7 @@ ] }, "gemini/veo-3.0-generate-preview": { + "deprecation_date": "2025-11-12", "litellm_provider": "gemini", "max_input_tokens": 1024, "max_tokens": 1024, @@ -25126,6 +25164,7 @@ "source": "https://docs.mistral.ai/capabilities/code_generation/" }, "text-embedding-004": { + "deprecation_date": "2026-01-14", "input_cost_per_character": 2.5e-08, "input_cost_per_token": 1e-07, "litellm_provider": "vertex_ai-embedding-models", @@ -27896,6 +27935,7 @@ "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing" }, "vertex_ai/imagen-3.0-generate-002": { + "deprecation_date": "2025-11-10", "litellm_provider": "vertex_ai-image-models", "mode": "image_generation", "output_cost_per_image": 0.04, @@ -28406,6 +28446,7 @@ ] }, "vertex_ai/veo-3.0-fast-generate-preview": { + "deprecation_date": "2025-11-12", "litellm_provider": "vertex_ai-video-models", "max_input_tokens": 1024, "max_tokens": 1024, @@ -28420,6 +28461,7 @@ ] }, "vertex_ai/veo-3.0-generate-preview": { + "deprecation_date": "2025-11-12", "litellm_provider": "vertex_ai-video-models", "max_input_tokens": 1024, "max_tokens": 1024, diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index fa32f60c073..b77cc40d6dc 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -522,6 +522,7 @@ class LiteLLMRoutes(enum.Enum): "/spend/tags", "/spend/calculate", "/spend/logs", + "/cost/estimate", ] global_spend_tracking_routes = [ @@ -3825,3 +3826,46 @@ class LiteLLM_ManagedVectorStoresTable(LiteLLMPydanticObjectBase): class ResponseLiteLLM_ManagedVectorStore(TypedDict, total=False): vector_store: LiteLLM_ManagedVectorStoresTable + + +class CostEstimateRequest(LiteLLMPydanticObjectBase): + """Request body for /cost/estimate endpoint.""" + + model: str = Field(description="Model name (from /model_group/info)") + input_tokens: int = Field(description="Expected input tokens per request", ge=0) + output_tokens: int = Field(description="Expected output tokens per request", ge=0) + num_requests_per_day: Optional[int] = Field( + default=None, description="Number of requests per day", ge=0 + ) + num_requests_per_month: Optional[int] = Field( + default=None, description="Number of requests per month", ge=0 + ) + + +class CostEstimateResponse(LiteLLMPydanticObjectBase): + """Response body for /cost/estimate endpoint.""" + + model: str + input_tokens: int + output_tokens: int + num_requests_per_day: Optional[int] = None + num_requests_per_month: Optional[int] = None + # Per-request costs + cost_per_request: float = Field(description="Total cost per request (includes margin)") + input_cost_per_request: float = Field(description="Input token cost per request (before margin)") + output_cost_per_request: float = Field(description="Output token cost per request (before margin)") + margin_cost_per_request: float = Field(default=0.0, description="Margin/fee added per request") + # Daily costs (if num_requests_per_day provided) + daily_cost: Optional[float] = Field(default=None, description="Total daily cost (includes margin)") + daily_input_cost: Optional[float] = Field(default=None, description="Daily input token cost") + daily_output_cost: Optional[float] = Field(default=None, description="Daily output token cost") + daily_margin_cost: Optional[float] = Field(default=None, description="Daily margin/fee") + # Monthly costs (if num_requests_per_month provided) + monthly_cost: Optional[float] = Field(default=None, description="Total monthly cost (includes margin)") + monthly_input_cost: Optional[float] = Field(default=None, description="Monthly input token cost") + monthly_output_cost: Optional[float] = Field(default=None, description="Monthly output token cost") + monthly_margin_cost: Optional[float] = Field(default=None, description="Monthly margin/fee") + # Pricing info + input_cost_per_token: Optional[float] = None + output_cost_per_token: Optional[float] = None + provider: Optional[str] = None diff --git a/litellm/proxy/management_endpoints/cost_tracking_settings.py b/litellm/proxy/management_endpoints/cost_tracking_settings.py index 86433a232c0..0622393ec8c 100644 --- a/litellm/proxy/management_endpoints/cost_tracking_settings.py +++ b/litellm/proxy/management_endpoints/cost_tracking_settings.py @@ -7,6 +7,7 @@ GET /config/cost_discount_config - Get current cost discount configuration PATCH /config/cost_discount_config - Update cost discount configuration GET /config/cost_margin_config - Get current cost margin configuration PATCH /config/cost_margin_config - Update cost margin configuration +POST /cost/estimate - Estimate cost for a given model and token counts """ from typing import Dict, Union @@ -15,13 +16,37 @@ from fastapi import APIRouter, Depends, HTTPException import litellm from litellm._logging import verbose_proxy_logger -from litellm.proxy._types import CommonProxyErrors, UserAPIKeyAuth +from litellm.cost_calculator import completion_cost +from litellm.proxy._types import ( + CommonProxyErrors, + CostEstimateRequest, + CostEstimateResponse, + UserAPIKeyAuth, +) from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.types.utils import LlmProvidersSet router = APIRouter() +def _calculate_period_costs( + num_requests, cost_per_request, input_cost, output_cost, margin_cost +): + """ + Calculate costs for a given number of requests. + + Returns tuple of (total_cost, input_cost, output_cost, margin_cost) or all None if num_requests is None/0. + """ + if not num_requests: + return None, None, None, None + return ( + cost_per_request * num_requests, + input_cost * num_requests, + output_cost * num_requests, + margin_cost * num_requests, + ) + + @router.get( "/config/cost_discount_config", tags=["Cost Tracking"], @@ -347,3 +372,144 @@ async def update_cost_margin_config( detail={"error": f"Failed to update cost margin config: {str(e)}"} ) + +@router.post( + "/cost/estimate", + tags=["Cost Tracking"], + dependencies=[Depends(user_api_key_auth)], + response_model=CostEstimateResponse, +) +async def estimate_cost( + request: CostEstimateRequest, + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +) -> CostEstimateResponse: + """ + Estimate cost for a given model and token counts. + + This endpoint uses the same cost calculation logic as actual requests, + including any configured margins and discounts. + + Parameters: + - model: Model name (e.g., "gpt-4", "claude-3-opus") + - input_tokens: Expected input tokens per request + - output_tokens: Expected output tokens per request + - num_requests_per_day: Number of requests per day (optional) + - num_requests_per_month: Number of requests per month (optional) + + Returns cost breakdown including: + - Per-request costs (input, output, margin) + - Daily costs (if num_requests_per_day provided) + - Monthly costs (if num_requests_per_month provided) + + Example: + ```json + { + "model": "gpt-4", + "input_tokens": 1000, + "output_tokens": 500, + "num_requests_per_day": 100, + "num_requests_per_month": 3000 + } + ``` + """ + from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.types.utils import Usage + from litellm.utils import ModelResponse + + # Create a mock response with usage for completion_cost + mock_response = ModelResponse( + model=request.model, + usage=Usage( + prompt_tokens=request.input_tokens, + completion_tokens=request.output_tokens, + total_tokens=request.input_tokens + request.output_tokens, + ), + ) + + # Create a logging object to capture cost breakdown + litellm_logging_obj = LiteLLMLoggingObj( + model=request.model, + messages=[], + stream=False, + call_type="completion", + start_time=None, + litellm_call_id="cost-estimate", + function_id="cost-estimate", + ) + + # Use completion_cost which handles all the logic including margins/discounts + try: + cost_per_request = completion_cost( + completion_response=mock_response, + model=request.model, + litellm_logging_obj=litellm_logging_obj, + ) + except Exception as e: + raise HTTPException( + status_code=404, + detail={ + "error": f"Could not calculate cost for model '{request.model}': {str(e)}" + }, + ) + + # Get cost breakdown from the logging object + cost_breakdown = litellm_logging_obj.cost_breakdown + + input_cost = cost_breakdown.get("input_cost", 0.0) if cost_breakdown else 0.0 + output_cost = cost_breakdown.get("output_cost", 0.0) if cost_breakdown else 0.0 + margin_cost = cost_breakdown.get("margin_total_amount", 0.0) if cost_breakdown else 0.0 + + # Get model info for per-token pricing display + try: + model_info = litellm.get_model_info(model=request.model) + input_cost_per_token = model_info.get("input_cost_per_token") + output_cost_per_token = model_info.get("output_cost_per_token") + custom_llm_provider = model_info.get("litellm_provider") + except Exception: + input_cost_per_token = None + output_cost_per_token = None + custom_llm_provider = None + + # Calculate daily and monthly costs + daily_cost, daily_input_cost, daily_output_cost, daily_margin_cost = ( + _calculate_period_costs( + num_requests=request.num_requests_per_day, + cost_per_request=cost_per_request, + input_cost=input_cost, + output_cost=output_cost, + margin_cost=margin_cost, + ) + ) + monthly_cost, monthly_input_cost, monthly_output_cost, monthly_margin_cost = ( + _calculate_period_costs( + num_requests=request.num_requests_per_month, + cost_per_request=cost_per_request, + input_cost=input_cost, + output_cost=output_cost, + margin_cost=margin_cost, + ) + ) + + return CostEstimateResponse( + model=request.model, + input_tokens=request.input_tokens, + output_tokens=request.output_tokens, + num_requests_per_day=request.num_requests_per_day, + num_requests_per_month=request.num_requests_per_month, + cost_per_request=cost_per_request, + input_cost_per_request=input_cost, + output_cost_per_request=output_cost, + margin_cost_per_request=margin_cost, + daily_cost=daily_cost, + daily_input_cost=daily_input_cost, + daily_output_cost=daily_output_cost, + daily_margin_cost=daily_margin_cost, + monthly_cost=monthly_cost, + monthly_input_cost=monthly_input_cost, + monthly_output_cost=monthly_output_cost, + monthly_margin_cost=monthly_margin_cost, + input_cost_per_token=input_cost_per_token, + output_cost_per_token=output_cost_per_token, + provider=custom_llm_provider, + ) + diff --git a/tests/litellm/proxy/management_endpoints/test_cost_estimate_endpoint.py b/tests/litellm/proxy/management_endpoints/test_cost_estimate_endpoint.py new file mode 100644 index 00000000000..f2d8d87855b --- /dev/null +++ b/tests/litellm/proxy/management_endpoints/test_cost_estimate_endpoint.py @@ -0,0 +1,75 @@ +""" +Tests for the /cost/estimate endpoint in cost_tracking_settings.py +""" + +from unittest.mock import MagicMock, patch + +import pytest + +from litellm.proxy._types import CostEstimateRequest, CostEstimateResponse +from litellm.proxy.management_endpoints.cost_tracking_settings import estimate_cost + + +class TestCostEstimateEndpoint: + """Tests for the cost estimation endpoint.""" + + @pytest.mark.asyncio + async def test_estimate_cost_daily_and_monthly(self): + """ + Test that cost estimation calculates daily and monthly costs correctly. + """ + request = CostEstimateRequest( + model="gpt-4", + input_tokens=1000, + output_tokens=500, + num_requests_per_day=100, + num_requests_per_month=3000, + ) + + with patch( + "litellm.proxy.management_endpoints.cost_tracking_settings.completion_cost" + ) as mock_completion_cost: + mock_completion_cost.return_value = 0.06 + + with patch("litellm.get_model_info") as mock_get_model_info: + mock_get_model_info.return_value = { + "input_cost_per_token": 0.00003, + "output_cost_per_token": 0.00006, + "litellm_provider": "openai", + } + + response = await estimate_cost( + request=request, + user_api_key_dict=MagicMock(), + ) + + assert response.model == "gpt-4" + assert response.cost_per_request == 0.06 + assert response.daily_cost == pytest.approx(6.0) # 0.06 * 100 + assert response.monthly_cost == pytest.approx(180.0) # 0.06 * 3000 + + @pytest.mark.asyncio + async def test_estimate_cost_model_not_found(self): + """ + Test that 404 is raised when model cost calculation fails. + """ + request = CostEstimateRequest( + model="nonexistent-model", + input_tokens=1000, + output_tokens=500, + ) + + with patch( + "litellm.proxy.management_endpoints.cost_tracking_settings.completion_cost" + ) as mock_completion_cost: + mock_completion_cost.side_effect = Exception("Model not found in cost map") + + from fastapi import HTTPException + + with pytest.raises(HTTPException) as exc_info: + await estimate_cost( + request=request, + user_api_key_dict=MagicMock(), + ) + + assert exc_info.value.status_code == 404 From 196509cbb1bcb226971c6f6766b8c8ac5b2b6f3f Mon Sep 17 00:00:00 2001 From: Costa Tsaousis Date: Mon, 5 Jan 2026 05:24:24 -0600 Subject: [PATCH 22/97] feat(mcp): parallelize tool fetching from multiple MCP servers (#18627) * feat(mcp): parallelize tool fetching from multiple MCP servers Replace sequential tool fetching with asyncio.gather() to reduce client timeouts when using multiple MCP servers. Changes: - mcp_server_manager.py: list_tools() now fetches tools in parallel - server.py: _get_tools_from_mcp_servers() now fetches tools in parallel Real-world impact (7 MCP servers example): - Sequential: ~4.5+ seconds (exceeds typical 5-second client timeouts) - Parallel: ~1.2 seconds (max of all servers) Fixes #18626 * fix: copy oauth2_headers to avoid shared dict mutation in parallel tasks --- .../mcp_server/mcp_server_manager.py | 22 ++++++++++------- .../proxy/_experimental/mcp_server/server.py | 24 ++++++++++++------- 2 files changed, 30 insertions(+), 16 deletions(-) diff --git a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py index 15c41ecbd5d..a0867ed45af 100644 --- a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py +++ b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py @@ -644,14 +644,14 @@ class MCPServerManager: """ allowed_mcp_servers = await self.get_allowed_mcp_servers(user_api_key_auth) - list_tools_result: List[MCPTool] = [] verbose_logger.debug("SERVER MANAGER LISTING TOOLS") - for server_id in allowed_mcp_servers: + async def _fetch_server_tools(server_id: str) -> List[MCPTool]: + """Fetch tools from a single server with error handling.""" server = self.get_mcp_server_by_id(server_id) if server is None: verbose_logger.warning(f"MCP Server {server_id} not found") - continue + return [] # Get server-specific auth header if available server_auth_header = None @@ -669,15 +669,21 @@ class MCPServerManager: server=server, mcp_auth_header=server_auth_header, ) - list_tools_result.extend(tools) - verbose_logger.info( - f"Successfully fetched {len(tools)} tools from server {server.name}" - ) + return tools except Exception as e: verbose_logger.warning( f"Failed to list tools from server {server.name}: {str(e)}. Continuing with other servers." ) - # Continue with other servers instead of failing completely + return [] + + # Fetch tools from all servers in parallel + tasks = [_fetch_server_tools(server_id) for server_id in allowed_mcp_servers] + results = await asyncio.gather(*tasks) + + # Flatten results into single list + list_tools_result: List[MCPTool] = [ + tool for tools in results for tool in tools + ] verbose_logger.info( f"Successfully fetched {len(list_tools_result)} tools total from all servers" diff --git a/litellm/proxy/_experimental/mcp_server/server.py b/litellm/proxy/_experimental/mcp_server/server.py index e00fdbfb930..9c7001266f0 100644 --- a/litellm/proxy/_experimental/mcp_server/server.py +++ b/litellm/proxy/_experimental/mcp_server/server.py @@ -709,7 +709,8 @@ if MCP_AVAILABLE: extra_headers: Optional[Dict[str, str]] = None if server.auth_type == MCPAuth.oauth2: - extra_headers = oauth2_headers + # Copy to avoid mutating the original dict (important for parallel fetching) + extra_headers = oauth2_headers.copy() if oauth2_headers else None if server.extra_headers and raw_headers: if extra_headers is None: @@ -755,11 +756,10 @@ if MCP_AVAILABLE: # Decide whether to add prefix based on number of allowed servers add_prefix = not (len(allowed_mcp_servers) == 1) - # Get tools from each allowed server - all_tools = [] - for server in allowed_mcp_servers: + async def _fetch_and_filter_server_tools(server: MCPServer) -> List[MCPTool]: + """Fetch and filter tools from a single server with error handling.""" if server is None: - continue + return [] server_auth_header, extra_headers = _prepare_mcp_server_headers( server=server, @@ -786,16 +786,24 @@ if MCP_AVAILABLE: user_api_key_auth=user_api_key_auth, ) - all_tools.extend(filtered_tools) - verbose_logger.debug( f"Successfully fetched {len(tools)} tools from server {server.name}, {len(filtered_tools)} after filtering" ) + return filtered_tools except Exception as e: verbose_logger.exception( f"Error getting tools from server {server.name}: {str(e)}" ) - # Continue with other servers instead of failing completely + return [] + + # Fetch tools from all servers in parallel + tasks = [ + _fetch_and_filter_server_tools(server) for server in allowed_mcp_servers + ] + results = await asyncio.gather(*tasks) + + # Flatten results into single list + all_tools: List[MCPTool] = [tool for tools in results for tool in tools] verbose_logger.info( f"Successfully fetched {len(all_tools)} tools total from all MCP servers" From 935824e4f9425c20edb8e7bfd63f33dea23ddf93 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Mon, 5 Jan 2026 16:56:22 +0530 Subject: [PATCH 23/97] Fix : test_sap_chat[False] --- litellm/llms/base_llm/chat/transformation.py | 1 + 1 file changed, 1 insertion(+) diff --git a/litellm/llms/base_llm/chat/transformation.py b/litellm/llms/base_llm/chat/transformation.py index 1867abde310..b592c23846d 100644 --- a/litellm/llms/base_llm/chat/transformation.py +++ b/litellm/llms/base_llm/chat/transformation.py @@ -101,6 +101,7 @@ class BaseConfig(ABC): ), ) and v is not None + and not callable(v) # Filter out any callable objects including mocks } def get_json_schema_from_pydantic_object( From c0f74c994ad49b1daeec8114d061026329ae7fbc Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Mon, 5 Jan 2026 17:02:54 +0530 Subject: [PATCH 24/97] Fix: test_transform_request_image_extraction --- .../ollama/test_ollama_chat_transformation.py | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/tests/test_litellm/llms/ollama/test_ollama_chat_transformation.py b/tests/test_litellm/llms/ollama/test_ollama_chat_transformation.py index 24defc6a0ab..fc4a3e43573 100644 --- a/tests/test_litellm/llms/ollama/test_ollama_chat_transformation.py +++ b/tests/test_litellm/llms/ollama/test_ollama_chat_transformation.py @@ -216,10 +216,8 @@ class TestOllamaChatConfigResponseFormat: # Verify image was extracted to images list assert "images" in result["messages"][0] assert len(result["messages"][0]["images"]) == 1 - assert ( - result["messages"][0]["images"][0] - == "data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQ..." - ) + # Ollama expects pure base64 data without the data URL prefix + assert result["messages"][0]["images"][0] == "/9j/4AAQSkZJRgABAQAAAQ..." def test_transform_request_multiple_images_extraction(self): """Test extraction of multiple images from a single message""" @@ -263,12 +261,9 @@ class TestOllamaChatConfigResponseFormat: # Verify both images were extracted assert "images" in result["messages"][0] assert len(result["messages"][0]["images"]) == 2 - assert ( - result["messages"][0]["images"][0] == "data:image/jpeg;base64,image1data..." - ) - assert ( - result["messages"][0]["images"][1] == "data:image/png;base64,image2data..." - ) + # Ollama expects pure base64 data without the data URL prefix + assert result["messages"][0]["images"][0] == "image1data..." + assert result["messages"][0]["images"][1] == "image2data..." def test_transform_request_image_url_as_string(self): """Test handling of image_url as direct string (edge case)""" From 2d7f0a1351d0a22501430d8ddf9be4243815b71e Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Mon, 5 Jan 2026 17:15:12 +0530 Subject: [PATCH 25/97] fix: test_create_simple_string_input --- litellm/utils.py | 2 ++ tests/test_litellm/interactions/base_interactions_test.py | 8 +++++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/litellm/utils.py b/litellm/utils.py index fd1ee48850a..65d793fa12f 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -6915,6 +6915,8 @@ def get_valid_models( ################################ # init litellm_params ################################# + from litellm.types.router import LiteLLM_Params + if litellm_params is None: litellm_params = LiteLLM_Params(model="") if api_key is not None: diff --git a/tests/test_litellm/interactions/base_interactions_test.py b/tests/test_litellm/interactions/base_interactions_test.py index b7748a45f32..fee5758ab5e 100644 --- a/tests/test_litellm/interactions/base_interactions_test.py +++ b/tests/test_litellm/interactions/base_interactions_test.py @@ -52,7 +52,13 @@ class BaseInteractionsTest(ABC): if response.usage: # Usage is a dict in InteractionsAPIResponse if isinstance(response.usage, dict): - assert response.usage.get("input_tokens") is not None or response.usage.get("output_tokens") is not None + # Check for both possible key formats: input_tokens/output_tokens or total_input_tokens/total_output_tokens + assert ( + response.usage.get("input_tokens") is not None + or response.usage.get("output_tokens") is not None + or response.usage.get("total_input_tokens") is not None + or response.usage.get("total_output_tokens") is not None + ) else: # If it's an object, check attributes assert hasattr(response.usage, "input_tokens") or hasattr(response.usage, "output_tokens") From d3107ac61ac627dc1cdc11f013a907970b153e31 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Mon, 5 Jan 2026 17:37:01 +0530 Subject: [PATCH 26/97] revert litellm/litellm/llms/gemini/interactions/transformation.py --- .../gemini/interactions/transformation.py | 24 +------------------ 1 file changed, 1 insertion(+), 23 deletions(-) diff --git a/litellm/llms/gemini/interactions/transformation.py b/litellm/llms/gemini/interactions/transformation.py index 769d70473b9..d21775eb236 100644 --- a/litellm/llms/gemini/interactions/transformation.py +++ b/litellm/llms/gemini/interactions/transformation.py @@ -139,7 +139,7 @@ class GoogleAIStudioInteractionsConfig(BaseInteractionsAPIConfig): raw_response: httpx.Response, logging_obj: LiteLLMLoggingObj, ) -> InteractionsAPIResponse: - """Parse response and transform to OpenAI-compatible format.""" + """Parse response - it already matches our response type.""" try: logging_obj.post_call( original_response=raw_response.text, @@ -155,18 +155,6 @@ class GoogleAIStudioInteractionsConfig(BaseInteractionsAPIConfig): verbose_logger.debug("Google AI Interactions response: %s", raw_json) - # Transform usage to OpenAI format - if "usage" in raw_json and raw_json["usage"]: - usage = raw_json["usage"] - # Map Google's field names to OpenAI's field names - if "total_input_tokens" in usage: - usage["input_tokens"] = usage["total_input_tokens"] - if "total_output_tokens" in usage: - usage["output_tokens"] = usage["total_output_tokens"] - if "total_tokens" in usage: - # Keep total_tokens as is - it's the same in both formats - pass - response = InteractionsAPIResponse(**raw_json) response._hidden_params["headers"] = dict(raw_response.headers) response._hidden_params["additional_headers"] = process_response_headers(dict(raw_response.headers)) @@ -212,16 +200,6 @@ class GoogleAIStudioInteractionsConfig(BaseInteractionsAPIConfig): status_code=raw_response.status_code, headers=dict(raw_response.headers), ) - - # Transform usage to OpenAI format - if "usage" in raw_json and raw_json["usage"]: - usage = raw_json["usage"] - # Map Google's field names to OpenAI's field names - if "total_input_tokens" in usage: - usage["input_tokens"] = usage["total_input_tokens"] - if "total_output_tokens" in usage: - usage["output_tokens"] = usage["total_output_tokens"] - response = InteractionsAPIResponse(**raw_json) response._hidden_params["headers"] = dict(raw_response.headers) return response From bf1c5bef59d8ee6d69ed92eb0455fa62b2bbd687 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 5 Jan 2026 19:19:58 +0530 Subject: [PATCH 27/97] [UI] Add view for estimating costs across requests (#18645) * add estimate_cost endpoint * TestCostEstimateEndpoint * fix estimate_cost * add /cost/estimate to spend tracking routes * fix code QA checks * fixes endpoint * v0 cost estimator * v0 cost estimator * formatNumberWithCommas * fix admin view * docs * docs fix + export PDF/CSV * fixes for export --- .../docs/proxy/pricing_calculator.md | 142 +++++++++ docs/my-website/sidebars.js | 1 + .../cost_tracking_settings.tsx | 264 ++++++++++------- .../pricing_calculator/cost_results.tsx | 203 +++++++++++++ .../pricing_calculator/export_dropdown.tsx | 71 +++++ .../pricing_calculator/export_utils.ts | 276 ++++++++++++++++++ .../pricing_calculator/index.tsx | 31 ++ .../pricing_calculator/pricing_form.tsx | 104 +++++++ .../pricing_calculator/types.ts | 13 + .../pricing_calculator/use_cost_estimate.ts | 87 ++++++ .../components/CostTrackingSettings/types.ts | 31 ++ 11 files changed, 1111 insertions(+), 112 deletions(-) create mode 100644 docs/my-website/docs/proxy/pricing_calculator.md create mode 100644 ui/litellm-dashboard/src/components/CostTrackingSettings/pricing_calculator/cost_results.tsx create mode 100644 ui/litellm-dashboard/src/components/CostTrackingSettings/pricing_calculator/export_dropdown.tsx create mode 100644 ui/litellm-dashboard/src/components/CostTrackingSettings/pricing_calculator/export_utils.ts create mode 100644 ui/litellm-dashboard/src/components/CostTrackingSettings/pricing_calculator/index.tsx create mode 100644 ui/litellm-dashboard/src/components/CostTrackingSettings/pricing_calculator/pricing_form.tsx create mode 100644 ui/litellm-dashboard/src/components/CostTrackingSettings/pricing_calculator/types.ts create mode 100644 ui/litellm-dashboard/src/components/CostTrackingSettings/pricing_calculator/use_cost_estimate.ts diff --git a/docs/my-website/docs/proxy/pricing_calculator.md b/docs/my-website/docs/proxy/pricing_calculator.md new file mode 100644 index 00000000000..498db76f6c3 --- /dev/null +++ b/docs/my-website/docs/proxy/pricing_calculator.md @@ -0,0 +1,142 @@ +# Pricing Calculator (Cost Estimation) + +Estimate LLM costs based on expected token usage and request volume. This tool helps developers and platform teams forecast spending before deploying models to production. + +## When to Use This Feature + +Use the Pricing Calculator to: +- **Budget planning** - Estimate monthly costs before committing to a model +- **Model comparison** - Compare costs across different models for your use case +- **Capacity planning** - Understand cost implications of scaling request volume +- **Cost optimization** - Identify the most cost-effective model for your token requirements + +## Using the Pricing Calculator + +This walkthrough shows how to estimate LLM costs using the Pricing Calculator in the LiteLLM UI. + +### Step 1: Navigate to Settings + +From the LiteLLM dashboard, click on **Settings** in the left sidebar. + +![Click Settings](https://colony-recorder.s3.amazonaws.com/files/2026-01-05/183c437e-bda9-48b4-ab8f-95f023ba1146/ascreenshot_a1013487f545484194a9a4929eef4c49_text_export.jpeg) + +### Step 2: Open Cost Tracking + +Click on **Cost Tracking** to access the cost configuration options. + +![Click Cost Tracking](https://colony-recorder.s3.amazonaws.com/files/2026-01-05/05c92350-cbae-42ed-935b-e96a26003de8/ascreenshot_cc85f175a6664fc5be8dfdcc1759b442_text_export.jpeg) + +### Step 3: Open Pricing Calculator + +Click on **Pricing Calculator** to expand the calculator panel. This section allows you to estimate LLM costs based on expected token usage and request volume. + +![Click Pricing Calculator](https://colony-recorder.s3.amazonaws.com/files/2026-01-05/31ab5547-fa7d-4abd-b41a-7b4bbc0401f7/ascreenshot_f7f8b098ceba4b5199e5cbc60dddfd0a_text_export.jpeg) + +### Step 4: Select a Model + +Click the **Model** dropdown to select the model you want to estimate costs for. + +![Click Model field](https://colony-recorder.s3.amazonaws.com/files/2026-01-05/a6c236ce-3154-42a8-9701-120e3f7a017b/ascreenshot_635c61b832594e809f8ab79b5b3f32e1_text_export.jpeg) + +Choose a model from the list. The models shown are the ones configured on your LiteLLM proxy. + +![Select model](https://colony-recorder.s3.amazonaws.com/files/2026-01-05/96c4ebc4-1b88-4dea-b3b2-ea32fde36d9e/ascreenshot_7c2920f05a984ebbb530a8a85e669537_text_export.jpeg) + +### Step 5: Configure Token Counts + +Enter the expected **Input Tokens (per request)** - this is the average number of tokens in your prompts. + +![Click Input Tokens field](https://colony-recorder.s3.amazonaws.com/files/2026-01-05/d0b5ad8a-56e4-4f73-ac66-e1d728c81dc5/ascreenshot_42502082d6204a3891e0a2c3e89a1e38_text_export.jpeg) + +Enter the expected **Output Tokens (per request)** - this is the average number of tokens in model responses. + +![Click Output Tokens field](https://colony-recorder.s3.amazonaws.com/files/2026-01-05/d7481177-c63c-47f5-9316-1e87695f67f9/ascreenshot_8718cac4c0d14a82ab9f2b71795250c2_text_export.jpeg) + +### Step 6: Set Request Volume + +Enter your expected request volume. You can specify **Requests per Day** and/or **Requests per Month**. + +![Click Requests per Month field](https://colony-recorder.s3.amazonaws.com/files/2026-01-05/42270e11-93f1-41dc-b9c7-3bb6971ced31/ascreenshot_79f2ea9937b34e48ab1ff832ce7f7cb7_text_export.jpeg) + +For example, enter `10000000` for 10 million requests per month. + +![Enter request volume](https://colony-recorder.s3.amazonaws.com/files/2026-01-05/5e6c4338-ff87-44dd-9059-7577217fa3c8/ascreenshot_15c36610dc914536ac9446470eb39f05_text_export.jpeg) + +### Step 7: View Cost Estimates + +The calculator automatically updates as you change values. View the cost breakdown including: + +- **Per-Request Cost** - Total cost, input cost, output cost, and margin/fee per request +- **Daily Costs** - Aggregated costs if you specified requests per day +- **Monthly Costs** - Aggregated costs if you specified requests per month + +![View cost estimates](https://colony-recorder.s3.amazonaws.com/files/2026-01-05/4436cd11-df58-47cb-9742-c0d08865a61c/ascreenshot_f961298a4231464ea841bc4d184f731e_text_export.jpeg) + +### Step 8: Export the Report + +Click the **Export** button to download your cost estimate. You can export as: + +- **PDF** - Opens a print dialog to save as PDF (great for sharing with stakeholders) +- **CSV** - Downloads a spreadsheet-compatible file for further analysis + +## Cost Breakdown Details + +The Pricing Calculator shows: + +| Field | Description | +|-------|-------------| +| **Total Cost** | Complete cost including any configured margins | +| **Input Cost** | Cost for input/prompt tokens | +| **Output Cost** | Cost for output/completion tokens | +| **Margin/Fee** | Any configured [provider margins](/docs/proxy/provider_margins) | +| **Token Pricing** | Per-token rates (shown as $/1M tokens) | + +## API Endpoint + +You can also estimate costs programmatically using the `/cost/estimate` endpoint: + +```bash +curl -X POST "http://localhost:4000/cost/estimate" \ + -H "Authorization: Bearer sk-1234" \ + -H "Content-Type: application/json" \ + -d '{ + "model": "gpt-4", + "input_tokens": 1000, + "output_tokens": 500, + "num_requests_per_day": 1000, + "num_requests_per_month": 30000 + }' +``` + +**Response:** +```json +{ + "model": "gpt-4", + "input_tokens": 1000, + "output_tokens": 500, + "num_requests_per_day": 1000, + "num_requests_per_month": 30000, + "cost_per_request": 0.045, + "input_cost_per_request": 0.03, + "output_cost_per_request": 0.015, + "margin_cost_per_request": 0.0, + "daily_cost": 45.0, + "daily_input_cost": 30.0, + "daily_output_cost": 15.0, + "daily_margin_cost": 0.0, + "monthly_cost": 1350.0, + "monthly_input_cost": 900.0, + "monthly_output_cost": 450.0, + "monthly_margin_cost": 0.0, + "input_cost_per_token": 3e-05, + "output_cost_per_token": 6e-05, + "provider": "openai" +} +``` + +## Related Features + +- [Provider Margins](/docs/proxy/provider_margins) - Add fees or margins to LLM costs +- [Provider Discounts](/docs/proxy/provider_discounts) - Apply discounts to provider costs +- [Cost Tracking](/docs/proxy/cost_tracking) - Track and monitor LLM spend + diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js index cd25f293aca..3793aec037f 100644 --- a/docs/my-website/sidebars.js +++ b/docs/my-website/sidebars.js @@ -390,6 +390,7 @@ const sidebars = { items: [ "proxy/cost_tracking", "proxy/custom_pricing", + "proxy/pricing_calculator", "proxy/provider_margins", "proxy/provider_discounts", "proxy/sync_models_github", diff --git a/ui/litellm-dashboard/src/components/CostTrackingSettings/cost_tracking_settings.tsx b/ui/litellm-dashboard/src/components/CostTrackingSettings/cost_tracking_settings.tsx index 32ffd55efa0..3b9ea30e128 100644 --- a/ui/litellm-dashboard/src/components/CostTrackingSettings/cost_tracking_settings.tsx +++ b/ui/litellm-dashboard/src/components/CostTrackingSettings/cost_tracking_settings.tsx @@ -6,11 +6,13 @@ import ProviderDiscountTable from "./provider_discount_table"; import AddProviderForm from "./add_provider_form"; import ProviderMarginTable from "./provider_margin_table"; import AddMarginForm from "./add_margin_form"; +import PricingCalculator from "./pricing_calculator/index"; import { ExclamationCircleOutlined } from "@ant-design/icons"; import { DocsMenu } from "../HelpLink"; import HowItWorks from "./how_it_works"; import { useDiscountConfig } from "./use_discount_config"; import { useMarginConfig } from "./use_margin_config"; +import { fetchAvailableModels, ModelGroup } from "../playground/llm_calls/fetch_models"; const DOCS_LINKS = [ { label: "Custom pricing for models", href: "https://docs.litellm.ai/docs/proxy/custom_pricing" }, @@ -31,9 +33,12 @@ const CostTrackingSettings: React.FC = ({ const [marginType, setMarginType] = useState<"percentage" | "fixed">("percentage"); const [percentageValue, setPercentageValue] = useState(""); const [fixedAmountValue, setFixedAmountValue] = useState(""); + const [models, setModels] = useState([]); const [form] = Form.useForm(); const [marginForm] = Form.useForm(); const [modal, contextHolder] = Modal.useModal(); + + const isProxyAdmin = userRole === "proxy_admin" || userRole === "Admin"; // Use custom hooks for discount and margin config const { @@ -57,6 +62,17 @@ const CostTrackingSettings: React.FC = ({ Promise.all([fetchDiscountConfig(), fetchMarginConfig()]).finally(() => { setIsFetching(false); }); + + // Fetch models for pricing calculator (available to all roles) + const loadModels = async () => { + try { + const modelGroups = await fetchAvailableModels(accessToken); + setModels(modelGroups.map((m: ModelGroup) => m.model_group)); + } catch (error) { + console.error("Error fetching models:", error); + } + }; + loadModels(); } }, [accessToken, fetchDiscountConfig, fetchMarginConfig]); @@ -152,129 +168,153 @@ const CostTrackingSettings: React.FC = ({ {/* Main Content Card with Accordions */}
- {/* Accordion 1: Provider Discounts */} - - -
- Provider Discounts - - Apply percentage-based discounts to reduce costs for specific providers - -
-
- - - - Discounts - Test It - - - -
-
- -
- {isFetching ? ( -
- Loading configuration... -
- ) : Object.keys(discountConfig).length > 0 ? ( - - ) : ( -
- + +
+ Provider Discounts + + Apply percentage-based discounts to reduce costs for specific providers + +
+ + + + + Discounts + Test It + + + +
+
+
- )} -
-
- -
- -
-
-
-
-
- + {isFetching ? ( +
+ Loading configuration... +
+ ) : Object.keys(discountConfig).length > 0 ? ( + + ) : ( +
+ + + + + No provider discounts configured + + + Click "Add Provider Discount" to get started + +
+ )} +
+ + +
+ +
+
+ + + + + )} - {/* Accordion 2: Fee/Price Margin */} - + {/* Accordion 2: Fee/Price Margin - Only for proxy admins */} + {isProxyAdmin && ( + + +
+ Fee/Price Margin + + Add fees or margins to LLM costs for internal billing and cost recovery + +
+
+ +
+
+ +
+ {isFetching ? ( +
+ Loading configuration... +
+ ) : Object.keys(marginConfig).length > 0 ? ( + + ) : ( +
+ + + + + No provider margins configured + + + Click "Add Provider Margin" to get started + +
+ )} +
+
+
+ )} + + {/* Accordion 3: Pricing Calculator - Available to all roles */} +
- Fee/Price Margin + Pricing Calculator - Add fees or margins to LLM costs for internal billing and cost recovery + Estimate LLM costs based on expected token usage and request volume
-
- -
- {isFetching ? ( -
- Loading configuration... -
- ) : Object.keys(marginConfig).length > 0 ? ( - - ) : ( -
- - - - - No provider margins configured - - - Click "Add Provider Margin" to get started - -
- )} +
diff --git a/ui/litellm-dashboard/src/components/CostTrackingSettings/pricing_calculator/cost_results.tsx b/ui/litellm-dashboard/src/components/CostTrackingSettings/pricing_calculator/cost_results.tsx new file mode 100644 index 00000000000..03d5ca0e518 --- /dev/null +++ b/ui/litellm-dashboard/src/components/CostTrackingSettings/pricing_calculator/cost_results.tsx @@ -0,0 +1,203 @@ +import React from "react"; +import { Text } from "@tremor/react"; +import { Card, Statistic, Row, Col, Divider, Spin } from "antd"; +import { DollarOutlined, LoadingOutlined } from "@ant-design/icons"; +import { CostEstimateResponse } from "../types"; +import { formatNumberWithCommas } from "@/utils/dataUtils"; +import ExportDropdown from "./export_dropdown"; + +interface CostResultsProps { + result: CostEstimateResponse | null; + loading: boolean; +} + +const formatCost = (value: number | null | undefined): string => { + if (value === null || value === undefined) return "-"; + if (value === 0) return "$0"; + if (value < 0.0001) return `$${value.toExponential(2)}`; + if (value < 1) return `$${value.toFixed(4)}`; + return `$${formatNumberWithCommas(value, 2, true)}`; +}; + +const formatRequests = (value: number | null | undefined): string => { + if (value === null || value === undefined) return "-"; + return formatNumberWithCommas(value, 0, true); +}; + +const CostResults: React.FC = ({ result, loading }) => { + if (!result && !loading) { + return ( +
+ + Select a model to see cost estimates + +
+ ); + } + + if (loading && !result) { + return ( +
+ } /> + Calculating costs... +
+ ); + } + + if (!result) return null; + + return ( +
+ + +
+
+ Cost Estimate + + Model: {result.model} {result.provider && `(${result.provider})`} + +
+
+ {loading && } size="small" />} + +
+
+ + + + + } + /> + + + + + + + + + 0 ? "#faad14" : undefined, + }} + /> + + + + + {result.daily_cost !== null && ( + + + + } + /> + + + + + + + + + 0 ? "#faad14" : undefined, + }} + /> + + + + )} + + {result.monthly_cost !== null && ( + + + + } + /> + + + + + + + + + 0 ? "#faad14" : undefined, + }} + /> + + + + )} + + {(result.input_cost_per_token || result.output_cost_per_token) && ( +
+ Token Pricing: + {result.input_cost_per_token && ( + Input: ${formatNumberWithCommas(result.input_cost_per_token * 1_000_000, 2)}/1M tokens + )} + {result.input_cost_per_token && result.output_cost_per_token && " | "} + {result.output_cost_per_token && ( + Output: ${formatNumberWithCommas(result.output_cost_per_token * 1_000_000, 2)}/1M tokens + )} +
+ )} +
+ ); +}; + +export default CostResults; diff --git a/ui/litellm-dashboard/src/components/CostTrackingSettings/pricing_calculator/export_dropdown.tsx b/ui/litellm-dashboard/src/components/CostTrackingSettings/pricing_calculator/export_dropdown.tsx new file mode 100644 index 00000000000..e8a681021d6 --- /dev/null +++ b/ui/litellm-dashboard/src/components/CostTrackingSettings/pricing_calculator/export_dropdown.tsx @@ -0,0 +1,71 @@ +import React, { useState, useRef, useEffect } from "react"; +import { Button } from "@tremor/react"; +import { DownloadOutlined, FilePdfOutlined, FileExcelOutlined } from "@ant-design/icons"; +import { CostEstimateResponse } from "../types"; +import { exportToPDF, exportToCSV } from "./export_utils"; + +interface ExportDropdownProps { + result: CostEstimateResponse; +} + +const ExportDropdown: React.FC = ({ result }) => { + const [isOpen, setIsOpen] = useState(false); + const menuRef = useRef(null); + + useEffect(() => { + const handleClickOutside = (event: MouseEvent) => { + if (menuRef.current && !menuRef.current.contains(event.target as Node)) { + setIsOpen(false); + } + }; + + if (isOpen) { + document.addEventListener("mousedown", handleClickOutside); + } + + return () => { + document.removeEventListener("mousedown", handleClickOutside); + }; + }, [isOpen]); + + return ( +
+ + + {isOpen && ( +
+ + +
+ )} +
+ ); +}; + +export default ExportDropdown; + diff --git a/ui/litellm-dashboard/src/components/CostTrackingSettings/pricing_calculator/export_utils.ts b/ui/litellm-dashboard/src/components/CostTrackingSettings/pricing_calculator/export_utils.ts new file mode 100644 index 00000000000..e02e8288456 --- /dev/null +++ b/ui/litellm-dashboard/src/components/CostTrackingSettings/pricing_calculator/export_utils.ts @@ -0,0 +1,276 @@ +import { CostEstimateResponse } from "../types"; +import { formatNumberWithCommas } from "@/utils/dataUtils"; + +const formatCostForExport = (value: number | null | undefined): string => { + if (value === null || value === undefined) return "-"; + if (value === 0) return "$0.00"; + if (value < 0.01) return `$${value.toFixed(6)}`; + if (value < 1) return `$${value.toFixed(4)}`; + return `$${formatNumberWithCommas(value, 2)}`; +}; + +const formatRequestsForExport = (value: number | null | undefined): string => { + if (value === null || value === undefined) return "-"; + return formatNumberWithCommas(value, 0); +}; + +export const exportToPDF = (result: CostEstimateResponse): void => { + const printWindow = window.open("", "_blank"); + if (!printWindow) { + alert("Please allow popups to export PDF"); + return; + } + + const html = ` + + + + Cost Estimate Report - ${result.model} + + + +

LLM Cost Estimate Report

+ +
+

Model: ${result.model}

+ ${result.provider ? `

Provider: ${result.provider}

` : ""} +

Input Tokens per Request: ${formatRequestsForExport(result.input_tokens)}

+

Output Tokens per Request: ${formatRequestsForExport(result.output_tokens)}

+ ${result.num_requests_per_day ? `

Requests per Day: ${formatRequestsForExport(result.num_requests_per_day)}

` : ""} + ${result.num_requests_per_month ? `

Requests per Month: ${formatRequestsForExport(result.num_requests_per_month)}

` : ""} +
+ +

Per-Request Cost Breakdown

+ + + + + + + + + + + + + + + + + + + + + +
Cost TypeAmount
Input Cost${formatCostForExport(result.input_cost_per_request)}
Output Cost${formatCostForExport(result.output_cost_per_request)}
Margin/Fee${formatCostForExport(result.margin_cost_per_request)}
Total per Request${formatCostForExport(result.cost_per_request)}
+ + ${result.daily_cost !== null ? ` +

Daily Cost Estimate (${formatRequestsForExport(result.num_requests_per_day)} requests/day)

+ + + + + + + + + + + + + + + + + + + + + +
Cost TypeAmount
Input Cost${formatCostForExport(result.daily_input_cost)}
Output Cost${formatCostForExport(result.daily_output_cost)}
Margin/Fee${formatCostForExport(result.daily_margin_cost)}
Total Daily${formatCostForExport(result.daily_cost)}
+ ` : ""} + + ${result.monthly_cost !== null ? ` +

Monthly Cost Estimate (${formatRequestsForExport(result.num_requests_per_month)} requests/month)

+ + + + + + + + + + + + + + + + + + + + + +
Cost TypeAmount
Input Cost${formatCostForExport(result.monthly_input_cost)}
Output Cost${formatCostForExport(result.monthly_output_cost)}
Margin/Fee${formatCostForExport(result.monthly_margin_cost)}
Total Monthly${formatCostForExport(result.monthly_cost)}
+ ` : ""} + + ${result.input_cost_per_token || result.output_cost_per_token ? ` +

Token Pricing

+ + + + + + ${result.input_cost_per_token ? ` + + + + + ` : ""} + ${result.output_cost_per_token ? ` + + + + + ` : ""} +
Token TypePrice per 1M Tokens
Input Tokens$${(result.input_cost_per_token * 1000000).toFixed(2)}
Output Tokens$${(result.output_cost_per_token * 1000000).toFixed(2)}
+ ` : ""} + + + + + `; + + printWindow.document.write(html); + printWindow.document.close(); + printWindow.onload = () => { + printWindow.print(); + }; +}; + +export const exportToCSV = (result: CostEstimateResponse): void => { + const rows = [ + ["LLM Cost Estimate Report"], + [""], + ["Configuration"], + ["Model", result.model], + ["Provider", result.provider || "-"], + ["Input Tokens per Request", result.input_tokens.toString()], + ["Output Tokens per Request", result.output_tokens.toString()], + ["Requests per Day", result.num_requests_per_day?.toString() || "-"], + ["Requests per Month", result.num_requests_per_month?.toString() || "-"], + [""], + ["Per-Request Costs"], + ["Input Cost", result.input_cost_per_request.toString()], + ["Output Cost", result.output_cost_per_request.toString()], + ["Margin/Fee", result.margin_cost_per_request.toString()], + ["Total per Request", result.cost_per_request.toString()], + ]; + + if (result.daily_cost !== null) { + rows.push( + [""], + ["Daily Costs"], + ["Daily Input Cost", result.daily_input_cost?.toString() || "-"], + ["Daily Output Cost", result.daily_output_cost?.toString() || "-"], + ["Daily Margin/Fee", result.daily_margin_cost?.toString() || "-"], + ["Total Daily", result.daily_cost.toString()] + ); + } + + if (result.monthly_cost !== null) { + rows.push( + [""], + ["Monthly Costs"], + ["Monthly Input Cost", result.monthly_input_cost?.toString() || "-"], + ["Monthly Output Cost", result.monthly_output_cost?.toString() || "-"], + ["Monthly Margin/Fee", result.monthly_margin_cost?.toString() || "-"], + ["Total Monthly", result.monthly_cost.toString()] + ); + } + + if (result.input_cost_per_token || result.output_cost_per_token) { + rows.push( + [""], + ["Token Pricing (per 1M tokens)"], + ["Input Token Price", result.input_cost_per_token ? `$${(result.input_cost_per_token * 1000000).toFixed(2)}` : "-"], + ["Output Token Price", result.output_cost_per_token ? `$${(result.output_cost_per_token * 1000000).toFixed(2)}` : "-"] + ); + } + + const csv = rows.map(row => row.join(",")).join("\n"); + const blob = new Blob([csv], { type: "text/csv;charset=utf-8;" }); + const url = window.URL.createObjectURL(blob); + const a = document.createElement("a"); + a.href = url; + a.download = `cost_estimate_${result.model.replace(/\//g, "_")}_${new Date().toISOString().split("T")[0]}.csv`; + document.body.appendChild(a); + a.click(); + document.body.removeChild(a); + window.URL.revokeObjectURL(url); +}; + diff --git a/ui/litellm-dashboard/src/components/CostTrackingSettings/pricing_calculator/index.tsx b/ui/litellm-dashboard/src/components/CostTrackingSettings/pricing_calculator/index.tsx new file mode 100644 index 00000000000..fff6475e8a2 --- /dev/null +++ b/ui/litellm-dashboard/src/components/CostTrackingSettings/pricing_calculator/index.tsx @@ -0,0 +1,31 @@ +import React, { useCallback } from "react"; +import PricingForm from "./pricing_form"; +import CostResults from "./cost_results"; +import { useCostEstimate } from "./use_cost_estimate"; +import { PricingCalculatorProps, PricingFormValues } from "./types"; + +const PricingCalculator: React.FC = ({ + accessToken, + models, +}) => { + const { loading, result, debouncedFetch } = useCostEstimate(accessToken); + + const handleValuesChange = useCallback( + (_changedValues: Partial, allValues: PricingFormValues) => { + if (allValues.model) { + debouncedFetch(allValues); + } + }, + [debouncedFetch] + ); + + return ( +
+ + +
+ ); +}; + +export default PricingCalculator; + diff --git a/ui/litellm-dashboard/src/components/CostTrackingSettings/pricing_calculator/pricing_form.tsx b/ui/litellm-dashboard/src/components/CostTrackingSettings/pricing_calculator/pricing_form.tsx new file mode 100644 index 00000000000..c91a19e5516 --- /dev/null +++ b/ui/litellm-dashboard/src/components/CostTrackingSettings/pricing_calculator/pricing_form.tsx @@ -0,0 +1,104 @@ +import React from "react"; +import { Form, InputNumber, Select, Row, Col } from "antd"; +import { PricingFormValues } from "./types"; + +interface PricingFormProps { + models: string[]; + onValuesChange: (changedValues: Partial, allValues: PricingFormValues) => void; +} + +const PricingForm: React.FC = ({ models, onValuesChange }) => { + return ( +
+ + + + handleEntryChange(record.id, "model", value)} + optionFilterProp="label" + filterOption={(input, option) => + String(option?.label ?? "").toLowerCase().includes(input.toLowerCase()) + } + options={models.map((model) => ({ + value: model, + label: model, + }))} + style={{ width: "100%" }} + size="small" + /> + ), + }, + { + title: "Input Tokens", + dataIndex: "input_tokens", + key: "input_tokens", + width: "18%", + render: (_: number, record: ModelEntry) => ( + handleEntryChange(record.id, "input_tokens", value ?? 0)} + style={{ width: "100%" }} + size="small" + formatter={(value) => `${value}`.replace(/\B(?=(\d{3})+(?!\d))/g, ",")} + /> + ), + }, + { + title: "Output Tokens", + dataIndex: "output_tokens", + key: "output_tokens", + width: "18%", + render: (_: number, record: ModelEntry) => ( + handleEntryChange(record.id, "output_tokens", value ?? 0)} + style={{ width: "100%" }} + size="small" + formatter={(value) => `${value}`.replace(/\B(?=(\d{3})+(?!\d))/g, ",")} + /> + ), + }, + { + title: `Requests/${timePeriod === "day" ? "Day" : "Month"}`, + dataIndex: timePeriod === "day" ? "num_requests_per_day" : "num_requests_per_month", + key: "num_requests", + width: "20%", + render: (_: number | undefined, record: ModelEntry) => ( + + handleEntryChange( + record.id, + timePeriod === "day" ? "num_requests_per_day" : "num_requests_per_month", + value ?? undefined + ) + } + style={{ width: "100%" }} + size="small" + placeholder="-" + formatter={(value) => (value ? `${value}`.replace(/\B(?=(\d{3})+(?!\d))/g, ",") : "")} + /> + ), + }, + { + title: "", + key: "actions", + width: 50, + render: (_: unknown, record: ModelEntry) => ( +