From b11833c737fe329512828c086090ffeca8a53082 Mon Sep 17 00:00:00 2001 From: milan-berri Date: Wed, 3 Jun 2026 19:55:45 +0300 Subject: [PATCH 1/7] fix(key_generate): allow team members to create keys on org-scoped teams (#29310) * fix(key_generate): allow team members to create keys on org-scoped teams When a virtual key is created for a team, enterprise logic inherits the team's organization_id onto the key (add_team_organization_id). Since the VERIA-55 org-IDOR fix, /key/generate then required the caller to be an explicit LiteLLM_OrganizationMembership member of that org, returning 403 "Caller is not a member of organization_id=". Admins normally only add users to teams (not orgs), so self-serve key creation regressed for any user on an org-scoped team (regression since v1.84.0-rc.1). Skip the org-membership check when organization_id was inherited from the key's team (organization_id == team_table.organization_id). Team-level authorization already gates this path, so team membership is sufficient. The membership check still runs when a caller assigns an organization_id that did not come from the key's team, preserving the IDOR protection. Adds regression tests covering both the team-inherited (allowed) and foreign-org (still blocked) cases. Co-authored-by: Cursor * test(key_generate): cover mismatched team org IDOR path on generate Add test_generate_key_foreign_org_with_mismatched_team_still_enforces_membership for the case where a team is present but request organization_id differs from team_table.organization_id. Enterprise inheritance is no-op'd in the test so the guard is exercised directly; membership validation must still run. Addresses Greptile review on #29310. Co-authored-by: Cursor --------- Co-authored-by: Cursor --- .../key_management_endpoints.py | 7 +- .../test_key_management_endpoints.py | 243 ++++++++++++++++++ 2 files changed, 249 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index 80ded0bdd16..cf90f0661b3 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -894,7 +894,12 @@ async def _common_key_generation_helper( # noqa: PLR0915 user_api_key_dict.user_role is not None and user_api_key_dict.user_role == LitellmUserRoles.PROXY_ADMIN.value ) - if not _is_proxy_admin: + _org_inherited_from_team = ( + team_table is not None + and team_table.organization_id is not None + and data.organization_id == team_table.organization_id + ) + if not _is_proxy_admin and not _org_inherited_from_team: await _validate_caller_can_assign_key_org( user_api_key_dict=user_api_key_dict, organization_id=data.organization_id, diff --git a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py index 8fb242372e3..cda22da6ebd 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py @@ -3094,6 +3094,249 @@ async def test_generate_key_with_object_permission(): assert "object_permission" not in key_data +@pytest.mark.asyncio +async def test_generate_key_team_member_inherits_org_skips_membership_check(): + """Regression: a team member creating a key for an org-scoped team must not + be blocked by the org-membership check. + + When ``organization_id`` is inherited from the key's team (via + ``apply_enterprise_key_management_params`` -> ``add_team_organization_id``), + the caller already passed team-level authorization. Requiring an explicit + ``LiteLLM_OrganizationMembership`` row on top of that broke the normal admin + workflow (admins only add users to teams). This asserts the org-membership + check is skipped when the org id came from the caller's team. + """ + from unittest.mock import AsyncMock, MagicMock, patch + + from litellm.proxy._types import GenerateKeyRequest, LitellmUserRoles + from litellm.proxy.auth.user_api_key_auth import UserAPIKeyAuth + from litellm.proxy.management_endpoints.key_management_endpoints import ( + _common_key_generation_helper, + ) + + org_id = "org-from-team" + + # Team belongs to an org; caller is a team member but NOT an explicit member + # of that organization (the regression scenario). + mock_team_table = MagicMock() + mock_team_table.organization_id = org_id + mock_team_table.metadata = None + + mock_validate_org = AsyncMock() + mock_generate_key = AsyncMock( + return_value={ + "key": "sk-test-key", + "expires": None, + "user_id": "alice", + "team_id": "team-1", + } + ) + + with ( + patch("litellm.proxy.proxy_server.prisma_client", MagicMock()), + patch("litellm.proxy.proxy_server.llm_router", None), + patch("litellm.proxy.proxy_server.premium_user", True), + patch("litellm.proxy.proxy_server.litellm_proxy_admin_name", "admin"), + patch( + "litellm.proxy.management_endpoints.key_management_endpoints.validate_key_mcp_servers_against_team", + new_callable=AsyncMock, + ), + patch( + "litellm.proxy.management_endpoints.key_management_endpoints.validate_key_search_tools_against_team", + new_callable=AsyncMock, + ), + patch( + "litellm.proxy.management_endpoints.key_management_endpoints._validate_caller_can_assign_key_org", + mock_validate_org, + ), + patch( + "litellm.proxy.management_endpoints.key_management_endpoints.get_org_object", + new_callable=AsyncMock, + return_value=MagicMock(litellm_budget_table=None), + ), + patch( + "litellm.proxy.management_endpoints.key_management_endpoints._check_org_key_limits", + new_callable=AsyncMock, + ), + patch( + "litellm.proxy.management_endpoints.key_management_endpoints.generate_key_helper_fn", + mock_generate_key, + ), + ): + result = await _common_key_generation_helper( + data=GenerateKeyRequest( + user_id="alice", + team_id="team-1", + organization_id=org_id, + ), + user_api_key_dict=UserAPIKeyAuth( + user_id="alice", + user_role=LitellmUserRoles.INTERNAL_USER.value, + ), + litellm_changed_by=None, + team_table=mock_team_table, + ) + + # Key creation proceeded for the team member ... + mock_generate_key.assert_awaited_once() + assert result is not None + # ... and the org-membership check was bypassed because organization_id was + # inherited from the caller's team. + mock_validate_org.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_generate_key_foreign_org_without_team_still_enforces_membership(): + """VERIA-55: a caller assigning a key to an organization that was NOT + inherited from a team must still pass the org-membership check. + + This guards the IDOR fix: ``team_table is None`` (or an org id that does not + match the team) means the org id did not come from team context, so the + explicit membership validation must run. + """ + from unittest.mock import AsyncMock, MagicMock, patch + + from litellm.proxy._types import GenerateKeyRequest, LitellmUserRoles + from litellm.proxy.auth.user_api_key_auth import UserAPIKeyAuth + from litellm.proxy.management_endpoints.key_management_endpoints import ( + _common_key_generation_helper, + ) + + foreign_org_id = "someone-elses-org" + + mock_validate_org = AsyncMock() + mock_generate_key = AsyncMock( + return_value={ + "key": "sk-test-key", + "expires": None, + "user_id": "alice", + "team_id": None, + } + ) + + with ( + patch("litellm.proxy.proxy_server.prisma_client", MagicMock()), + patch("litellm.proxy.proxy_server.llm_router", None), + patch("litellm.proxy.proxy_server.premium_user", True), + patch("litellm.proxy.proxy_server.litellm_proxy_admin_name", "admin"), + patch( + "litellm.proxy.management_endpoints.key_management_endpoints._validate_caller_can_assign_key_org", + mock_validate_org, + ), + patch( + "litellm.proxy.management_endpoints.key_management_endpoints.get_org_object", + new_callable=AsyncMock, + return_value=MagicMock(litellm_budget_table=None), + ), + patch( + "litellm.proxy.management_endpoints.key_management_endpoints._check_org_key_limits", + new_callable=AsyncMock, + ), + patch( + "litellm.proxy.management_endpoints.key_management_endpoints.generate_key_helper_fn", + mock_generate_key, + ), + ): + await _common_key_generation_helper( + data=GenerateKeyRequest( + user_id="alice", + organization_id=foreign_org_id, + ), + user_api_key_dict=UserAPIKeyAuth( + user_id="alice", + user_role=LitellmUserRoles.INTERNAL_USER.value, + ), + litellm_changed_by=None, + team_table=None, + ) + + # No team context -> the org-membership check must still run. + mock_validate_org.assert_awaited_once() + assert mock_validate_org.call_args.kwargs["organization_id"] == foreign_org_id + + +@pytest.mark.asyncio +async def test_generate_key_foreign_org_with_mismatched_team_still_enforces_membership(): + """VERIA-55: when a team is present but its organization_id differs from the + organization_id on the key request, the org-membership check must still run.""" + from unittest.mock import AsyncMock, MagicMock, patch + + from litellm.proxy._types import GenerateKeyRequest, LitellmUserRoles + from litellm.proxy.auth.user_api_key_auth import UserAPIKeyAuth + from litellm.proxy.management_endpoints.key_management_endpoints import ( + _common_key_generation_helper, + ) + + team_org_id = "other-org" + foreign_org_id = "someone-elses-org" + + mock_team_table = MagicMock() + mock_team_table.organization_id = team_org_id + mock_team_table.metadata = None + + mock_validate_org = AsyncMock() + mock_generate_key = AsyncMock( + return_value={ + "key": "sk-test-key", + "expires": None, + "user_id": "alice", + "team_id": "team-1", + } + ) + + with ( + patch("litellm.proxy.proxy_server.prisma_client", MagicMock()), + patch("litellm.proxy.proxy_server.llm_router", None), + patch("litellm.proxy.proxy_server.premium_user", True), + patch("litellm.proxy.proxy_server.litellm_proxy_admin_name", "admin"), + patch( + "litellm.proxy.management_endpoints.key_management_endpoints.validate_key_mcp_servers_against_team", + new_callable=AsyncMock, + ), + patch( + "litellm.proxy.management_endpoints.key_management_endpoints.validate_key_search_tools_against_team", + new_callable=AsyncMock, + ), + patch( + "litellm_enterprise.proxy.management_endpoints.key_management_endpoints.apply_enterprise_key_management_params", + side_effect=lambda data, team_table: data, + ), + patch( + "litellm.proxy.management_endpoints.key_management_endpoints._validate_caller_can_assign_key_org", + mock_validate_org, + ), + patch( + "litellm.proxy.management_endpoints.key_management_endpoints.get_org_object", + new_callable=AsyncMock, + return_value=MagicMock(litellm_budget_table=None), + ), + patch( + "litellm.proxy.management_endpoints.key_management_endpoints._check_org_key_limits", + new_callable=AsyncMock, + ), + patch( + "litellm.proxy.management_endpoints.key_management_endpoints.generate_key_helper_fn", + mock_generate_key, + ), + ): + await _common_key_generation_helper( + data=GenerateKeyRequest( + user_id="alice", + team_id="team-1", + organization_id=foreign_org_id, + ), + user_api_key_dict=UserAPIKeyAuth( + user_id="alice", + user_role=LitellmUserRoles.INTERNAL_USER.value, + ), + litellm_changed_by=None, + team_table=mock_team_table, + ) + + mock_validate_org.assert_awaited_once() + assert mock_validate_org.call_args.kwargs["organization_id"] == foreign_org_id + + # ============================================ # Organization Key Limit Tests # ============================================ From f3e2167730a99b1d378315c9fc3feb283a5cf84f Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 3 Jun 2026 10:17:38 -0700 Subject: [PATCH 2/7] test(pass-through): move Gemini pass-through tests to gemini-3.1-flash-lite (#29595) * test(pass-through): move Gemini pass-through tests to gemini-3.1-flash-lite gemini-2.5-flash-lite is a generation behind and is slated for discontinuation on Vertex AI no earlier than October 16, 2026, so the pass-through suite was exercising an aging model. Every reference now points at gemini-3.1-flash-lite, which is GA and already priced in the cost map so the spend-logging assertions still compute a real cost test_vertex.test.js also gains jest.retryTimes(3) to match the sibling spend tests. The CI failures were intermittent 429 RESOURCE_EXHAUSTED from Vertex quota pressure, and that file was the only one without a retry, so a single rate-limited request was failing the whole job * test(pass-through): point Vertex tests at the global endpoint for gemini-3.1-flash-lite gemini-3.1-flash-lite is not served on the Vertex us-central1 regional endpoint for the CI project, so the Vertex pass-through tests were returning a deterministic 404 "Publisher Model ... was not found or your project does not have access to it" while the Gemini API tests passed. Move the Vertex clients to the global location, which the pass-through router maps to aiplatform.googleapis.com, where the 3.1 family is served --- .../test_gemini_with_spend.test.js | 4 ++-- tests/pass_through_tests/test_local_gemini.js | 4 ++-- tests/pass_through_tests/test_local_vertex.js | 4 ++-- tests/pass_through_tests/test_vertex.test.js | 11 +++++++---- tests/pass_through_tests/test_vertex_ai.py | 12 ++++++------ .../test_vertex_with_spend.test.js | 8 ++++---- 6 files changed, 23 insertions(+), 20 deletions(-) diff --git a/tests/pass_through_tests/test_gemini_with_spend.test.js b/tests/pass_through_tests/test_gemini_with_spend.test.js index 989bbc4b8e3..b9a25d3a3ed 100644 --- a/tests/pass_through_tests/test_gemini_with_spend.test.js +++ b/tests/pass_through_tests/test_gemini_with_spend.test.js @@ -32,7 +32,7 @@ describe('Gemini AI Tests', () => { }; const model = genAI.getGenerativeModel({ - model: 'gemini-2.5-flash-lite' + model: 'gemini-3.1-flash-lite' }, requestOptions); const prompt = 'Say "hello test" and nothing else'; @@ -83,7 +83,7 @@ describe('Gemini AI Tests', () => { }; const model = genAI.getGenerativeModel({ - model: 'gemini-2.5-flash-lite' + model: 'gemini-3.1-flash-lite' }, requestOptions); const prompt = 'Say "hello test" and nothing else'; diff --git a/tests/pass_through_tests/test_local_gemini.js b/tests/pass_through_tests/test_local_gemini.js index 0a72ca5cd7b..dc033a51f18 100644 --- a/tests/pass_through_tests/test_local_gemini.js +++ b/tests/pass_through_tests/test_local_gemini.js @@ -1,13 +1,13 @@ const { GoogleGenerativeAI, ModelParams, RequestOptions } = require("@google/generative-ai"); const modelParams = { - model: 'gemini-2.5-flash-lite', + model: 'gemini-3.1-flash-lite', }; const requestOptions = { baseUrl: 'http://127.0.0.1:4000/gemini', customHeaders: { - "tags": "gemini-js-sdk,gemini-2.5-flash-lite" + "tags": "gemini-js-sdk,gemini-3.1-flash-lite" } }; diff --git a/tests/pass_through_tests/test_local_vertex.js b/tests/pass_through_tests/test_local_vertex.js index 149635e2d6f..7cfe31db95b 100644 --- a/tests/pass_through_tests/test_local_vertex.js +++ b/tests/pass_through_tests/test_local_vertex.js @@ -4,7 +4,7 @@ const { VertexAI, RequestOptions } = require('@google-cloud/vertexai'); const vertexAI = new VertexAI({ project: 'litellm-ci-cd', - location: 'us-central1', + location: 'global', apiEndpoint: "127.0.0.1:4000/vertex-ai" }); @@ -20,7 +20,7 @@ const requestOptions = { }; const generativeModel = vertexAI.getGenerativeModel( - { model: 'gemini-2.5-flash-lite' }, + { model: 'gemini-3.1-flash-lite' }, requestOptions ); diff --git a/tests/pass_through_tests/test_vertex.test.js b/tests/pass_through_tests/test_vertex.test.js index 7b5edf6acd7..e0e879c2897 100644 --- a/tests/pass_through_tests/test_vertex.test.js +++ b/tests/pass_through_tests/test_vertex.test.js @@ -56,6 +56,9 @@ beforeAll(() => { loadVertexAiCredentials(); }); +// Configure Jest to retry flaky tests up to 3 times (useful for 429 rate limiting) +jest.retryTimes(3); + // Non-streaming Vertex generateContent can exceed 5s in CI / under load const VERTEX_TEST_TIMEOUT_MS = 30000; @@ -65,7 +68,7 @@ describe('Vertex AI Tests', () => { async () => { const vertexAI = new VertexAI({ project: 'litellm-ci-cd', - location: 'us-central1', + location: 'global', apiEndpoint: "localhost:4000/vertex-ai" }); @@ -78,7 +81,7 @@ describe('Vertex AI Tests', () => { }; const generativeModel = vertexAI.getGenerativeModel( - { model: 'gemini-2.5-flash-lite' }, + { model: 'gemini-3.1-flash-lite' }, requestOptions ); @@ -108,13 +111,13 @@ describe('Vertex AI Tests', () => { async () => { const vertexAI = new VertexAI({ project: 'litellm-ci-cd', - location: 'us-central1', + location: 'global', apiEndpoint: "localhost:4000/vertex-ai" }); const customHeaders = new Headers({"x-litellm-api-key": "sk-1234"}); const requestOptions = {customHeaders: customHeaders}; const generativeModel = vertexAI.getGenerativeModel( - {model: 'gemini-2.5-flash-lite'}, + {model: 'gemini-3.1-flash-lite'}, requestOptions ); const request = {contents: [{role: 'user', parts: [{text: 'What is 2+2?'}]}]}; diff --git a/tests/pass_through_tests/test_vertex_ai.py b/tests/pass_through_tests/test_vertex_ai.py index 73bf03c5000..bf1200489aa 100644 --- a/tests/pass_through_tests/test_vertex_ai.py +++ b/tests/pass_through_tests/test_vertex_ai.py @@ -103,12 +103,12 @@ async def test_basic_vertex_ai_pass_through_with_spendlog(): vertexai.init( project="litellm-ci-cd", - location="us-central1", + location="global", api_endpoint=f"{LITE_LLM_ENDPOINT}/vertex_ai", api_transport="rest", ) - model = GenerativeModel(model_name="gemini-2.5-flash-lite") + model = GenerativeModel(model_name="gemini-3.1-flash-lite") response = model.generate_content("hi") print("response", response) @@ -143,12 +143,12 @@ async def test_basic_vertex_ai_pass_through_streaming_with_spendlog(): vertexai.init( project="litellm-ci-cd", - location="us-central1", + location="global", api_endpoint=f"{LITE_LLM_ENDPOINT}/vertex_ai", api_transport="rest", ) - model = GenerativeModel(model_name="gemini-2.5-flash-lite") + model = GenerativeModel(model_name="gemini-3.1-flash-lite") response = model.generate_content("hi", stream=True) for chunk in response: @@ -182,7 +182,7 @@ async def test_vertex_ai_pass_through_endpoint_context_caching(): vertexai.init( project="litellm-ci-cd", - location="us-central1", + location="global", api_endpoint=f"{LITE_LLM_ENDPOINT}/vertex_ai", api_transport="rest", ) @@ -204,7 +204,7 @@ async def test_vertex_ai_pass_through_endpoint_context_caching(): ] cached_content = caching.CachedContent.create( - model_name="gemini-2.5-flash-lite-001", + model_name="gemini-3.1-flash-lite", system_instruction=system_instruction, contents=contents, ttl=datetime.timedelta(minutes=60), diff --git a/tests/pass_through_tests/test_vertex_with_spend.test.js b/tests/pass_through_tests/test_vertex_with_spend.test.js index 142a1cec8ff..4dee890dc78 100644 --- a/tests/pass_through_tests/test_vertex_with_spend.test.js +++ b/tests/pass_through_tests/test_vertex_with_spend.test.js @@ -71,7 +71,7 @@ describe('Vertex AI Tests', () => { test('should successfully generate non-streaming content with tags', async () => { const vertexAI = new VertexAI({ project: 'litellm-ci-cd', - location: 'us-central1', + location: 'global', apiEndpoint: "127.0.0.1:4000/vertex_ai" }); @@ -85,7 +85,7 @@ describe('Vertex AI Tests', () => { }; const generativeModel = vertexAI.getGenerativeModel( - { model: 'gemini-2.5-flash-lite' }, + { model: 'gemini-3.1-flash-lite' }, requestOptions ); @@ -130,7 +130,7 @@ describe('Vertex AI Tests', () => { test('should successfully generate streaming content with tags', async () => { const vertexAI = new VertexAI({ project: 'litellm-ci-cd', - location: 'us-central1', + location: 'global', apiEndpoint: "127.0.0.1:4000/vertex_ai" }); @@ -144,7 +144,7 @@ describe('Vertex AI Tests', () => { }; const generativeModel = vertexAI.getGenerativeModel( - { model: 'gemini-2.5-flash-lite' }, + { model: 'gemini-3.1-flash-lite' }, requestOptions ); From c7ab9adde5634932a42c0a3639bfe6067934ecfb Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Wed, 3 Jun 2026 23:31:51 +0530 Subject: [PATCH 3/7] Litellm oss staging 030626 (#29578) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Fix incorrect agent API request example payload structure (#29556) * fix(otel): add litellm_metadata fallback in _get_span_context and _end_proxy_span_from_kwargs (#29427) * fix(otel): add litellm_metadata fallback in _get_span_context and _end_proxy_span_from_kwargs On /v1/messages and other LITELLM_METADATA_ROUTES, the parent OTel span is stored in litellm_params['litellm_metadata'] instead of litellm_params['metadata']. When the request body contains a native 'metadata' field (e.g. Anthropic's {"user_id": "..."}), litellm_params['metadata'] gets overwritten and the parent span is lost, producing orphan root spans with a different trace_id. Add fallback checks to litellm_metadata in: - _get_span_context(): so child spans find the correct parent - _end_proxy_span_from_kwargs(): so the proxy span gets closed Fixes: https://github.com/BerriAI/litellm/issues/27934 * test(otel): tighten assertions per Greptile review - test_span_context_metadata_takes_priority: assert litellm_metadata span is never accessed, proving metadata takes priority - test_span_context_no_parent_when_neither_has_span: assert both ctx and detected_span are None --------- Co-authored-by: shin-berri Co-authored-by: yuneng-jiang Co-authored-by: Aneesh-Fiddler Co-authored-by: Sameer Kankute * fix: remove premature end-user budget check from get_end_user_object (#29420) * fix(proxy): remove premature end-user budget check from get_end_user_object Problem: - `_check_end_user_budget()` was called inside `get_end_user_object()` - This caused budget checks to run BEFORE `skip_budget_checks` could be evaluated - Zero-cost models (e.g., local vLLM) were incorrectly blocked when end-users exceeded their budget, even though they should bypass budget checks Solution: - Remove `_check_end_user_budget()` calls from `get_end_user_object()` - Budget enforcement now happens exclusively in `common_checks()` where `skip_budget_checks` context is available - `get_end_user_object()` keeps `route` as optional in function parameter for backwards compatibility and future implementation. * refactor(tests): update budget enforcement tests to reflect changes in get_end_user_object - test_get_end_user_object() verifies data fetching - test_check_end_user_budget() verifies enforcement - test_budget_enforcement_blocks_over_budget_users() integrates _check_end_user_budget() - test_resolve_end_user_reraises_budget_exceeded() is now test_resolve_end_user since no budget exceeded is thrown in get_end_user_object() * Gemini /images/generate and /images/edits billing fixes + add support for size and aspect ratio params (#29534) * Fix Gemini image config mapping * Address Gemini image config review * Format Gemini image generation transform * Fix Gemini image token usage logging * Share Gemini image request helpers * Fix Gemini Imagen model routing * Fixes as per self code review * Fixes per internal code review * Stop gating Imagen imageSize forwarding * Document Gemini image size mapping source * chore: retrigger lint * Clarify Gemini candidate count precedence * Add Inception provider (#29522) * add inception as provider (chat, fim) * linting * seperate test suite for chat and fim * fix test coverage * fix: model hub custom pricing model info (#29293) * Opik user auth key metadata extractors (#28397) * fix: enhance Opik metadata extraction to include user API key auth context fixed after refactoring to extractor logic * test: add unit tests for OPik metadata extraction logic * fix: enhance extract_opik_metadata function to prioritize metadata sources for improved accuracy * fix(ci): clarified comments and edited unit tests * test: add unit tests for OPik metadata extraction with auth and requester overrides * fix(ui): replace fixed favicon.ico with current api get /get_favicon (#29532) Signed-off-by: José Luis Di Biase * fix(vertex/gemini): keep tool_call reference when a text-only assistant message follows (#29561) `_gemini_convert_messages_with_history` tracks `last_message_with_tool_calls` so a following tool result can be matched back to its tool call. The assignment was inside a branch guarded by `assistant_msg.get("tool_calls", []) is not None`, which is also True for a text-only assistant message (an empty list is not None). As a result, an assistant message with no tool calls that appears between a tool call and its tool result overwrote the reference, and conversion failed with: Exception: Missing corresponding tool call for tool response message. This shape is common: a model emits a short narration/assistant message after a tool call before the tool result is appended. Only update `last_message_with_tool_calls` when the assistant message actually carries tool_calls (or a function_call). Adds a regression test. Co-authored-by: shin-berri Co-authored-by: yuneng-jiang Co-authored-by: Claude Opus 4.8 * Add 1-hour cache write pricing for EU/AU/JP Bedrock Anthropic models (#28572) * fix(thinking): handle None thinking param in is_thinking_enabled (#28598) Squash-merged by litellm-agent from Terrajlz's PR. * feat(helm): support tpl rendering in podAnnotations (#28609) Squash-merged by litellm-agent from devauxbr's PR. * Forward custom_llm_provider through the Responses API bridge (Fixes #28505) (#28575) * Forward custom_llm_provider through the Responses API bridge (Fixes #28505) When a Chat Completions request to a GPT-5.4+ model contains both `tools` and `reasoning_effort`, `completion()` auto-routes through `responses_api_bridge`. The bridge handler called `litellm.responses()` / `litellm.aresponses()` without forwarding the already-resolved `custom_llm_provider`, so the downstream call re-invoked `get_llm_provider()` with `custom_llm_provider=None` and stripped a second provider prefix from a `provider/provider/model` deployment string. For a deployment configured as `openai/openai/openai/gpt-5.5`, the bridge flow sent `openai/gpt-5.5` to the upstream API instead of the correct `openai/openai/gpt-5.5`. Upstream APIs that enforce model-name allow-lists rejected this as `key_model_access_denied`. Fix: pass the locally-resolved `custom_llm_provider` into both the sync `responses()` and async `aresponses()` calls so the downstream `_resolve_model_provider_for_responses` sees an explicit provider and skips the second prefix-strip. New regression test `tests/test_litellm/completion_extras/test_responses_bridge_provider_propagation.py` pins both call sites: each must forward `custom_llm_provider`. * fix(28505): set custom_llm_provider on request_data instead of as duplicate kwarg Greptile flagged that the previous patch passed custom_llm_provider as an explicit kwarg to responses()/aresponses() while request_data already carried it via the spread of sanitized_litellm_params, which would raise TypeError: got multiple values for keyword argument on every real bridge call. Switches to assigning request_data['custom_llm_provider'] before the call so the resolved provider wins over whatever sanitized_litellm_params spread in, without duplicating the kwarg. Updates the regression test to seed request_data with a sentinel custom_llm_provider so it actually exercises the overwrite path (the previous test mocked transform_request with a minimal dict and never hit the conflict). * chore: trigger shin-agent re-eval on retargeted staging base * chore: trigger shin-agent re-eval against updated Greptile state * Add 1-hour cache write pricing for EU/AU/JP Bedrock Anthropic models The 1-hour prompt-cache write tier (`cache_creation_input_token_cost_above_1hr`) was added to the us./global. variants of the Claude 4.5/4.6/4.7 family on Bedrock, but the eu./au./jp. cross-region inference profiles were left without it. AWS Bedrock pricing applies the same +10% regional premium across all geo profiles, so eu./au./jp. should carry the same 1-hour rates as us. (1.6x the 5-minute regional rate). Without these fields, cost tracking on EU/AU/JP Bedrock 1-hour-TTL prompt caching falls back to the 5-minute write rate and undercounts spend by ~60% for European, Australian, and Japanese tenants. Adds the 1-hour tier (and Sonnet 4.5's long-context >200K tier where AWS publishes one) to 14 regional Bedrock entries in both `model_prices_and_context_window.json` and the bundled `model_prices_and_context_window_backup.json`: - eu./au. Opus 4.6 ($11.00 / MTok) - eu./au. Opus 4.7 ($11.00 / MTok) - eu./au./jp. Sonnet 4.6 ($6.60 / MTok) - eu./au./jp. Sonnet 4.5 ($6.60 / MTok regular, $13.20 / MTok LC) - eu./au./jp. Haiku 4.5 ($2.20 / MTok) Also extends `tests/test_litellm/test_bedrock_anthropic_1hr_cache_pricing.py` with a `REGIONAL_EXPECTED` parametrized block covering all 13 new entries plus the existing 1.6x ratio invariant. Note: `eu.anthropic.claude-opus-4-5-20251101-v1:0` carries the wrong 5m rate today (base 6.25e-06 instead of regional 6.875e-06), which would break the 1.6x ratio check. It is intentionally left out of this PR so the scope stays "1-hour cache tier addition" — a separate follow-up should correct the EU 5m rates for Opus 4.5. --------- Co-authored-by: Terrajlz Co-authored-by: Bruno Devaux Co-authored-by: Sameer Kankute * Add 1-hour cache write pricing tier for Vertex AI Anthropic models (#28569) * fix(thinking): handle None thinking param in is_thinking_enabled (#28598) Squash-merged by litellm-agent from Terrajlz's PR. * feat(helm): support tpl rendering in podAnnotations (#28609) Squash-merged by litellm-agent from devauxbr's PR. * Forward custom_llm_provider through the Responses API bridge (Fixes #28505) (#28575) * Forward custom_llm_provider through the Responses API bridge (Fixes #28505) When a Chat Completions request to a GPT-5.4+ model contains both `tools` and `reasoning_effort`, `completion()` auto-routes through `responses_api_bridge`. The bridge handler called `litellm.responses()` / `litellm.aresponses()` without forwarding the already-resolved `custom_llm_provider`, so the downstream call re-invoked `get_llm_provider()` with `custom_llm_provider=None` and stripped a second provider prefix from a `provider/provider/model` deployment string. For a deployment configured as `openai/openai/openai/gpt-5.5`, the bridge flow sent `openai/gpt-5.5` to the upstream API instead of the correct `openai/openai/gpt-5.5`. Upstream APIs that enforce model-name allow-lists rejected this as `key_model_access_denied`. Fix: pass the locally-resolved `custom_llm_provider` into both the sync `responses()` and async `aresponses()` calls so the downstream `_resolve_model_provider_for_responses` sees an explicit provider and skips the second prefix-strip. New regression test `tests/test_litellm/completion_extras/test_responses_bridge_provider_propagation.py` pins both call sites: each must forward `custom_llm_provider`. * fix(28505): set custom_llm_provider on request_data instead of as duplicate kwarg Greptile flagged that the previous patch passed custom_llm_provider as an explicit kwarg to responses()/aresponses() while request_data already carried it via the spread of sanitized_litellm_params, which would raise TypeError: got multiple values for keyword argument on every real bridge call. Switches to assigning request_data['custom_llm_provider'] before the call so the resolved provider wins over whatever sanitized_litellm_params spread in, without duplicating the kwarg. Updates the regression test to seed request_data with a sentinel custom_llm_provider so it actually exercises the overwrite path (the previous test mocked transform_request with a minimal dict and never hit the conflict). * chore: trigger shin-agent re-eval on retargeted staging base * chore: trigger shin-agent re-eval against updated Greptile state * Add 1-hour cache write pricing tier for Vertex AI Anthropic models GCP Vertex AI publishes a separate 1-hour cache write column for the Claude family (1.6x the 5-minute write rate, matching the documented Bedrock ratio). LiteLLM's Vertex AI Anthropic entries only carry the 5-minute tier, so any request that uses `cache_control: {"ttl": "1h"}` on Vertex AI Claude is undercounted in cost tracking by ~60%. The runtime side already supports the 1-hour tier — `VertexAIAnthropicConfig` extends `AnthropicConfig`, populating `ephemeral_1h_input_tokens`, and `_calculate_cache_creation_cost` reads `cache_creation_input_token_cost_above_1hr`. Only the price registry was missing data. Adds the field to 19 vertex_ai/claude-* entries across both `model_prices_and_context_window.json` and the bundled `model_prices_and_context_window_backup.json`: - Haiku 4.5 ($1.25 -> $2.00 / MTok) - Sonnet 3.7 / 4 / 4.5 / 4.6 ($3.75 -> $6.00 / MTok) - Opus 4.5 / 4.6 / 4.7 ($6.25 -> $10.00 / MTok) - Opus 4 / 4.1 ($18.75 -> $30.00 / MTok) Adds `tests/test_litellm/test_vertex_anthropic_1hr_cache_pricing.py` mirroring the Bedrock equivalent — pins each (5m, 1h) pair per model and asserts the 1.6x ratio across the family. Fixes #27781. --------- Co-authored-by: Terrajlz Co-authored-by: Bruno Devaux Co-authored-by: Sameer Kankute * Fix Gemini multimodal function responses (#29325) Co-authored-by: shin-berri Co-authored-by: yuneng-jiang * address greptile review: add _transform_image_usage method and model-map supports_image_size flag - Add _transform_image_usage instance method to GoogleImageGenConfig that delegates to transform_gemini_image_usage, fixing the regression test - Replace hardcoded "2.5-flash" string check in supports_gemini_image_size with a get_model_info lookup on supports_image_size (default true) - Add supports_image_size: false to all gemini-2.5-flash model entries in model_prices_and_context_window.json so capability is controlled via the model map rather than embedded in code * fix test failures: schema validation, mypy type, model info plumbing, pricing test - Add supports_image_size to ModelInfoBase TypedDict so get_model_info surfaces it - Pass supports_image_size through _get_model_info_helper constructor call - Fix supports_gemini_image_size to use value is not False (None means unset, defaults to True) - Add supports_image_size to JSON schema in test_aaamodel_prices_and_context_window_json_is_valid - Correct gemini-3.1-flash-lite pricing assertions in test to match JSON values * Add Azure AI Kimi K2.6 metadata (#27052) * Add Azure AI Kimi K2.6 metadata * Scope Kimi metadata test cost map setup * fall back to substring check for models not in model_prices_and_context_window.json Models like gemini-2.5-flash-image-preview are not in the pricing JSON, so get_model_info raises. Fall back to "2.5-flash" not in model when the JSON has no explicit supports_image_size entry for the model. * fix(inception): don't forward global litellm.api_key to Inception FIM Match the Inception chat config: resolve only an Inception-specific key (param, litellm.inception_key, or INCEPTION_API_KEY) for the text-completion FIM path. The global litellm.api_key (often an OpenAI key) was both leaking to api.inceptionlabs.ai and taking precedence over the configured Inception key when set. * fix(auth): enforce end-user budget on custom-auth path that skips common_checks get_end_user_object() no longer raises BudgetExceededError, so custom-auth deployments with custom_auth_run_common_checks unset (which skip the centralized common_checks gate) stopped enforcing the end-user budget, letting an over-budget end user keep making requests. Re-enforce the budget in _run_post_custom_auth_checks on that path. --------- Signed-off-by: José Luis Di Biase Co-authored-by: Isha <72744901+IshaMeera@users.noreply.github.com> Co-authored-by: aneeshsangvikar Co-authored-by: shin-berri Co-authored-by: yuneng-jiang Co-authored-by: Aneesh-Fiddler Co-authored-by: Suleiman Elkhoury <108065141+suleimanelkhoury@users.noreply.github.com> Co-authored-by: Dmitriy Alergant <93501479+DmitriyAlergant@users.noreply.github.com> Co-authored-by: Yanis Miraoui Co-authored-by: Lovro Seder Co-authored-by: Thomas Mildner <12685945+Thomas-Mildner@users.noreply.github.com> Co-authored-by: José Luis Di Biase Co-authored-by: Lai Quang Huy <64073540+1qh@users.noreply.github.com> Co-authored-by: Claude Opus 4.8 Co-authored-by: Filippo Menghi <113345637+Cyberfilo@users.noreply.github.com> Co-authored-by: Terrajlz Co-authored-by: Bruno Devaux Co-authored-by: ZHONG Ziwen <67355585+zzw-math@users.noreply.github.com> Co-authored-by: Emerson Gomes Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- litellm/__init__.py | 17 + litellm/_lazy_imports_registry.py | 10 + .../handler.py | 15 + litellm/constants.py | 4 + litellm/integrations/opentelemetry.py | 14 + .../opik/opik_payload_builder/extractors.py | 24 +- .../get_llm_provider_logic.py | 10 + .../litellm_core_utils/llm_cost_calc/utils.py | 50 +- .../prompt_templates/factory.py | 12 +- litellm/llms/gemini/common_utils.py | 243 +++++- .../llms/gemini/image_edit/cost_calculator.py | 23 +- .../llms/gemini/image_edit/transformation.py | 67 +- .../gemini/image_generation/transformation.py | 99 +-- .../llms/gemini/image_usage_transformation.py | 73 ++ litellm/llms/inception/__init__.py | 0 litellm/llms/inception/chat/__init__.py | 0 litellm/llms/inception/chat/transformation.py | 54 ++ litellm/llms/inception/completion/__init__.py | 0 .../inception/completion/transformation.py | 43 ++ .../llms/vertex_ai/gemini/transformation.py | 14 +- litellm/main.py | 62 ++ ...odel_prices_and_context_window_backup.json | 703 +++++++++--------- litellm/proxy/agent_endpoints/endpoints.py | 122 ++- litellm/proxy/auth/auth_checks.py | 14 +- litellm/proxy/auth/user_api_key_auth.py | 12 +- litellm/router.py | 8 + litellm/types/images/main.py | 1 + litellm/types/llms/gemini.py | 7 +- litellm/types/llms/openai.py | 1 + litellm/types/llms/vertex_ai.py | 6 + litellm/types/utils.py | 3 + litellm/utils.py | 30 + model_prices_and_context_window.json | 141 +++- provider_endpoints_support.json | 18 + tests/llm_translation/test_gemini.py | 201 +++++ tests/proxy_unit_tests/test_auth_checks.py | 67 +- .../test_default_end_user_budget_simple.py | 28 +- tests/proxy_unit_tests/test_proxy_server.py | 2 + .../completion_extras/__init__.py | 0 ...t_responses_bridge_provider_propagation.py | 116 +++ .../integrations/opik/test_opik_extractors.py | 84 +++ .../integrations/test_opentelemetry.py | 135 +++- ...llm_core_utils_prompt_templates_factory.py | 45 +- .../test_azure_ai_kimi_k26_metadata.py | 76 ++ .../test_gemini_image_edit_transformation.py | 111 ++- .../llms/gemini/test_cost_calculator.py | 186 ++++- ..._gemini_image_generation_transformation.py | 240 ++++++ tests/test_litellm/llms/inception/__init__.py | 0 .../test_inception_chat_transformation.py | 326 ++++++++ ...est_inception_completion_transformation.py | 300 ++++++++ ...st_tool_call_followed_by_text_assistant.py | 57 ++ .../test_vertex_ai_gemini_transformation.py | 181 ++--- .../proxy/auth/test_auth_checks.py | 40 +- .../auth/test_custom_auth_end_user_budget.py | 85 ++- ...est_bedrock_anthropic_1hr_cache_pricing.py | 33 +- tests/test_litellm/test_cost_calculator.py | 10 +- tests/test_litellm/test_router.py | 68 ++ tests/test_litellm/test_utils.py | 1 + ui/litellm-dashboard/src/app/layout.tsx | 2 +- 59 files changed, 3534 insertions(+), 760 deletions(-) create mode 100644 litellm/llms/gemini/image_usage_transformation.py create mode 100644 litellm/llms/inception/__init__.py create mode 100644 litellm/llms/inception/chat/__init__.py create mode 100644 litellm/llms/inception/chat/transformation.py create mode 100644 litellm/llms/inception/completion/__init__.py create mode 100644 litellm/llms/inception/completion/transformation.py create mode 100644 tests/test_litellm/completion_extras/__init__.py create mode 100644 tests/test_litellm/completion_extras/test_responses_bridge_provider_propagation.py create mode 100644 tests/test_litellm/integrations/opik/test_opik_extractors.py create mode 100644 tests/test_litellm/llms/azure_ai/test_azure_ai_kimi_k26_metadata.py create mode 100644 tests/test_litellm/llms/gemini/test_gemini_image_generation_transformation.py create mode 100644 tests/test_litellm/llms/inception/__init__.py create mode 100644 tests/test_litellm/llms/inception/test_inception_chat_transformation.py create mode 100644 tests/test_litellm/llms/inception/test_inception_completion_transformation.py create mode 100644 tests/test_litellm/llms/vertex_ai/gemini/test_tool_call_followed_by_text_assistant.py diff --git a/litellm/__init__.py b/litellm/__init__.py index bae15f0362c..c954f5fd31e 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -278,6 +278,7 @@ ovhcloud_key: Optional[str] = None lemonade_key: Optional[str] = None sap_service_key: Optional[str] = None amazon_nova_api_key: Optional[str] = None +inception_key: Optional[str] = None common_cloud_provider_auth_params: dict = { "params": ["project", "region_name", "token"], "providers": ["vertex_ai", "bedrock", "watsonx", "azure", "vertex_ai_beta"], @@ -551,6 +552,7 @@ cohere_models: Set = set() cohere_chat_models: Set = set() mistral_chat_models: Set = set() text_completion_codestral_models: Set = set() +text_completion_inception_models: Set = set() anthropic_models: Set = set() openrouter_models: Set = set() datarobot_models: Set = set() @@ -628,6 +630,7 @@ publicai_models: Set = set() v0_models: Set = set() morph_models: Set = set() lambda_ai_models: Set = set() +inception_models: Set = set() hyperbolic_models: Set = set() black_forest_labs_models: Set = set() recraft_models: Set = set() @@ -792,6 +795,8 @@ def add_known_models(model_cost_map: Optional[Dict] = None): fireworks_ai_embedding_models.add(key) elif value.get("litellm_provider") == "text-completion-codestral": text_completion_codestral_models.add(key) + elif value.get("litellm_provider") == "text-completion-inception": + text_completion_inception_models.add(key) elif value.get("litellm_provider") == "xai": xai_models.add(key) elif value.get("litellm_provider") == "zai": @@ -878,6 +883,8 @@ def add_known_models(model_cost_map: Optional[Dict] = None): morph_models.add(key) elif value.get("litellm_provider") == "lambda_ai": lambda_ai_models.add(key) + elif value.get("litellm_provider") == "inception": + inception_models.add(key) elif value.get("litellm_provider") == "hyperbolic": hyperbolic_models.add(key) elif value.get("litellm_provider") == "black_forest_labs": @@ -980,6 +987,7 @@ model_list = list( | watsonx_models | gemini_models | text_completion_codestral_models + | text_completion_inception_models | xai_models | zai_models | fal_ai_models @@ -1018,6 +1026,7 @@ model_list = list( | v0_models | morph_models | lambda_ai_models + | inception_models | black_forest_labs_models | recraft_models | cometapi_models @@ -1074,6 +1083,7 @@ models_by_provider: dict = { "fireworks_ai": fireworks_ai_models | fireworks_ai_embedding_models, "aleph_alpha": aleph_alpha_models, "text-completion-codestral": text_completion_codestral_models, + "text-completion-inception": text_completion_inception_models, "xai": xai_models, "zai": zai_models, "fal_ai": fal_ai_models, @@ -1118,6 +1128,7 @@ models_by_provider: dict = { "v0": v0_models, "morph": morph_models, "lambda_ai": lambda_ai_models, + "inception": inception_models, "hyperbolic": hyperbolic_models, "black_forest_labs": black_forest_labs_models, "recraft": recraft_models, @@ -1869,6 +1880,9 @@ if TYPE_CHECKING: from .llms.codestral.completion.transformation import ( CodestralTextCompletionConfig as CodestralTextCompletionConfig, ) + from .llms.inception.completion.transformation import ( + InceptionTextCompletionConfig as InceptionTextCompletionConfig, + ) from .llms.azure.azure import ( AzureOpenAIAssistantsAPIConfig as AzureOpenAIAssistantsAPIConfig, ) @@ -1937,6 +1951,9 @@ if TYPE_CHECKING: from .llms.lambda_ai.chat.transformation import ( LambdaAIChatConfig as LambdaAIChatConfig, ) + from .llms.inception.chat.transformation import ( + InceptionChatConfig as InceptionChatConfig, + ) from .llms.hyperbolic.chat.transformation import ( HyperbolicChatConfig as HyperbolicChatConfig, ) diff --git a/litellm/_lazy_imports_registry.py b/litellm/_lazy_imports_registry.py index 17eb6609292..bdc3289b87c 100644 --- a/litellm/_lazy_imports_registry.py +++ b/litellm/_lazy_imports_registry.py @@ -267,6 +267,7 @@ LLM_CONFIG_NAMES = ( "AIMLChatConfig", "VolcEngineChatConfig", "CodestralTextCompletionConfig", + "InceptionTextCompletionConfig", "AzureOpenAIAssistantsAPIConfig", "HerokuChatConfig", "CometAPIConfig", @@ -310,6 +311,7 @@ LLM_CONFIG_NAMES = ( "MorphChatConfig", "RAGFlowConfig", "LambdaAIChatConfig", + "InceptionChatConfig", "HyperbolicChatConfig", "VercelAIGatewayConfig", "OVHCloudChatConfig", @@ -1040,6 +1042,10 @@ _LLM_CONFIGS_IMPORT_MAP = { ".llms.codestral.completion.transformation", "CodestralTextCompletionConfig", ), + "InceptionTextCompletionConfig": ( + ".llms.inception.completion.transformation", + "InceptionTextCompletionConfig", + ), "AzureOpenAIAssistantsAPIConfig": ( ".llms.azure.azure", "AzureOpenAIAssistantsAPIConfig", @@ -1154,6 +1160,10 @@ _LLM_CONFIGS_IMPORT_MAP = { "MorphChatConfig": (".llms.morph.chat.transformation", "MorphChatConfig"), "RAGFlowConfig": (".llms.ragflow.chat.transformation", "RAGFlowConfig"), "LambdaAIChatConfig": (".llms.lambda_ai.chat.transformation", "LambdaAIChatConfig"), + "InceptionChatConfig": ( + ".llms.inception.chat.transformation", + "InceptionChatConfig", + ), "HyperbolicChatConfig": ( ".llms.hyperbolic.chat.transformation", "HyperbolicChatConfig", diff --git a/litellm/completion_extras/litellm_responses_transformation/handler.py b/litellm/completion_extras/litellm_responses_transformation/handler.py index 2de7bda6467..87c26b776e8 100644 --- a/litellm/completion_extras/litellm_responses_transformation/handler.py +++ b/litellm/completion_extras/litellm_responses_transformation/handler.py @@ -182,6 +182,14 @@ class ResponsesToCompletionBridgeHandler: client=kwargs.get("client"), ) + # Pin the resolved provider so `responses()` doesn't re-run + # `get_llm_provider()` on the model string and strip a second + # provider prefix (see GitHub issue #28505). request_data already + # carries `custom_llm_provider` via the spread of + # `sanitized_litellm_params`; overwriting it on the dict (rather + # than adding an explicit kwarg) avoids the duplicate-keyword + # TypeError that would otherwise fire on the real bridge path. + request_data["custom_llm_provider"] = custom_llm_provider result = responses( **request_data, ) @@ -268,6 +276,13 @@ class ResponsesToCompletionBridgeHandler: except Exception as e: raise e + # Pin the resolved provider so `aresponses()` doesn't re-run + # `get_llm_provider()` on the model string and strip a second + # provider prefix (see GitHub issue #28505). Set on request_data + # rather than passed as a separate kwarg to avoid the duplicate- + # keyword TypeError when `sanitized_litellm_params` already + # carries `custom_llm_provider`. + request_data["custom_llm_provider"] = custom_llm_provider result = await aresponses( **request_data, aresponses=True, diff --git a/litellm/constants.py b/litellm/constants.py index df15050e652..26e25d0cef3 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -585,6 +585,7 @@ LITELLM_CHAT_PROVIDERS = [ "volcengine", "codestral", "text-completion-codestral", + "text-completion-inception", "deepseek", "sambanova", "maritalk", @@ -620,6 +621,7 @@ LITELLM_CHAT_PROVIDERS = [ "oci", "morph", "lambda_ai", + "inception", "vercel_ai_gateway", "wandb", "ovhcloud", @@ -779,6 +781,7 @@ openai_compatible_endpoints: List = [ "https://api.v0.dev/v1", "https://api.morphllm.com/v1", "https://api.lambda.ai/v1", + "https://api.inceptionlabs.ai/v1", "https://api.hyperbolic.xyz/v1", "https://ai-gateway.helicone.ai/", "https://ai-gateway.vercel.sh/v1", @@ -835,6 +838,7 @@ openai_compatible_providers: List = [ "helicone", "morph", "lambda_ai", + "inception", "hyperbolic", "vercel_ai_gateway", "aiml", diff --git a/litellm/integrations/opentelemetry.py b/litellm/integrations/opentelemetry.py index ce5cfa2f525..24780eb4bfc 100644 --- a/litellm/integrations/opentelemetry.py +++ b/litellm/integrations/opentelemetry.py @@ -1012,6 +1012,13 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): litellm_params = kwargs.get("litellm_params", {}) or {} _metadata = litellm_params.get("metadata", {}) or {} proxy_span = _metadata.get("litellm_parent_otel_span", None) + + # Fallback: check litellm_metadata (used by /v1/messages and other + # LITELLM_METADATA_ROUTES). + if proxy_span is None: + _litellm_metadata = litellm_params.get("litellm_metadata", {}) or {} + proxy_span = _litellm_metadata.get("litellm_parent_otel_span", None) + if ( proxy_span is not None and getattr(proxy_span, "name", None) == LITELLM_PROXY_REQUEST_SPAN_NAME @@ -2718,6 +2725,13 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): _metadata = litellm_params.get("metadata", {}) or {} parent_otel_span = _metadata.get("litellm_parent_otel_span", None) + # Fallback: check litellm_metadata (used by /v1/messages and other + # LITELLM_METADATA_ROUTES that store proxy-internal metadata + # separately from the provider's native "metadata" field). + if parent_otel_span is None: + _litellm_metadata = litellm_params.get("litellm_metadata", {}) or {} + parent_otel_span = _litellm_metadata.get("litellm_parent_otel_span", None) + # Priority 1: Explicit parent span from metadata if parent_otel_span is not None: verbose_logger.debug( diff --git a/litellm/integrations/opik/opik_payload_builder/extractors.py b/litellm/integrations/opik/opik_payload_builder/extractors.py index 9779ccddacf..1e3a664acc1 100644 --- a/litellm/integrations/opik/opik_payload_builder/extractors.py +++ b/litellm/integrations/opik/opik_payload_builder/extractors.py @@ -39,20 +39,32 @@ def extract_opik_metadata( standard_logging_metadata: Dict[str, Any], ) -> Dict[str, Any]: """ - Extract and merge Opik metadata from request and requester. + Merge Opik metadata from three sources in increasing priority order: + + 1. user_api_key_auth_metadata– lowest priority (operator-level defaults) + 2. litellm_metadata (request)– overrides auth-key defaults + 3. requester_metadata – highest priority (e.g. proxy header overrides) Args: - litellm_metadata: Metadata from litellm_params - standard_logging_metadata: Metadata from standard_logging_object + litellm_metadata: Metadata from litellm_params.mak + standard_logging_metadata: Metadata from standard_logging_object. Returns: - Merged Opik metadata dictionary + Merged Opik metadata dictionary. """ - opik_meta = litellm_metadata.get("opik", {}).copy() + # Start with auth-key defaults (lowest priority). + auth_meta = standard_logging_metadata.get("user_api_key_auth_metadata") or {} + opik_meta = (auth_meta.get("opik") or {}).copy() + # Request-level values override auth-key defaults. + request_opik = litellm_metadata.get("opik") or {} + opik_meta.update(request_opik) + + # Requester-level values win over everything else. requester_metadata = standard_logging_metadata.get("requester_metadata", {}) or {} requester_opik = requester_metadata.get("opik", {}) or {} - opik_meta.update(requester_opik) + if requester_opik: + opik_meta.update(requester_opik) _logging.verbose_logger.debug( f"litellm_opik_metadata - {json.dumps(opik_meta, default=str)}" diff --git a/litellm/litellm_core_utils/get_llm_provider_logic.py b/litellm/litellm_core_utils/get_llm_provider_logic.py index bb3f3fae9f0..a71000f00f8 100644 --- a/litellm/litellm_core_utils/get_llm_provider_logic.py +++ b/litellm/litellm_core_utils/get_llm_provider_logic.py @@ -373,6 +373,9 @@ def get_llm_provider( # noqa: PLR0915 elif endpoint == "https://api.lambda.ai/v1": custom_llm_provider = "lambda_ai" dynamic_api_key = get_secret_str("LAMBDA_API_KEY") + elif endpoint == "https://api.inceptionlabs.ai/v1": + custom_llm_provider = "inception" + dynamic_api_key = get_secret_str("INCEPTION_API_KEY") elif endpoint == "https://api.hyperbolic.xyz/v1": custom_llm_provider = "hyperbolic" dynamic_api_key = get_secret_str("HYPERBOLIC_API_KEY") @@ -954,6 +957,13 @@ def _get_openai_compatible_provider_info( # noqa: PLR0915 ) = litellm.LambdaAIChatConfig()._get_openai_compatible_provider_info( api_base, api_key ) + elif custom_llm_provider == "inception": + ( + api_base, + dynamic_api_key, + ) = litellm.InceptionChatConfig()._get_openai_compatible_provider_info( + api_base, api_key + ) elif custom_llm_provider == "hyperbolic": ( api_base, diff --git a/litellm/litellm_core_utils/llm_cost_calc/utils.py b/litellm/litellm_core_utils/llm_cost_calc/utils.py index 882561ed2e8..f39c942f90f 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/utils.py +++ b/litellm/litellm_core_utils/llm_cost_calc/utils.py @@ -34,6 +34,14 @@ _IMAGE_RESPONSE_CALL_TYPES = frozenset( _VALID_DATA_RESIDENCIES = frozenset(r.value for r in DataResidency) +def _get_token_detail_value(details: object, key: str) -> Optional[int]: + if isinstance(details, dict): + value = details.get(key) + else: + value = getattr(details, key, None) + return value if isinstance(value, int) else None + + def _is_above_128k(tokens: float) -> bool: if tokens > 128000: return True @@ -870,17 +878,47 @@ def calculate_image_response_cost_from_usage( cached_tokens=0, ) + output_tokens_details = getattr(usage, "completion_tokens_details", None) + if output_tokens_details is None: + output_tokens_details = getattr(usage, "output_tokens_details", None) + + if output_tokens_details is None: + completion_tokens_details = CompletionTokensDetailsWrapper( + text_tokens=0, + image_tokens=completion_tokens, + reasoning_tokens=0, + audio_tokens=0, + ) + else: + text_tokens = _get_token_detail_value(output_tokens_details, "text_tokens") or 0 + image_tokens = ( + _get_token_detail_value(output_tokens_details, "image_tokens") or 0 + ) + audio_tokens = ( + _get_token_detail_value(output_tokens_details, "audio_tokens") or 0 + ) + reasoning_tokens = ( + _get_token_detail_value(output_tokens_details, "reasoning_tokens") or 0 + ) + known_output_tokens = ( + text_tokens + image_tokens + audio_tokens + reasoning_tokens + ) + if completion_tokens > known_output_tokens: + text_tokens += completion_tokens - known_output_tokens + + completion_tokens_details = CompletionTokensDetailsWrapper( + text_tokens=text_tokens, + image_tokens=image_tokens, + reasoning_tokens=reasoning_tokens, + audio_tokens=audio_tokens, + ) + normalized_usage = Usage( prompt_tokens=prompt_tokens, completion_tokens=completion_tokens, total_tokens=total_tokens, prompt_tokens_details=prompt_tokens_details, - completion_tokens_details=CompletionTokensDetailsWrapper( - text_tokens=0, - image_tokens=completion_tokens, - reasoning_tokens=0, - audio_tokens=0, - ), + completion_tokens_details=completion_tokens_details, ) prompt_cost, completion_cost = generic_cost_per_token( diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index 46e9b43a429..1460dbaf0a9 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -1670,15 +1670,15 @@ def convert_to_gemini_tool_call_result( # noqa: PLR0915 if gemini_call_id: _function_response["id"] = gemini_call_id - # Create part with function_response, and optionally inline_data for images (Computer Use) _part: VertexPartType = {"function_response": _function_response} - # For Computer Use, if we have images/files, we need separate parts: - # - One part with function_response - # - One part per inline_data item - # Gemini's PartType is a oneof, so we can't have both in the same part + # For multimodal function responses, Gemini expects media parts nested + # inside functionResponse.parts instead of sibling content parts. if inline_data_list: - return [_part] + [{"inline_data": d} for d in inline_data_list] + _function_response["parts"] = [ + {"inline_data": inline_data} for inline_data in inline_data_list + ] + return [_part] return _part diff --git a/litellm/llms/gemini/common_utils.py b/litellm/llms/gemini/common_utils.py index bc963d62b5f..42a807983b9 100644 --- a/litellm/llms/gemini/common_utils.py +++ b/litellm/llms/gemini/common_utils.py @@ -1,6 +1,8 @@ import base64 import datetime -from typing import Any, Dict, List, Optional, Union +import json +import math +from typing import Any, Dict, List, Optional, Sequence, Union import httpx @@ -12,6 +14,245 @@ from litellm.secret_managers.main import get_secret_str from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import TokenCountResponse +GEMINI_IMAGE_ASPECT_RATIOS: Dict[str, float] = { + "1:1": 1 / 1, + "1:4": 1 / 4, + "1:8": 1 / 8, + "2:3": 2 / 3, + "3:2": 3 / 2, + "3:4": 3 / 4, + "4:1": 4 / 1, + "4:3": 4 / 3, + "4:5": 4 / 5, + "5:4": 5 / 4, + "8:1": 8 / 1, + "9:16": 9 / 16, + "16:9": 16 / 9, + "21:9": 21 / 9, +} + +# Supported aspect ratio dimensions from Google Gemini image generation docs: +# https://ai.google.dev/gemini-api/docs/image-generation#aspect_ratios_and_image_size +GEMINI_IMAGE_SIZE_TO_ASPECT_RATIO: Dict[tuple[int, int], str] = { + (512, 512): "1:1", + (1024, 1024): "1:1", + (2048, 2048): "1:1", + (4096, 4096): "1:1", + (256, 1024): "1:4", + (512, 2048): "1:4", + (1024, 4096): "1:4", + (2048, 8192): "1:4", + (192, 1536): "1:8", + (384, 3072): "1:8", + (768, 6144): "1:8", + (1536, 12288): "1:8", + (424, 632): "2:3", + (848, 1264): "2:3", + (1696, 2528): "2:3", + (3392, 5056): "2:3", + (632, 424): "3:2", + (1264, 848): "3:2", + (2528, 1696): "3:2", + (5056, 3392): "3:2", + (448, 600): "3:4", + (896, 1200): "3:4", + (1792, 2400): "3:4", + (3584, 4800): "3:4", + (1024, 256): "4:1", + (2048, 512): "4:1", + (4096, 1024): "4:1", + (8192, 2048): "4:1", + (600, 448): "4:3", + (1200, 896): "4:3", + (2400, 1792): "4:3", + (4800, 3584): "4:3", + (464, 576): "4:5", + (928, 1152): "4:5", + (1856, 2304): "4:5", + (3712, 4608): "4:5", + (576, 464): "5:4", + (1152, 928): "5:4", + (2304, 1856): "5:4", + (4608, 3712): "5:4", + (1536, 192): "8:1", + (3072, 384): "8:1", + (6144, 768): "8:1", + (12288, 1536): "8:1", + (384, 688): "9:16", + (768, 1376): "9:16", + (1536, 2752): "9:16", + (3072, 5504): "9:16", + (688, 384): "16:9", + (1376, 768): "16:9", + (2752, 1536): "16:9", + (5504, 3072): "16:9", + (792, 336): "21:9", + (1584, 672): "21:9", + (3168, 1344): "21:9", + (6336, 2688): "21:9", + (1280, 896): "4:3", + (896, 1280): "3:4", +} + + +def map_openai_size_to_gemini_image_config( + size: str, model: str +) -> Optional[Dict[str, str]]: + dimensions = _parse_openai_image_size(size) + if dimensions is None: + return None + + width, height = dimensions + image_config = { + "aspectRatio": _map_dimensions_to_gemini_aspect_ratio(width, height) + } + image_size = _map_dimensions_to_gemini_image_size(width, height) + if is_gemini_image_model(model): + if supports_gemini_image_size(model): + image_config["imageSize"] = image_size + else: + image_config["imageSize"] = image_size + return image_config + + +def supports_gemini_image_size(model: str) -> bool: + try: + model_info = litellm.get_model_info(model=model) + value = model_info.get("supports_image_size") + if value is not None: + return bool(value) + except Exception: + pass + return "2.5-flash" not in model + + +def is_gemini_image_model(model: str) -> bool: + base_model = model.split("/", 1)[-1] + return "gemini" in base_model + + +def map_openai_image_params_to_gemini( + params: Dict[str, Any], + model: str, + supported_params: Sequence[str], + optional_params: Optional[Dict[str, Any]] = None, + parse_image_config_string: bool = False, +) -> Dict[str, Any]: + optional_params = optional_params or {} + filtered_params = { + key: value for key, value in params.items() if key in supported_params + } + + mapped_params: Dict[str, Any] = {} + + if "n" in filtered_params and "n" not in optional_params: + mapped_params["sampleCount"] = filtered_params["n"] + + if "size" in filtered_params and "size" not in optional_params: + image_config = map_openai_size_to_gemini_image_config( + filtered_params["size"], + model, + ) + if image_config is not None: + if is_gemini_image_model(model): + mapped_params["imageConfig"] = image_config + else: + mapped_params["aspectRatio"] = image_config["aspectRatio"] + if "imageSize" in image_config: + mapped_params["imageSize"] = image_config["imageSize"] + + image_config_param = filtered_params.get("imageConfig") + if isinstance(image_config_param, str) and parse_image_config_string: + try: + image_config_param = json.loads(image_config_param) + except json.JSONDecodeError as exc: + raise litellm.UnsupportedParamsError( + model=model, + message="`imageConfig` must be valid JSON when provided as a string.", + ) from exc + if isinstance(image_config_param, dict): + mapped_params["imageConfig"] = image_config_param + + for key, value in filtered_params.items(): + if key not in ("n", "size", "imageConfig") and key not in optional_params: + mapped_params[key] = value + + return mapped_params + + +def get_gemini_image_generation_config( + model: str, + optional_params: Dict[str, Any], +) -> Dict[str, Any]: + generation_config: Dict[str, Any] = {"response_modalities": ["IMAGE", "TEXT"]} + + image_config: Dict[str, Any] = {} + if isinstance(optional_params.get("imageConfig"), dict): + image_config.update(optional_params["imageConfig"]) + + if not supports_gemini_image_size(model): + image_config.pop("imageSize", None) + + if image_config: + generation_config["imageConfig"] = image_config + + candidate_count = next( + ( + optional_params[key] + for key in ("candidateCount", "candidate_count", "sampleCount", "n") + if optional_params.get(key) is not None + ), + None, + ) + if candidate_count is not None: + generation_config["candidateCount"] = candidate_count + + return generation_config + + +def _parse_openai_image_size(size: str) -> Optional[tuple[int, int]]: + if size == "auto": + return None + + width_str, separator, height_str = size.lower().partition("x") + if not separator: + return None + + try: + width = int(width_str) + height = int(height_str) + except ValueError: + return None + + if width <= 0 or height <= 0: + return None + + return width, height + + +def _map_dimensions_to_gemini_aspect_ratio(width: int, height: int) -> str: + if (width, height) in GEMINI_IMAGE_SIZE_TO_ASPECT_RATIO: + return GEMINI_IMAGE_SIZE_TO_ASPECT_RATIO[(width, height)] + + requested_ratio = width / height + return min( + GEMINI_IMAGE_ASPECT_RATIOS, + key=lambda aspect_ratio: abs( + math.log(GEMINI_IMAGE_ASPECT_RATIOS[aspect_ratio] / requested_ratio) + ), + ) + + +def _map_dimensions_to_gemini_image_size(width: int, height: int) -> str: + effective_square_side = math.sqrt(width * height) + if effective_square_side < 768: + return "512" + if effective_square_side < 1536: + return "1K" + if effective_square_side < 3072: + return "2K" + return "4K" + class GeminiError(BaseLLMException): pass diff --git a/litellm/llms/gemini/image_edit/cost_calculator.py b/litellm/llms/gemini/image_edit/cost_calculator.py index 2e332a7fc00..956edb849a0 100644 --- a/litellm/llms/gemini/image_edit/cost_calculator.py +++ b/litellm/llms/gemini/image_edit/cost_calculator.py @@ -4,8 +4,9 @@ Gemini Image Edit Cost Calculator from typing import Any -import litellm -from litellm.types.utils import ImageResponse +from litellm.llms.gemini.image_generation.cost_calculator import ( + cost_calculator as image_generation_cost_calculator, +) def cost_calculator( @@ -15,20 +16,10 @@ def cost_calculator( """ Gemini image edit cost calculator. - Mirrors image generation pricing: charge per returned image based on - model metadata (`output_cost_per_image`). + Gemini image edits and generations share image response billing behavior: + use provider token usage when present, otherwise fall back to per-image pricing. """ - model_info = litellm.get_model_info( + return image_generation_cost_calculator( model=model, - custom_llm_provider="gemini", + image_response=image_response, ) - - output_cost_per_image: float = model_info.get("output_cost_per_image") or 0.0 - - if not isinstance(image_response, ImageResponse): - raise ValueError( - f"image_response must be of type ImageResponse got type={type(image_response)}" - ) - - num_images = len(image_response.data or []) - return output_cost_per_image * num_images diff --git a/litellm/llms/gemini/image_edit/transformation.py b/litellm/llms/gemini/image_edit/transformation.py index c8aaab0e14e..2316361d6e7 100644 --- a/litellm/llms/gemini/image_edit/transformation.py +++ b/litellm/llms/gemini/image_edit/transformation.py @@ -7,10 +7,22 @@ from httpx._types import RequestFiles from litellm.images.utils import ImageEditRequestUtils from litellm.llms.base_llm.image_edit.transformation import BaseImageEditConfig +from litellm.llms.gemini.common_utils import ( + get_gemini_image_generation_config, + map_openai_image_params_to_gemini, +) +from litellm.llms.gemini.image_usage_transformation import ( + transform_gemini_image_usage, +) from litellm.secret_managers.main import get_secret_str from litellm.types.images.main import ImageEditOptionalRequestParams from litellm.types.router import GenericLiteLLMParams -from litellm.types.utils import FileTypes, ImageObject, ImageResponse, OpenAIImage +from litellm.types.utils import ( + FileTypes, + ImageObject, + ImageResponse, + OpenAIImage, +) if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj @@ -22,7 +34,7 @@ else: class GeminiImageEditConfig(BaseImageEditConfig): DEFAULT_BASE_URL: str = "https://generativelanguage.googleapis.com/v1beta" - SUPPORTED_PARAMS: List[str] = ["size"] + SUPPORTED_PARAMS: List[str] = ["n", "size", "imageConfig"] def get_supported_openai_params(self, model: str) -> List[str]: return list(self.SUPPORTED_PARAMS) @@ -33,21 +45,12 @@ class GeminiImageEditConfig(BaseImageEditConfig): model: str, drop_params: bool, ) -> Dict[str, Any]: - supported_params = self.get_supported_openai_params(model) - filtered_params = { - key: value - for key, value in image_edit_optional_params.items() - if key in supported_params - } - - mapped_params: Dict[str, Any] = {} - - if "size" in filtered_params: - mapped_params["aspectRatio"] = self._map_size_to_aspect_ratio( - filtered_params["size"] # type: ignore[arg-type] - ) - - return mapped_params + return map_openai_image_params_to_gemini( + params=image_edit_optional_params, # type: ignore[arg-type] + model=model, + supported_params=self.get_supported_openai_params(model), + parse_image_config_string=True, + ) def validate_environment( self, @@ -107,18 +110,10 @@ class GeminiImageEditConfig(BaseImageEditConfig): request_body: Dict[str, Any] = {"contents": contents} - generation_config: Dict[str, Any] = {} - - if "aspectRatio" in image_edit_optional_request_params: - # Move aspectRatio into imageConfig inside generationConfig - if "imageConfig" not in generation_config: - generation_config["imageConfig"] = {} - generation_config["imageConfig"]["aspectRatio"] = ( - image_edit_optional_request_params["aspectRatio"] - ) - - if generation_config: - request_body["generationConfig"] = generation_config + request_body["generationConfig"] = get_gemini_image_generation_config( + model=model, + optional_params=image_edit_optional_request_params, + ) empty_files = cast(RequestFiles, []) return request_body, empty_files @@ -156,18 +151,12 @@ class GeminiImageEditConfig(BaseImageEditConfig): ) model_response.data = cast(List[OpenAIImage], data_list) + if "usageMetadata" in response_json: + model_response.usage = transform_gemini_image_usage( + response_json["usageMetadata"] + ) return model_response - def _map_size_to_aspect_ratio(self, size: str) -> str: - aspect_ratio_map = { - "1024x1024": "1:1", - "1792x1024": "16:9", - "1024x1792": "9:16", - "1280x896": "4:3", - "896x1280": "3:4", - } - return aspect_ratio_map.get(size, "1:1") - def _prepare_inline_image_parts( self, image: Union[FileTypes, List[FileTypes]] ) -> List[Dict[str, Any]]: diff --git a/litellm/llms/gemini/image_generation/transformation.py b/litellm/llms/gemini/image_generation/transformation.py index 9c4cd008b8c..e6770a76bcb 100644 --- a/litellm/llms/gemini/image_generation/transformation.py +++ b/litellm/llms/gemini/image_generation/transformation.py @@ -5,18 +5,21 @@ import httpx from litellm.llms.base_llm.image_generation.transformation import ( BaseImageGenerationConfig, ) +from litellm.llms.gemini.common_utils import ( + get_gemini_image_generation_config, + is_gemini_image_model, + map_openai_image_params_to_gemini, +) +from litellm.llms.gemini.image_usage_transformation import ( + transform_gemini_image_usage, +) from litellm.secret_managers.main import get_secret_str from litellm.types.llms.gemini import GeminiImageGenerationRequest from litellm.types.llms.openai import ( AllMessageValues, OpenAIImageGenerationOptionalParams, ) -from litellm.types.utils import ( - ImageObject, - ImageResponse, - ImageUsage, - ImageUsageInputTokensDetails, -) +from litellm.types.utils import ImageObject, ImageResponse if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj @@ -36,7 +39,10 @@ class GoogleImageGenConfig(BaseImageGenerationConfig): Google AI Imagen API supported parameters https://ai.google.dev/gemini-api/docs/imagen """ - return ["n", "size"] + supported_params = ["n", "size"] + if is_gemini_image_model(model): + supported_params.append("imageConfig") + return supported_params # type: ignore[return-value] def map_openai_params( self, @@ -45,64 +51,11 @@ class GoogleImageGenConfig(BaseImageGenerationConfig): model: str, drop_params: bool, ) -> dict: - supported_params = self.get_supported_openai_params(model) - mapped_params = {} - - for k, v in non_default_params.items(): - if k not in optional_params.keys(): - if k in supported_params: - # Map OpenAI parameters to Google format - if k == "n": - mapped_params["sampleCount"] = v - elif k == "size": - # Map OpenAI size format to Google aspectRatio - mapped_params["aspectRatio"] = self._map_size_to_aspect_ratio(v) - else: - mapped_params[k] = v - return mapped_params - - def _map_size_to_aspect_ratio(self, size: str) -> str: - """ - https://ai.google.dev/gemini-api/docs/image-generation - - """ - aspect_ratio_map = { - "1024x1024": "1:1", - "1792x1024": "16:9", - "1024x1792": "9:16", - "1280x896": "4:3", - "896x1280": "3:4", - } - return aspect_ratio_map.get(size, "1:1") - - def _transform_image_usage(self, usage_metadata: dict) -> ImageUsage: - """ - Transform Gemini usageMetadata to ImageUsage format - """ - input_tokens_details = ImageUsageInputTokensDetails( - image_tokens=0, - text_tokens=0, - ) - - # Extract detailed token counts from promptTokensDetails - tokens_details = usage_metadata.get("promptTokensDetails", []) - for details in tokens_details: - if isinstance(details, dict): - modality = str(details.get("modality", "")).upper() - raw_token_count = details.get( - "tokenCount", details.get("token_count", 0) - ) - token_count = raw_token_count if isinstance(raw_token_count, int) else 0 - if modality == "TEXT": - input_tokens_details.text_tokens += token_count - elif modality == "IMAGE": - input_tokens_details.image_tokens += token_count - - return ImageUsage( - input_tokens=usage_metadata.get("promptTokenCount", 0), - input_tokens_details=input_tokens_details, - output_tokens=usage_metadata.get("candidatesTokenCount", 0), - total_tokens=usage_metadata.get("totalTokenCount", 0), + return map_openai_image_params_to_gemini( + params=non_default_params, + model=model, + supported_params=self.get_supported_openai_params(model), + optional_params=optional_params, ) def get_complete_url( @@ -127,7 +80,7 @@ class GoogleImageGenConfig(BaseImageGenerationConfig): complete_url = complete_url.rstrip("/") # Gemini Flash Image Preview models use generateContent endpoint - if "gemini" in model: + if is_gemini_image_model(model): complete_url = f"{complete_url}/models/{model}:generateContent" else: # All other Imagen models use predict endpoint @@ -179,10 +132,13 @@ class GoogleImageGenConfig(BaseImageGenerationConfig): } """ # For Gemini Flash Image Preview models, use standard Gemini format - if "gemini" in model: + if is_gemini_image_model(model): request_body: dict = { "contents": [{"parts": [{"text": prompt}]}], - "generationConfig": {"response_modalities": ["IMAGE", "TEXT"]}, + "generationConfig": get_gemini_image_generation_config( + model=model, + optional_params=optional_params, + ), } return request_body else: @@ -200,6 +156,9 @@ class GoogleImageGenConfig(BaseImageGenerationConfig): ) return request_body_obj.model_dump(exclude_none=True) + def _transform_image_usage(self, usage_metadata: dict): + return transform_gemini_image_usage(usage_metadata) + def transform_image_generation_response( self, model: str, @@ -229,7 +188,7 @@ class GoogleImageGenConfig(BaseImageGenerationConfig): model_response.data = [] # Handle different response formats based on model - if "gemini" in model: + if is_gemini_image_model(model): # Gemini Flash Image Preview models return in candidates format candidates = response_data.get("candidates", []) for candidate in candidates: @@ -255,7 +214,7 @@ class GoogleImageGenConfig(BaseImageGenerationConfig): # Extract usage metadata for Gemini models if "usageMetadata" in response_data: - model_response.usage = self._transform_image_usage( + model_response.usage = transform_gemini_image_usage( response_data["usageMetadata"] ) else: diff --git a/litellm/llms/gemini/image_usage_transformation.py b/litellm/llms/gemini/image_usage_transformation.py new file mode 100644 index 00000000000..5a55bdeffb1 --- /dev/null +++ b/litellm/llms/gemini/image_usage_transformation.py @@ -0,0 +1,73 @@ +from typing import Any + +from litellm.types.utils import ImageUsage, ImageUsageInputTokensDetails + + +def _get_token_count(details: dict) -> int: + raw_token_count = details.get("tokenCount", details.get("token_count", 0)) + return raw_token_count if isinstance(raw_token_count, int) else 0 + + +def _get_modality_token_details(usage_metadata: dict, *details_keys: str) -> list: + for details_key in details_keys: + details = usage_metadata.get(details_key) + if isinstance(details, list): + return details + return [] + + +def _sum_modality_token_details( + usage_metadata: dict, *details_keys: str +) -> ImageUsageInputTokensDetails: + tokens_details = ImageUsageInputTokensDetails( + image_tokens=0, + text_tokens=0, + ) + + for details in _get_modality_token_details(usage_metadata, *details_keys): + if isinstance(details, dict): + modality = str(details.get("modality", "")).upper() + token_count = _get_token_count(details) + if modality == "TEXT": + tokens_details.text_tokens += token_count + elif modality == "IMAGE": + tokens_details.image_tokens += token_count + + return tokens_details + + +def transform_gemini_image_usage(usage_metadata: dict) -> ImageUsage: + """ + Transform Gemini usageMetadata to ImageUsage format. + """ + input_tokens_details = _sum_modality_token_details( + usage_metadata, "promptTokensDetails", "prompt_tokens_details" + ) + output_tokens = usage_metadata.get("candidatesTokenCount", 0) + output_tokens_details = _sum_modality_token_details( + usage_metadata, "candidatesTokensDetails", "candidates_tokens_details" + ) + + if not _get_modality_token_details( + usage_metadata, "candidatesTokensDetails", "candidates_tokens_details" + ): + output_tokens_details.image_tokens = output_tokens + else: + known_output_tokens = ( + output_tokens_details.text_tokens + output_tokens_details.image_tokens + ) + if output_tokens > known_output_tokens: + output_tokens_details.text_tokens += output_tokens - known_output_tokens + + usage_payload: dict[str, Any] = { + "input_tokens": usage_metadata.get("promptTokenCount", 0), + "input_tokens_details": input_tokens_details, + "output_tokens": output_tokens, + "total_tokens": usage_metadata.get("totalTokenCount", 0), + "prompt_tokens": usage_metadata.get("promptTokenCount", 0), + "prompt_tokens_details": input_tokens_details.model_dump(), + "completion_tokens": output_tokens, + "completion_tokens_details": output_tokens_details.model_dump(), + "output_tokens_details": output_tokens_details.model_dump(), + } + return ImageUsage(**usage_payload) diff --git a/litellm/llms/inception/__init__.py b/litellm/llms/inception/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/litellm/llms/inception/chat/__init__.py b/litellm/llms/inception/chat/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/litellm/llms/inception/chat/transformation.py b/litellm/llms/inception/chat/transformation.py new file mode 100644 index 00000000000..d591f783a99 --- /dev/null +++ b/litellm/llms/inception/chat/transformation.py @@ -0,0 +1,54 @@ +""" +Translate from OpenAI's `/v1/chat/completions` to Inception's `/v1/chat/completions` + +Inception Labs (https://www.inceptionlabs.ai) serves the Mercury family of +diffusion LLMs through an OpenAI-compatible API, so we only need to point the +OpenAI-like handler at the Inception API base and pick up the Inception API key. +""" + +from typing import List, Optional, Tuple + +import litellm +from litellm.secret_managers.main import get_secret_str + +from ...openai_like.chat.transformation import OpenAILikeChatConfig + + +class InceptionChatConfig(OpenAILikeChatConfig): + """ + Inception is OpenAI-compatible with standard endpoints + """ + + @property + def custom_llm_provider(self) -> Optional[str]: + return "inception" + + def get_supported_openai_params(self, model: str) -> List: + return [ + "max_tokens", + "max_completion_tokens", + "temperature", + "stop", + "tools", + "tool_choice", + "stream", + "stream_options", + "response_format", + "reasoning_effort", + "reasoning_summary", + "reasoning_summary_wait", + "diffusing", + "realtime", + ] + + def _get_openai_compatible_provider_info( + self, api_base: Optional[str], api_key: Optional[str] + ) -> Tuple[Optional[str], Optional[str]]: + passed_api_base = api_base + api_base = api_base or get_secret_str("INCEPTION_API_BASE") or "https://api.inceptionlabs.ai/v1" # type: ignore + dynamic_api_key = api_key + if passed_api_base is None or api_key: + dynamic_api_key = ( + api_key or litellm.inception_key or get_secret_str("INCEPTION_API_KEY") + ) + return api_base, dynamic_api_key diff --git a/litellm/llms/inception/completion/__init__.py b/litellm/llms/inception/completion/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/litellm/llms/inception/completion/transformation.py b/litellm/llms/inception/completion/transformation.py new file mode 100644 index 00000000000..1035042f6bf --- /dev/null +++ b/litellm/llms/inception/completion/transformation.py @@ -0,0 +1,43 @@ +""" +Inception fill-in-the-middle (FIM) completions. + +Inception's FIM endpoint is OpenAI text-completion compatible: it takes a +`prompt` (prefix) plus an optional `suffix` and returns standard +`choices[].text`. It is served at `/v1/fim/completions` rather than +`/v1/completions`, so routing points the OpenAI client at the `/v1/fim` base +(see the `text-completion-inception` branch in `main.py`). +""" + +from typing import List + +from litellm.llms.openai.completion.transformation import OpenAITextCompletionConfig + + +class InceptionTextCompletionConfig(OpenAITextCompletionConfig): + def get_supported_openai_params(self, model: str) -> List: + return [ + "suffix", + "max_tokens", + "max_completion_tokens", + "top_p", + "frequency_penalty", + "presence_penalty", + "stop", + "stream", + "stream_options", + ] + + def map_openai_params( + self, + non_default_params: dict, + optional_params: dict, + model: str, + drop_params: bool, + ) -> dict: + supported_params = self.get_supported_openai_params(model) + for param, value in non_default_params.items(): + if param == "max_completion_tokens": + optional_params["max_tokens"] = value + elif param in supported_params: + optional_params[param] = value + return optional_params diff --git a/litellm/llms/vertex_ai/gemini/transformation.py b/litellm/llms/vertex_ai/gemini/transformation.py index 4f5846cc5b6..ef7bf82bfae 100644 --- a/litellm/llms/vertex_ai/gemini/transformation.py +++ b/litellm/llms/vertex_ai/gemini/transformation.py @@ -996,7 +996,19 @@ def _gemini_convert_messages_with_history( # noqa: PLR0915 excluded_keys=["thoughtSignature"], ): assistant_content.append(gemini_tool_call_part) - last_message_with_tool_calls = assistant_msg + # Only record this as the active tool-call message when it actually + # carries tool calls. The `if` guard above is also entered for a + # text-only assistant message (`assistant_msg.get("tool_calls", []) + # is not None` is True for an empty list), so without this check a + # later assistant message with no tool calls would clobber the + # reference. The following tool result would then be matched against + # an assistant message that has no tool_calls, raising "Missing + # corresponding tool call for tool response message". + if ( + assistant_msg.get("tool_calls") + or assistant_msg.get("function_call") is not None + ): + last_message_with_tool_calls = assistant_msg ## HANDLE SERVER-SIDE TOOL INVOCATIONS (context circulation) _psf = assistant_msg.get("provider_specific_fields") diff --git a/litellm/main.py b/litellm/main.py index 3ef094042e8..da8624d11b8 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -641,6 +641,7 @@ async def acompletion( # noqa: PLR0915 if ( custom_llm_provider == "text-completion-openai" or custom_llm_provider == "text-completion-codestral" + or custom_llm_provider == "text-completion-inception" ) and isinstance(response, TextCompletionResponse): response = litellm.OpenAITextCompletionConfig().convert_to_chat_model_response_object( response_object=response, @@ -3803,6 +3804,67 @@ def completion( # type: ignore # noqa: PLR0915 ): return _model_response response = _model_response + elif custom_llm_provider == "text-completion-inception": + passed_api_base = ( + api_base + or optional_params.pop("api_base", None) + or optional_params.pop("base_url", None) + ) + api_base = ( + passed_api_base + or get_secret_str("INCEPTION_API_BASE") + or "https://api.inceptionlabs.ai/v1" + ) + # FIM is served at `/v1/fim/completions`; the OpenAI client appends + # `/completions`, so point it at the `/v1/fim` base. + api_base = api_base.rstrip("/") + if not api_base.endswith("/fim"): + api_base += "/fim" + + # Don't forward the server-managed Inception key to a caller-supplied + # api_base; only resolve it for the default/server base, or when the + # caller passes their own key. + if passed_api_base is None or api_key: + api_key = ( + api_key + or litellm.inception_key + or get_secret_str("INCEPTION_API_KEY") + ) + + _response = openai_text_completions.completion( + model=model, + messages=messages, + model_response=model_response, + print_verbose=print_verbose, + api_key=api_key, # type: ignore[arg-type] + custom_llm_provider="text-completion-inception", + api_base=api_base, + acompletion=acompletion, + client=client, + logging_obj=logging, + optional_params=optional_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + timeout=timeout, # type: ignore + ) + + if ( + optional_params.get("stream", False) is False + and acompletion is False + and text_completion is False + ): + _response = litellm.OpenAITextCompletionConfig().convert_to_chat_model_response_object( + response_object=_response, model_response_object=model_response + ) + + if optional_params.get("stream", False) or acompletion is True: + logging.post_call( + input=messages, + api_key=api_key, + original_response=_response, + additional_args={"headers": headers}, + ) + response = _response elif custom_llm_provider in ("sagemaker_chat", "sagemaker_nova"): # boto3 reads keys from .env # sagemaker_chat: HF Messages API endpoints diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index a604aafa540..ed6de4fa6b7 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -1075,6 +1075,7 @@ }, "eu.anthropic.claude-opus-4-6-v1": { "cache_creation_input_token_cost": 6.875e-06, + "cache_creation_input_token_cost_above_1hr": 1.1e-05, "cache_read_input_token_cost": 5.5e-07, "input_cost_per_token": 5.5e-06, "litellm_provider": "bedrock_converse", @@ -1104,6 +1105,7 @@ }, "au.anthropic.claude-opus-4-6-v1": { "cache_creation_input_token_cost": 6.875e-06, + "cache_creation_input_token_cost_above_1hr": 1.1e-05, "cache_read_input_token_cost": 5.5e-07, "input_cost_per_token": 5.5e-06, "litellm_provider": "bedrock_converse", @@ -1241,6 +1243,7 @@ }, "eu.anthropic.claude-opus-4-7": { "cache_creation_input_token_cost": 6.875e-06, + "cache_creation_input_token_cost_above_1hr": 1.1e-05, "cache_read_input_token_cost": 5.5e-07, "input_cost_per_token": 5.5e-06, "litellm_provider": "bedrock_converse", @@ -1271,6 +1274,7 @@ }, "au.anthropic.claude-opus-4-7": { "cache_creation_input_token_cost": 6.875e-06, + "cache_creation_input_token_cost_above_1hr": 1.1e-05, "cache_read_input_token_cost": 5.5e-07, "input_cost_per_token": 5.5e-06, "litellm_provider": "bedrock_converse", @@ -1543,6 +1547,7 @@ }, "eu.anthropic.claude-sonnet-4-6": { "cache_creation_input_token_cost": 4.125e-06, + "cache_creation_input_token_cost_above_1hr": 6.6e-06, "cache_read_input_token_cost": 3.3e-07, "input_cost_per_token": 3.3e-06, "litellm_provider": "bedrock_converse", @@ -1571,6 +1576,7 @@ }, "au.anthropic.claude-sonnet-4-6": { "cache_creation_input_token_cost": 4.125e-06, + "cache_creation_input_token_cost_above_1hr": 6.6e-06, "cache_read_input_token_cost": 3.3e-07, "input_cost_per_token": 3.3e-06, "litellm_provider": "bedrock_converse", @@ -1599,6 +1605,7 @@ }, "jp.anthropic.claude-sonnet-4-6": { "cache_creation_input_token_cost": 4.125e-06, + "cache_creation_input_token_cost_above_1hr": 6.6e-06, "cache_read_input_token_cost": 3.3e-07, "input_cost_per_token": 3.3e-06, "litellm_provider": "bedrock_converse", @@ -1995,11 +2002,13 @@ }, "au.anthropic.claude-sonnet-4-5-20250929-v1:0": { "cache_creation_input_token_cost": 4.125e-06, + "cache_creation_input_token_cost_above_1hr": 6.6e-06, "cache_read_input_token_cost": 3.3e-07, "input_cost_per_token": 3.3e-06, "input_cost_per_token_above_200k_tokens": 6.6e-06, "output_cost_per_token_above_200k_tokens": 2.475e-05, "cache_creation_input_token_cost_above_200k_tokens": 8.25e-06, + "cache_creation_input_token_cost_above_1hr_above_200k_tokens": 1.32e-05, "cache_read_input_token_cost_above_200k_tokens": 6.6e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 200000, @@ -7494,6 +7503,27 @@ "supports_video_input": true, "supports_vision": true }, + "azure_ai/kimi-k2.6": { + "input_cost_per_token": 9.5e-07, + "litellm_provider": "azure_ai", + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 4e-06, + "source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/introducing-kimi-k2-6-in-microsoft-foundry/4513125", + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true + }, "azure_ai/ministral-3b": { "input_cost_per_token": 4e-08, "litellm_provider": "azure_ai", @@ -12682,7 +12712,8 @@ "litellm_provider": "deepinfra", "mode": "chat", "supports_tool_choice": true, - "supports_function_calling": true + "supports_function_calling": true, + "supports_image_size": false }, "deepinfra/google/gemini-2.5-pro": { "max_tokens": 1000000, @@ -13583,6 +13614,7 @@ }, "eu.anthropic.claude-haiku-4-5-20251001-v1:0": { "cache_creation_input_token_cost": 1.375e-06, + "cache_creation_input_token_cost_above_1hr": 2.2e-06, "cache_read_input_token_cost": 1.1e-07, "input_cost_per_token": 1.1e-06, "deprecation_date": "2026-10-15", @@ -13787,11 +13819,13 @@ }, "eu.anthropic.claude-sonnet-4-5-20250929-v1:0": { "cache_creation_input_token_cost": 4.125e-06, + "cache_creation_input_token_cost_above_1hr": 6.6e-06, "cache_read_input_token_cost": 3.3e-07, "input_cost_per_token": 3.3e-06, "input_cost_per_token_above_200k_tokens": 6.6e-06, "output_cost_per_token_above_200k_tokens": 2.475e-05, "cache_creation_input_token_cost_above_200k_tokens": 8.25e-06, + "cache_creation_input_token_cost_above_1hr_above_200k_tokens": 1.32e-05, "cache_read_input_token_cost_above_200k_tokens": 6.6e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 200000, @@ -15006,7 +15040,8 @@ "search_context_size_medium": 0.035, "search_context_size_high": 0.035 }, - "supports_service_tier": true + "supports_service_tier": true, + "supports_image_size": false }, "gemini-2.5-flash-image": { "cache_read_input_token_cost": 3e-08, @@ -15056,7 +15091,8 @@ "supports_vision": true, "supports_web_search": false, "tpm": 8000000, - "supports_service_tier": true + "supports_service_tier": true, + "supports_image_size": false }, "gemini-3-pro-image-preview": { "input_cost_per_image": 0.0011, @@ -15196,10 +15232,16 @@ "supports_service_tier": true }, "gemini-3.1-flash-lite": { - "cache_read_input_token_cost": 4.5e-08, - "cache_read_input_token_cost_per_audio_token": 9e-08, - "input_cost_per_audio_token": 9e-07, - "input_cost_per_token": 4.5e-07, + "cache_read_input_token_cost": 2.5e-08, + "cache_read_input_token_cost_batches": 1.25e-08, + "cache_read_input_token_cost_flex": 1.25e-08, + "cache_read_input_token_cost_per_audio_token": 5e-08, + "cache_read_input_token_cost_priority": 4.5e-08, + "input_cost_per_audio_token": 5e-07, + "input_cost_per_token": 2.5e-07, + "input_cost_per_token_batches": 1.25e-07, + "input_cost_per_token_flex": 1.25e-07, + "input_cost_per_token_priority": 4.5e-07, "litellm_provider": "vertex_ai-language-models", "max_audio_length_hours": 8.4, "max_audio_per_prompt": 1, @@ -15211,9 +15253,12 @@ "max_video_length": 1, "max_videos_per_prompt": 10, "mode": "chat", - "output_cost_per_reasoning_token": 2.7e-06, - "output_cost_per_token": 2.7e-06, - "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#gemini-models", + "output_cost_per_reasoning_token": 1.5e-06, + "output_cost_per_token": 1.5e-06, + "output_cost_per_token_batches": 7.5e-07, + "output_cost_per_token_flex": 7.5e-07, + "output_cost_per_token_priority": 2.7e-06, + "source": "https://ai.google.dev/gemini-api/docs/pricing#gemini-3.1-flash-lite", "supported_endpoints": [ "/v1/chat/completions", "/v1/completions", @@ -15336,7 +15381,8 @@ "search_context_size_medium": 0.035, "search_context_size_high": 0.035 }, - "supports_service_tier": true + "supports_service_tier": true, + "supports_image_size": false }, "gemini-2.5-flash-lite-preview-09-2025": { "cache_read_input_token_cost": 1e-08, @@ -15386,7 +15432,8 @@ "search_context_size_low": 0.035, "search_context_size_medium": 0.035, "search_context_size_high": 0.035 - } + }, + "supports_image_size": false }, "gemini-2.5-flash-preview-09-2025": { "cache_read_input_token_cost": 7.5e-08, @@ -15436,7 +15483,8 @@ "search_context_size_low": 0.035, "search_context_size_medium": 0.035, "search_context_size_high": 0.035 - } + }, + "supports_image_size": false }, "gemini-live-2.5-flash-preview-native-audio-09-2025": { "cache_read_input_token_cost": 7.5e-08, @@ -15587,7 +15635,8 @@ "search_context_size_low": 0.035, "search_context_size_medium": 0.035, "search_context_size_high": 0.035 - } + }, + "supports_image_size": false }, "gemini-2.5-pro": { "cache_read_input_token_cost": 1.25e-07, @@ -16597,7 +16646,8 @@ "search_context_size_medium": 0.035, "search_context_size_high": 0.035 }, - "supports_service_tier": true + "supports_service_tier": true, + "supports_image_size": false }, "gemini/gemini-2.5-flash-image": { "cache_read_input_token_cost": 3e-08, @@ -16653,7 +16703,8 @@ "search_context_size_medium": 0.035, "search_context_size_high": 0.035 }, - "supports_service_tier": true + "supports_service_tier": true, + "supports_image_size": false }, "gemini/gemini-3-pro-image-preview": { "input_cost_per_image": 0.0011, @@ -16832,7 +16883,8 @@ "search_context_size_medium": 0.035, "search_context_size_high": 0.035 }, - "supports_service_tier": true + "supports_service_tier": true, + "supports_image_size": false }, "gemini/gemini-2.5-flash-lite-preview-09-2025": { "cache_read_input_token_cost": 1e-08, @@ -16884,7 +16936,8 @@ "search_context_size_low": 0.035, "search_context_size_medium": 0.035, "search_context_size_high": 0.035 - } + }, + "supports_image_size": false }, "gemini/gemini-2.5-flash-preview-09-2025": { "cache_read_input_token_cost": 7.5e-08, @@ -16936,7 +16989,8 @@ "search_context_size_low": 0.035, "search_context_size_medium": 0.035, "search_context_size_high": 0.035 - } + }, + "supports_image_size": false }, "gemini/gemini-flash-latest": { "cache_read_input_token_cost": 7.5e-08, @@ -17093,7 +17147,8 @@ "search_context_size_low": 0.035, "search_context_size_medium": 0.035, "search_context_size_high": 0.035 - } + }, + "supports_image_size": false }, "gemini/gemini-2.5-flash-preview-tts": { "input_cost_per_token": 3e-07, @@ -17318,10 +17373,16 @@ "supports_service_tier": true }, "gemini/gemini-3.1-flash-lite": { - "cache_read_input_token_cost": 4.5e-08, - "cache_read_input_token_cost_per_audio_token": 9e-08, - "input_cost_per_audio_token": 9e-07, - "input_cost_per_token": 4.5e-07, + "cache_read_input_token_cost": 2.5e-08, + "cache_read_input_token_cost_batches": 1.25e-08, + "cache_read_input_token_cost_flex": 1.25e-08, + "cache_read_input_token_cost_per_audio_token": 5e-08, + "cache_read_input_token_cost_priority": 4.5e-08, + "input_cost_per_audio_token": 5e-07, + "input_cost_per_token": 2.5e-07, + "input_cost_per_token_batches": 1.25e-07, + "input_cost_per_token_flex": 1.25e-07, + "input_cost_per_token_priority": 4.5e-07, "litellm_provider": "gemini", "max_audio_length_hours": 8.4, "max_audio_per_prompt": 1, @@ -17333,10 +17394,13 @@ "max_video_length": 1, "max_videos_per_prompt": 10, "mode": "chat", - "output_cost_per_reasoning_token": 2.7e-06, - "output_cost_per_token": 2.7e-06, + "output_cost_per_reasoning_token": 1.5e-06, + "output_cost_per_token": 1.5e-06, + "output_cost_per_token_batches": 7.5e-07, + "output_cost_per_token_flex": 7.5e-07, + "output_cost_per_token_priority": 2.7e-06, "rpm": 15, - "source": "https://ai.google.dev/gemini-api/docs/pricing", + "source": "https://ai.google.dev/gemini-api/docs/pricing#gemini-3.1-flash-lite", "supported_endpoints": [ "/v1/chat/completions", "/v1/completions", @@ -18098,23 +18162,22 @@ }, "github_copilot/claude-haiku-4.5": { "litellm_provider": "github_copilot", - "max_input_tokens": 200000, - "max_output_tokens": 32000, - "max_tokens": 32000, + "max_input_tokens": 128000, + "max_output_tokens": 16000, + "max_tokens": 16000, "mode": "chat", "supported_endpoints": [ "/v1/chat/completions" ], "supports_function_calling": true, "supports_parallel_function_calling": true, - "supports_vision": true, - "supports_reasoning": true + "supports_vision": true }, "github_copilot/claude-opus-4.5": { "litellm_provider": "github_copilot", - "max_input_tokens": 200000, - "max_output_tokens": 32000, - "max_tokens": 32000, + "max_input_tokens": 128000, + "max_output_tokens": 16000, + "max_tokens": 16000, "mode": "chat", "supported_endpoints": [ "/v1/chat/completions" @@ -18122,7 +18185,6 @@ "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_vision": true, - "supports_reasoning": true, "supports_output_config": true }, "github_copilot/claude-opus-4.6-fast": { @@ -18138,22 +18200,6 @@ "supports_parallel_function_calling": true, "supports_vision": true }, - "github_copilot/claude-opus-4.7": { - "litellm_provider": "github_copilot", - "max_input_tokens": 200000, - "max_output_tokens": 64000, - "max_tokens": 64000, - "mode": "chat", - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/messages" - ], - "supports_vision": true, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_response_schema": true, - "supports_reasoning": true - }, "github_copilot/claude-opus-41": { "litellm_provider": "github_copilot", "max_input_tokens": 80000, @@ -18180,33 +18226,16 @@ }, "github_copilot/claude-sonnet-4.5": { "litellm_provider": "github_copilot", - "max_input_tokens": 200000, - "max_output_tokens": 32000, - "max_tokens": 32000, + "max_input_tokens": 128000, + "max_output_tokens": 16000, + "max_tokens": 16000, "mode": "chat", "supported_endpoints": [ "/v1/chat/completions" ], "supports_function_calling": true, "supports_parallel_function_calling": true, - "supports_vision": true, - "supports_reasoning": true - }, - "github_copilot/claude-sonnet-4.6": { - "litellm_provider": "github_copilot", - "max_input_tokens": 200000, - "max_output_tokens": 32000, - "max_tokens": 32000, - "mode": "chat", - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/messages" - ], - "supports_vision": true, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_response_schema": true, - "supports_reasoning": true + "supports_vision": true }, "github_copilot/gemini-2.5-pro": { "litellm_provider": "github_copilot", @@ -18216,25 +18245,7 @@ "mode": "chat", "supports_function_calling": true, "supports_parallel_function_calling": true, - "supports_vision": true, - "supported_endpoints": [ - "/v1/chat/completions" - ], - "supports_reasoning": true - }, - "github_copilot/gemini-3-flash-preview": { - "litellm_provider": "github_copilot", - "max_input_tokens": 128000, - "max_output_tokens": 64000, - "max_tokens": 64000, - "mode": "chat", - "supported_endpoints": [ - "/v1/chat/completions" - ], - "supports_vision": true, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_reasoning": true + "supports_vision": true }, "github_copilot/gemini-3-pro-preview": { "litellm_provider": "github_copilot", @@ -18246,30 +18257,13 @@ "supports_parallel_function_calling": true, "supports_vision": true }, - "github_copilot/gemini-3.1-pro-preview": { - "litellm_provider": "github_copilot", - "max_input_tokens": 128000, - "max_output_tokens": 64000, - "max_tokens": 64000, - "mode": "chat", - "supported_endpoints": [ - "/v1/chat/completions" - ], - "supports_vision": true, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_reasoning": true - }, "github_copilot/gpt-3.5-turbo": { "litellm_provider": "github_copilot", "max_input_tokens": 16384, "max_output_tokens": 4096, "max_tokens": 4096, "mode": "chat", - "supports_function_calling": true, - "supported_endpoints": [ - "/v1/chat/completions" - ] + "supports_function_calling": true }, "github_copilot/gpt-3.5-turbo-0613": { "litellm_provider": "github_copilot", @@ -18277,10 +18271,7 @@ "max_output_tokens": 4096, "max_tokens": 4096, "mode": "chat", - "supports_function_calling": true, - "supported_endpoints": [ - "/v1/chat/completions" - ] + "supports_function_calling": true }, "github_copilot/gpt-4": { "litellm_provider": "github_copilot", @@ -18288,22 +18279,7 @@ "max_output_tokens": 4096, "max_tokens": 4096, "mode": "chat", - "supports_function_calling": true, - "supported_endpoints": [ - "/v1/chat/completions" - ] - }, - "github_copilot/gpt-4-0125-preview": { - "litellm_provider": "github_copilot", - "max_input_tokens": 128000, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "chat", - "supported_endpoints": [ - "/v1/chat/completions" - ], - "supports_function_calling": true, - "supports_parallel_function_calling": true + "supports_function_calling": true }, "github_copilot/gpt-4-0613": { "litellm_provider": "github_copilot", @@ -18311,22 +18287,16 @@ "max_output_tokens": 4096, "max_tokens": 4096, "mode": "chat", - "supports_function_calling": true, - "supported_endpoints": [ - "/v1/chat/completions" - ] + "supports_function_calling": true }, "github_copilot/gpt-4-o-preview": { "litellm_provider": "github_copilot", - "max_input_tokens": 128000, + "max_input_tokens": 64000, "max_output_tokens": 4096, "max_tokens": 4096, "mode": "chat", "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supported_endpoints": [ - "/v1/chat/completions" - ] + "supports_parallel_function_calling": true }, "github_copilot/gpt-4.1": { "litellm_provider": "github_copilot", @@ -18337,10 +18307,7 @@ "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_response_schema": true, - "supports_vision": true, - "supported_endpoints": [ - "/v1/chat/completions" - ] + "supports_vision": true }, "github_copilot/gpt-4.1-2025-04-14": { "litellm_provider": "github_copilot", @@ -18351,89 +18318,68 @@ "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_response_schema": true, - "supports_vision": true, - "supported_endpoints": [ - "/v1/chat/completions" - ] + "supports_vision": true }, "github_copilot/gpt-41-copilot": { "litellm_provider": "github_copilot", - "mode": "chat" + "mode": "completion" }, "github_copilot/gpt-4o": { "litellm_provider": "github_copilot", - "max_input_tokens": 128000, + "max_input_tokens": 64000, "max_output_tokens": 4096, "max_tokens": 4096, "mode": "chat", "supports_function_calling": true, "supports_parallel_function_calling": true, - "supports_vision": true, - "supported_endpoints": [ - "/v1/chat/completions" - ] + "supports_vision": true }, "github_copilot/gpt-4o-2024-05-13": { "litellm_provider": "github_copilot", - "max_input_tokens": 128000, + "max_input_tokens": 64000, "max_output_tokens": 4096, "max_tokens": 4096, "mode": "chat", "supports_function_calling": true, "supports_parallel_function_calling": true, - "supports_vision": true, - "supported_endpoints": [ - "/v1/chat/completions" - ] + "supports_vision": true }, "github_copilot/gpt-4o-2024-08-06": { "litellm_provider": "github_copilot", - "max_input_tokens": 128000, + "max_input_tokens": 64000, "max_output_tokens": 16384, "max_tokens": 16384, "mode": "chat", "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supported_endpoints": [ - "/v1/chat/completions" - ] + "supports_parallel_function_calling": true }, "github_copilot/gpt-4o-2024-11-20": { "litellm_provider": "github_copilot", - "max_input_tokens": 128000, + "max_input_tokens": 64000, "max_output_tokens": 16384, "max_tokens": 16384, "mode": "chat", "supports_function_calling": true, "supports_parallel_function_calling": true, - "supports_vision": true, - "supported_endpoints": [ - "/v1/chat/completions" - ] + "supports_vision": true }, "github_copilot/gpt-4o-mini": { "litellm_provider": "github_copilot", - "max_input_tokens": 128000, + "max_input_tokens": 64000, "max_output_tokens": 4096, "max_tokens": 4096, "mode": "chat", "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supported_endpoints": [ - "/v1/chat/completions" - ] + "supports_parallel_function_calling": true }, "github_copilot/gpt-4o-mini-2024-07-18": { "litellm_provider": "github_copilot", - "max_input_tokens": 128000, + "max_input_tokens": 64000, "max_output_tokens": 4096, "max_tokens": 4096, "mode": "chat", "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supported_endpoints": [ - "/v1/chat/completions" - ] + "supports_parallel_function_calling": true }, "github_copilot/gpt-5": { "litellm_provider": "github_copilot", @@ -18452,19 +18398,14 @@ }, "github_copilot/gpt-5-mini": { "litellm_provider": "github_copilot", - "max_input_tokens": 264000, + "max_input_tokens": 128000, "max_output_tokens": 64000, "max_tokens": 64000, "mode": "chat", "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_response_schema": true, - "supports_vision": true, - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/responses" - ], - "supports_reasoning": true + "supports_vision": true }, "github_copilot/gpt-5.1": { "litellm_provider": "github_copilot", @@ -18497,7 +18438,7 @@ }, "github_copilot/gpt-5.2": { "litellm_provider": "github_copilot", - "max_input_tokens": 264000, + "max_input_tokens": 128000, "max_output_tokens": 64000, "max_tokens": 64000, "mode": "chat", @@ -18508,27 +18449,11 @@ "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_response_schema": true, - "supports_vision": true, - "supports_reasoning": true - }, - "github_copilot/gpt-5.2-codex": { - "litellm_provider": "github_copilot", - "max_input_tokens": 400000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "responses", - "supported_endpoints": [ - "/v1/responses" - ], - "supports_vision": true, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_response_schema": true, - "supports_reasoning": true + "supports_vision": true }, "github_copilot/gpt-5.3-codex": { "litellm_provider": "github_copilot", - "max_input_tokens": 400000, + "max_input_tokens": 128000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "responses", @@ -18538,96 +18463,25 @@ "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_response_schema": true, - "supports_vision": true, - "supports_reasoning": true - }, - "github_copilot/gpt-5.4": { - "litellm_provider": "github_copilot", - "max_input_tokens": 400000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "chat", - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/responses" - ], - "supports_vision": true, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_response_schema": true, - "supports_reasoning": true - }, - "github_copilot/gpt-5.4-mini": { - "litellm_provider": "github_copilot", - "max_input_tokens": 400000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "responses", - "supported_endpoints": [ - "/v1/responses" - ], - "supports_vision": true, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_response_schema": true, - "supports_reasoning": true - }, - "github_copilot/gpt-5.5": { - "litellm_provider": "github_copilot", - "max_input_tokens": 400000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "responses", - "supported_endpoints": [ - "/v1/responses" - ], - "supports_vision": true, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_response_schema": true, - "supports_reasoning": true - }, - "github_copilot/oswe-vscode-prime": { - "litellm_provider": "github_copilot", - "max_input_tokens": 264000, - "max_output_tokens": 64000, - "max_tokens": 64000, - "mode": "chat", - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/responses" - ], - "supports_vision": true, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_response_schema": true + "supports_vision": true }, "github_copilot/text-embedding-3-small": { "litellm_provider": "github_copilot", "max_input_tokens": 8191, "max_tokens": 8191, - "mode": "embedding", - "supported_endpoints": [ - "/v1/embeddings" - ] + "mode": "embedding" }, "github_copilot/text-embedding-3-small-inference": { "litellm_provider": "github_copilot", "max_input_tokens": 8191, "max_tokens": 8191, - "mode": "embedding", - "supported_endpoints": [ - "/v1/embeddings" - ] + "mode": "embedding" }, "github_copilot/text-embedding-ada-002": { "litellm_provider": "github_copilot", "max_input_tokens": 8191, "max_tokens": 8191, - "mode": "embedding", - "supported_endpoints": [ - "/v1/embeddings" - ] + "mode": "embedding" }, "chatgpt/gpt-5.4": { "litellm_provider": "chatgpt", @@ -23278,11 +23132,13 @@ }, "jp.anthropic.claude-sonnet-4-5-20250929-v1:0": { "cache_creation_input_token_cost": 4.125e-06, + "cache_creation_input_token_cost_above_1hr": 6.6e-06, "cache_read_input_token_cost": 3.3e-07, "input_cost_per_token": 3.3e-06, "input_cost_per_token_above_200k_tokens": 6.6e-06, "output_cost_per_token_above_200k_tokens": 2.475e-05, "cache_creation_input_token_cost_above_200k_tokens": 8.25e-06, + "cache_creation_input_token_cost_above_1hr_above_200k_tokens": 1.32e-05, "cache_read_input_token_cost_above_200k_tokens": 6.6e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 200000, @@ -23308,6 +23164,7 @@ }, "jp.anthropic.claude-haiku-4-5-20251001-v1:0": { "cache_creation_input_token_cost": 1.375e-06, + "cache_creation_input_token_cost_above_1hr": 2.2e-06, "cache_read_input_token_cost": 1.1e-07, "input_cost_per_token": 1.1e-06, "litellm_provider": "bedrock_converse", @@ -23420,6 +23277,31 @@ "supports_system_messages": true, "supports_tool_choice": true }, + "inception/mercury-2": { + "cache_read_input_token_cost": 2.5e-08, + "input_cost_per_token": 2.5e-07, + "litellm_provider": "inception", + "max_input_tokens": 128000, + "max_output_tokens": 50000, + "max_tokens": 50000, + "mode": "chat", + "output_cost_per_token": 7.5e-07, + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "text-completion-inception/mercury-edit-2": { + "cache_read_input_token_cost": 2.5e-08, + "input_cost_per_token": 2.5e-07, + "litellm_provider": "text-completion-inception", + "max_input_tokens": 32000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "completion", + "output_cost_per_token": 7.5e-07 + }, "lambda_ai/deepseek-llama3.3-70b": { "input_cost_per_token": 2e-07, "litellm_provider": "lambda_ai", @@ -26432,6 +26314,32 @@ "supports_vision": true, "supports_web_search": true }, + "oci/meta.llama-3.1-8b-instruct": { + "input_cost_per_token": 7.2e-07, + "litellm_provider": "oci", + "max_input_tokens": 128000, + "max_output_tokens": 4000, + "max_tokens": 4000, + "mode": "chat", + "output_cost_per_token": 7.2e-07, + "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", + "supports_function_calling": true, + "supports_response_schema": false, + "supports_native_streaming": true + }, + "oci/meta.llama-3.1-70b-instruct": { + "input_cost_per_token": 7.2e-07, + "litellm_provider": "oci", + "max_input_tokens": 128000, + "max_output_tokens": 4000, + "max_tokens": 4000, + "mode": "chat", + "output_cost_per_token": 7.2e-07, + "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", + "supports_function_calling": true, + "supports_response_schema": false, + "supports_native_streaming": true + }, "oci/meta.llama-3.1-405b-instruct": { "input_cost_per_token": 1.068e-05, "litellm_provider": "oci", @@ -26442,7 +26350,8 @@ "output_cost_per_token": 1.068e-05, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", "supports_function_calling": true, - "supports_response_schema": false + "supports_response_schema": false, + "supports_native_streaming": true }, "oci/meta.llama-3.2-90b-vision-instruct": { "input_cost_per_token": 2e-06, @@ -26455,6 +26364,7 @@ "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", "supports_function_calling": true, "supports_response_schema": false, + "supports_native_streaming": true, "supports_vision": true }, "oci/meta.llama-3.3-70b-instruct": { @@ -26467,31 +26377,35 @@ "output_cost_per_token": 7.2e-07, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", "supports_function_calling": true, - "supports_response_schema": false + "supports_response_schema": false, + "supports_native_streaming": true }, "oci/meta.llama-4-maverick-17b-128e-instruct-fp8": { "input_cost_per_token": 7.2e-07, "litellm_provider": "oci", - "max_input_tokens": 512000, - "max_output_tokens": 4000, - "max_tokens": 4000, + "max_input_tokens": 1048576, + "max_output_tokens": 8192, + "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 7.2e-07, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", "supports_function_calling": true, - "supports_response_schema": false + "supports_response_schema": false, + "supports_native_streaming": true, + "supports_vision": true }, "oci/meta.llama-4-scout-17b-16e-instruct": { "input_cost_per_token": 7.2e-07, "litellm_provider": "oci", - "max_input_tokens": 192000, - "max_output_tokens": 4000, - "max_tokens": 4000, + "max_input_tokens": 10485760, + "max_output_tokens": 8192, + "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 7.2e-07, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", "supports_function_calling": true, - "supports_response_schema": false + "supports_response_schema": false, + "supports_native_streaming": true }, "oci/xai.grok-3": { "input_cost_per_token": 3e-06, @@ -26503,7 +26417,8 @@ "output_cost_per_token": 1.5e-05, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", "supports_function_calling": true, - "supports_response_schema": false + "supports_response_schema": false, + "supports_native_streaming": true }, "oci/xai.grok-3-fast": { "input_cost_per_token": 5e-06, @@ -26515,7 +26430,8 @@ "output_cost_per_token": 2.5e-05, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", "supports_function_calling": true, - "supports_response_schema": false + "supports_response_schema": false, + "supports_native_streaming": true }, "oci/xai.grok-3-mini": { "input_cost_per_token": 3e-07, @@ -26527,7 +26443,8 @@ "output_cost_per_token": 5e-07, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", "supports_function_calling": true, - "supports_response_schema": false + "supports_response_schema": false, + "supports_native_streaming": true }, "oci/xai.grok-3-mini-fast": { "input_cost_per_token": 6e-07, @@ -26539,7 +26456,8 @@ "output_cost_per_token": 4e-06, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", "supports_function_calling": true, - "supports_response_schema": false + "supports_response_schema": false, + "supports_native_streaming": true }, "oci/xai.grok-4": { "input_cost_per_token": 3e-06, @@ -26551,7 +26469,8 @@ "output_cost_per_token": 1.5e-05, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", "supports_function_calling": true, - "supports_response_schema": false + "supports_response_schema": false, + "supports_native_streaming": true }, "oci/cohere.command-latest": { "input_cost_per_token": 1.56e-06, @@ -26563,7 +26482,8 @@ "output_cost_per_token": 1.56e-06, "source": "https://www.oracle.com/cloud/ai/generative-ai/pricing/", "supports_function_calling": true, - "supports_response_schema": false + "supports_response_schema": false, + "supports_native_streaming": true }, "oci/cohere.command-a-03-2025": { "input_cost_per_token": 1.56e-06, @@ -26575,7 +26495,8 @@ "output_cost_per_token": 1.56e-06, "source": "https://www.oracle.com/cloud/ai/generative-ai/pricing/", "supports_function_calling": true, - "supports_response_schema": false + "supports_response_schema": false, + "supports_native_streaming": true }, "oci/cohere.command-plus-latest": { "input_cost_per_token": 1.56e-06, @@ -26587,7 +26508,88 @@ "output_cost_per_token": 1.56e-06, "source": "https://www.oracle.com/cloud/ai/generative-ai/pricing/", "supports_function_calling": true, - "supports_response_schema": false + "supports_response_schema": false, + "supports_native_streaming": true + }, + "oci/google.gemini-2.5-flash": { + "input_cost_per_token": 1.5e-07, + "litellm_provider": "oci", + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "output_cost_per_token": 6e-07, + "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", + "supports_function_calling": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_native_streaming": true, + "supports_image_size": false + }, + "oci/google.gemini-2.5-pro": { + "input_cost_per_token": 1.25e-06, + "litellm_provider": "oci", + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "output_cost_per_token": 1e-05, + "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", + "supports_function_calling": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_native_streaming": true + }, + "oci/google.gemini-2.5-flash-lite": { + "input_cost_per_token": 7.5e-08, + "litellm_provider": "oci", + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "output_cost_per_token": 3e-07, + "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", + "supports_function_calling": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_native_streaming": true, + "supports_image_size": false + }, + "oci/cohere.command-a-vision": { + "input_cost_per_token": 1.56e-06, + "litellm_provider": "oci", + "max_input_tokens": 256000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1.56e-06, + "source": "https://www.oracle.com/cloud/ai/generative-ai/pricing/", + "supports_function_calling": true, + "supports_response_schema": false, + "supports_native_streaming": true, + "supports_vision": true + }, + "oci/cohere.command-a-reasoning": { + "input_cost_per_token": 1.56e-06, + "litellm_provider": "oci", + "max_input_tokens": 256000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1.56e-06, + "source": "https://www.oracle.com/cloud/ai/generative-ai/pricing/", + "supports_function_calling": false, + "supports_response_schema": false, + "supports_native_streaming": true + }, + "oci/cohere.embed-multilingual-image-v3.0": { + "input_cost_per_token": 1e-07, + "litellm_provider": "oci", + "max_input_tokens": 512, + "mode": "embedding", + "output_vector_size": 1024, + "source": "https://www.oracle.com/cloud/ai/generative-ai/pricing/", + "supports_vision": true }, "oci/cohere.command-a-reasoning-08-2025": { "input_cost_per_token": 1.56e-06, @@ -26663,18 +26665,6 @@ "supports_response_schema": false, "supports_vision": true }, - "oci/meta.llama-3.1-70b-instruct": { - "input_cost_per_token": 7.2e-07, - "litellm_provider": "oci", - "max_input_tokens": 128000, - "max_output_tokens": 4000, - "max_tokens": 4000, - "mode": "chat", - "output_cost_per_token": 7.2e-07, - "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", - "supports_function_calling": true, - "supports_response_schema": false - }, "oci/meta.llama-3.3-70b-instruct-fp8-dynamic": { "input_cost_per_token": 7.2e-07, "litellm_provider": "oci", @@ -26792,45 +26782,6 @@ "supports_response_schema": true, "supports_vision": true }, - "oci/google.gemini-2.5-pro": { - "input_cost_per_token": 1.25e-06, - "litellm_provider": "oci", - "max_input_tokens": 1048576, - "max_output_tokens": 65536, - "max_tokens": 65536, - "mode": "chat", - "output_cost_per_token": 1e-05, - "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", - "supports_function_calling": true, - "supports_response_schema": true, - "supports_vision": true - }, - "oci/google.gemini-2.5-flash": { - "input_cost_per_token": 1.5e-07, - "litellm_provider": "oci", - "max_input_tokens": 1048576, - "max_output_tokens": 65536, - "max_tokens": 65536, - "mode": "chat", - "output_cost_per_token": 6e-07, - "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", - "supports_function_calling": true, - "supports_response_schema": true, - "supports_vision": true - }, - "oci/google.gemini-2.5-flash-lite": { - "input_cost_per_token": 7.5e-08, - "litellm_provider": "oci", - "max_input_tokens": 1048576, - "max_output_tokens": 65536, - "max_tokens": 65536, - "mode": "chat", - "output_cost_per_token": 3e-07, - "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", - "supports_function_calling": true, - "supports_response_schema": true, - "supports_vision": true - }, "oci/cohere.embed-english-v3.0": { "input_cost_per_token": 1e-07, "litellm_provider": "oci", @@ -27638,7 +27589,8 @@ "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "supports_image_size": false }, "openrouter/google/gemini-2.5-pro": { "input_cost_per_audio_token": 7e-07, @@ -29508,7 +29460,8 @@ "mode": "responses", "supports_web_search": true, "supports_reasoning": false, - "supports_function_calling": true + "supports_function_calling": true, + "supports_image_size": false }, "perplexity/xai/grok-4-1-fast-non-reasoning": { "litellm_provider": "perplexity", @@ -30090,7 +30043,8 @@ "supports_vision": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_response_schema": true + "supports_response_schema": true, + "supports_image_size": false }, "replicate/openai/gpt-oss-120b": { "input_cost_per_token": 1.8e-07, @@ -31909,6 +31863,7 @@ }, "au.anthropic.claude-haiku-4-5-20251001-v1:0": { "cache_creation_input_token_cost": 1.375e-06, + "cache_creation_input_token_cost_above_1hr": 2.2e-06, "cache_read_input_token_cost": 1.1e-07, "input_cost_per_token": 1.1e-06, "litellm_provider": "bedrock_converse", @@ -32788,7 +32743,8 @@ "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, - "supports_response_schema": true + "supports_response_schema": true, + "supports_image_size": false }, "vercel_ai_gateway/google/gemini-2.5-pro": { "input_cost_per_token": 2.5e-06, @@ -33555,6 +33511,7 @@ }, "vertex_ai/claude-haiku-4-5": { "cache_creation_input_token_cost": 1.25e-06, + "cache_creation_input_token_cost_above_1hr": 2e-06, "cache_read_input_token_cost": 1e-07, "input_cost_per_token": 1e-06, "litellm_provider": "vertex_ai-anthropic_models", @@ -33576,6 +33533,7 @@ }, "vertex_ai/claude-haiku-4-5@20251001": { "cache_creation_input_token_cost": 1.25e-06, + "cache_creation_input_token_cost_above_1hr": 2e-06, "cache_read_input_token_cost": 1e-07, "input_cost_per_token": 1e-06, "litellm_provider": "vertex_ai-anthropic_models", @@ -33626,6 +33584,7 @@ }, "vertex_ai/claude-3-7-sonnet@20250219": { "cache_creation_input_token_cost": 3.75e-06, + "cache_creation_input_token_cost_above_1hr": 6e-06, "cache_read_input_token_cost": 3e-07, "deprecation_date": "2026-05-11", "input_cost_per_token": 3e-06, @@ -33725,6 +33684,7 @@ }, "vertex_ai/claude-opus-4": { "cache_creation_input_token_cost": 1.875e-05, + "cache_creation_input_token_cost_above_1hr": 3e-05, "cache_read_input_token_cost": 1.5e-06, "input_cost_per_token": 1.5e-05, "litellm_provider": "vertex_ai-anthropic_models", @@ -33750,6 +33710,7 @@ }, "vertex_ai/claude-opus-4-1": { "cache_creation_input_token_cost": 1.875e-05, + "cache_creation_input_token_cost_above_1hr": 3e-05, "cache_read_input_token_cost": 1.5e-06, "input_cost_per_token": 1.5e-05, "input_cost_per_token_batches": 7.5e-06, @@ -33767,6 +33728,7 @@ }, "vertex_ai/claude-opus-4-1@20250805": { "cache_creation_input_token_cost": 1.875e-05, + "cache_creation_input_token_cost_above_1hr": 3e-05, "cache_read_input_token_cost": 1.5e-06, "input_cost_per_token": 1.5e-05, "input_cost_per_token_batches": 7.5e-06, @@ -33784,6 +33746,7 @@ }, "vertex_ai/claude-opus-4-5": { "cache_creation_input_token_cost": 6.25e-06, + "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, "input_cost_per_token": 5e-06, "litellm_provider": "vertex_ai-anthropic_models", @@ -33810,6 +33773,7 @@ }, "vertex_ai/claude-opus-4-5@20251101": { "cache_creation_input_token_cost": 6.25e-06, + "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, "input_cost_per_token": 5e-06, "litellm_provider": "vertex_ai-anthropic_models", @@ -33837,6 +33801,7 @@ }, "vertex_ai/claude-opus-4-6": { "cache_creation_input_token_cost": 6.25e-06, + "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, "input_cost_per_token": 5e-06, "litellm_provider": "vertex_ai-anthropic_models", @@ -33864,6 +33829,7 @@ }, "vertex_ai/claude-opus-4-6@default": { "cache_creation_input_token_cost": 6.25e-06, + "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, "input_cost_per_token": 5e-06, "litellm_provider": "vertex_ai-anthropic_models", @@ -33891,6 +33857,7 @@ }, "vertex_ai/claude-opus-4-7": { "cache_creation_input_token_cost": 6.25e-06, + "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, "input_cost_per_token": 5e-06, "litellm_provider": "vertex_ai-anthropic_models", @@ -33918,6 +33885,7 @@ }, "vertex_ai/claude-opus-4-7@default": { "cache_creation_input_token_cost": 6.25e-06, + "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, "input_cost_per_token": 5e-06, "litellm_provider": "vertex_ai-anthropic_models", @@ -34001,6 +33969,7 @@ }, "vertex_ai/claude-sonnet-4-5": { "cache_creation_input_token_cost": 3.75e-06, + "cache_creation_input_token_cost_above_1hr": 6e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_token": 3e-06, "input_cost_per_token_above_200k_tokens": 6e-06, @@ -34027,6 +33996,7 @@ }, "vertex_ai/claude-sonnet-4-6": { "cache_creation_input_token_cost": 3.75e-06, + "cache_creation_input_token_cost_above_1hr": 6e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_token": 3e-06, "litellm_provider": "vertex_ai-anthropic_models", @@ -34054,6 +34024,7 @@ }, "vertex_ai/claude-sonnet-4-5@20250929": { "cache_creation_input_token_cost": 3.75e-06, + "cache_creation_input_token_cost_above_1hr": 6e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_token": 3e-06, "input_cost_per_token_above_200k_tokens": 6e-06, @@ -34081,6 +34052,7 @@ }, "vertex_ai/claude-opus-4@20250514": { "cache_creation_input_token_cost": 1.875e-05, + "cache_creation_input_token_cost_above_1hr": 3e-05, "cache_read_input_token_cost": 1.5e-06, "input_cost_per_token": 1.5e-05, "litellm_provider": "vertex_ai-anthropic_models", @@ -34106,6 +34078,7 @@ }, "vertex_ai/claude-sonnet-4": { "cache_creation_input_token_cost": 3.75e-06, + "cache_creation_input_token_cost_above_1hr": 6e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_token": 3e-06, "input_cost_per_token_above_200k_tokens": 6e-06, @@ -34135,6 +34108,7 @@ }, "vertex_ai/claude-sonnet-4@20250514": { "cache_creation_input_token_cost": 3.75e-06, + "cache_creation_input_token_cost_above_1hr": 6e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_token": 3e-06, "input_cost_per_token_above_200k_tokens": 6e-06, @@ -34342,7 +34316,8 @@ "supports_url_context": true, "supports_vision": true, "supports_web_search": false, - "tpm": 8000000 + "tpm": 8000000, + "supports_image_size": false }, "vertex_ai/gemini-3-pro-image-preview": { "input_cost_per_image": 0.0011, @@ -34430,10 +34405,16 @@ "web_search_billing_unit": "per_query" }, "vertex_ai/gemini-3.1-flash-lite": { - "cache_read_input_token_cost": 4.5e-08, - "cache_read_input_token_cost_per_audio_token": 9e-08, - "input_cost_per_audio_token": 9e-07, - "input_cost_per_token": 4.5e-07, + "cache_read_input_token_cost": 2.5e-08, + "cache_read_input_token_cost_batches": 1.25e-08, + "cache_read_input_token_cost_flex": 1.25e-08, + "cache_read_input_token_cost_per_audio_token": 5e-08, + "cache_read_input_token_cost_priority": 4.5e-08, + "input_cost_per_audio_token": 5e-07, + "input_cost_per_token": 2.5e-07, + "input_cost_per_token_batches": 1.25e-07, + "input_cost_per_token_flex": 1.25e-07, + "input_cost_per_token_priority": 4.5e-07, "litellm_provider": "vertex_ai-language-models", "max_audio_length_hours": 8.4, "max_audio_per_prompt": 1, @@ -34445,8 +34426,11 @@ "max_video_length": 1, "max_videos_per_prompt": 10, "mode": "chat", - "output_cost_per_reasoning_token": 2.7e-06, - "output_cost_per_token": 2.7e-06, + "output_cost_per_reasoning_token": 1.5e-06, + "output_cost_per_token": 1.5e-06, + "output_cost_per_token_batches": 7.5e-07, + "output_cost_per_token_flex": 7.5e-07, + "output_cost_per_token_priority": 2.7e-06, "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#gemini-models", "supported_endpoints": [ "/v1/chat/completions", @@ -41151,6 +41135,7 @@ }, "vertex_ai/claude-sonnet-4-6@default": { "cache_creation_input_token_cost": 3.75e-06, + "cache_creation_input_token_cost_above_1hr": 6e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_token": 3e-06, "litellm_provider": "vertex_ai-anthropic_models", diff --git a/litellm/proxy/agent_endpoints/endpoints.py b/litellm/proxy/agent_endpoints/endpoints.py index 082e314b08d..19dbfe33d32 100644 --- a/litellm/proxy/agent_endpoints/endpoints.py +++ b/litellm/proxy/agent_endpoints/endpoints.py @@ -319,36 +319,34 @@ async def create_agent( Example Request: ```bash - curl -X POST "http://localhost:4000/agents" \\ + curl -X POST "http://localhost:4000/v1/agents" \\ -H "Authorization: Bearer " \\ -H "Content-Type: application/json" \\ -d '{ - "agent": { - "agent_name": "my-custom-agent", - "agent_card_params": { - "protocolVersion": "1.0", - "name": "Hello World Agent", - "description": "Just a hello world agent", - "url": "http://localhost:9999/", - "version": "1.0.0", - "defaultInputModes": ["text"], - "defaultOutputModes": ["text"], - "capabilities": { - "streaming": true - }, - "skills": [ - { - "id": "hello_world", - "name": "Returns hello world", - "description": "just returns hello world", - "tags": ["hello world"], - "examples": ["hi", "hello world"] - } - ] + "agent_name": "my-custom-agent", + "agent_card_params": { + "protocolVersion": "1.0", + "name": "Hello World Agent", + "description": "Just a hello world agent", + "url": "http://localhost:9999/", + "version": "1.0.0", + "defaultInputModes": ["text"], + "defaultOutputModes": ["text"], + "capabilities": { + "streaming": true }, - "litellm_params": { - "make_public": true - } + "skills": [ + { + "id": "hello_world", + "name": "Returns hello world", + "description": "just returns hello world", + "tags": ["hello world"], + "examples": ["hi", "hello world"] + } + ] + }, + "litellm_params": { + "make_public": true } }' ``` @@ -441,7 +439,7 @@ async def get_agent_by_id( Example Request: ```bash - curl -X GET "http://localhost:4000/agents/123e4567-e89b-12d3-a456-426614174000" \\ + curl -X GET "http://localhost:4000/v1/agents/123e4567-e89b-12d3-a456-426614174000" \\ -H "Authorization: Bearer " ``` """ @@ -535,28 +533,26 @@ async def update_agent( Example Request: ```bash - curl -X PUT "http://localhost:4000/agents/123e4567-e89b-12d3-a456-426614174000" \\ + curl -X PUT "http://localhost:4000/v1/agents/123e4567-e89b-12d3-a456-426614174000" \\ -H "Authorization: Bearer " \\ -H "Content-Type: application/json" \\ -d '{ - "agent": { - "agent_name": "updated-agent", - "agent_card_params": { - "protocolVersion": "1.0", - "name": "Updated Agent", - "description": "Updated description", - "url": "http://localhost:9999/", - "version": "1.1.0", - "defaultInputModes": ["text"], - "defaultOutputModes": ["text"], - "capabilities": { - "streaming": true - }, - "skills": [] + "agent_name": "updated-agent", + "agent_card_params": { + "protocolVersion": "1.0", + "name": "Updated Agent", + "description": "Updated description", + "url": "http://localhost:9999/", + "version": "1.1.0", + "defaultInputModes": ["text"], + "defaultOutputModes": ["text"], + "capabilities": { + "streaming": true }, - "litellm_params": { - "make_public": false - } + "skills": [] + }, + "litellm_params": { + "make_public": false } }' ``` @@ -645,28 +641,26 @@ async def patch_agent( Example Request: ```bash - curl -X PUT "http://localhost:4000/agents/123e4567-e89b-12d3-a456-426614174000" \\ + curl -X PATCH "http://localhost:4000/v1/agents/123e4567-e89b-12d3-a456-426614174000" \\ -H "Authorization: Bearer " \\ -H "Content-Type: application/json" \\ -d '{ - "agent": { - "agent_name": "updated-agent", - "agent_card_params": { - "protocolVersion": "1.0", - "name": "Updated Agent", - "description": "Updated description", - "url": "http://localhost:9999/", - "version": "1.1.0", - "defaultInputModes": ["text"], - "defaultOutputModes": ["text"], - "capabilities": { - "streaming": true - }, - "skills": [] + "agent_name": "updated-agent", + "agent_card_params": { + "protocolVersion": "1.0", + "name": "Updated Agent", + "description": "Updated description", + "url": "http://localhost:9999/", + "version": "1.1.0", + "defaultInputModes": ["text"], + "defaultOutputModes": ["text"], + "capabilities": { + "streaming": true }, - "litellm_params": { - "make_public": false - } + "skills": [] + }, + "litellm_params": { + "make_public": false } }' ``` @@ -753,7 +747,7 @@ async def delete_agent( Example Request: ```bash - curl -X DELETE "http://localhost:4000/agents/123e4567-e89b-12d3-a456-426614174000" \\ + curl -X DELETE "http://localhost:4000/v1/agents/123e4567-e89b-12d3-a456-426614174000" \\ -H "Authorization: Bearer " ``` diff --git a/litellm/proxy/auth/auth_checks.py b/litellm/proxy/auth/auth_checks.py index e8d05031a5a..93a64889458 100644 --- a/litellm/proxy/auth/auth_checks.py +++ b/litellm/proxy/auth/auth_checks.py @@ -1127,7 +1127,7 @@ async def get_end_user_object( end_user_id: Optional[str], prisma_client: Optional[PrismaClient], user_api_key_cache: UserApiKeyCache, - route: str, + route: Optional[str] = "", parent_otel_span: Optional[Span] = None, proxy_logging_obj: Optional[ProxyLogging] = None, ) -> Optional[LiteLLM_EndUserTable]: @@ -1171,9 +1171,6 @@ async def get_end_user_object( parent_otel_span=parent_otel_span, ) - # Check budget limits - await _check_end_user_budget(end_user_obj=return_obj, route=route) - return return_obj # Fetch from database @@ -1204,14 +1201,9 @@ async def get_end_user_object( model_type=LiteLLM_EndUserTable, ) - # Check budget limits - await _check_end_user_budget(end_user_obj=_response, route=route) - return _response - except Exception as e: - if isinstance(e, litellm.BudgetExceededError): - raise e + except Exception: return None @@ -1308,8 +1300,6 @@ async def _end_user_id_exists_in_db( ) if end_user_obj is not None: return True - except litellm.BudgetExceededError: - raise except Exception as e: verbose_proxy_logger.debug( f"end_user validation: get_end_user_object lookup failed: {e}" diff --git a/litellm/proxy/auth/user_api_key_auth.py b/litellm/proxy/auth/user_api_key_auth.py index 2e6cd1f8e70..9d4efbaeeee 100644 --- a/litellm/proxy/auth/user_api_key_auth.py +++ b/litellm/proxy/auth/user_api_key_auth.py @@ -30,6 +30,7 @@ from litellm.proxy._types import * from litellm.proxy.auth.auth_checks import ( ExperimentalUIJWTToken, _cache_key_object, + _check_end_user_budget, _delete_cache_key_object, _get_user_role, _is_model_cost_zero, @@ -1762,8 +1763,7 @@ async def _user_api_key_auth_builder( # noqa: PLR0915 async def _safe_fetch(label: str, awaitable): """Run an awaitable and return its result. Re-raises authentication / authorization failures (HTTPException, ProxyException, - BudgetExceededError — which ``get_end_user_object`` raises for - end-user budget violations) so they propagate to the caller. + BudgetExceededError) so they propagate to the caller. Other exceptions (e.g. transient DB errors fetching context) are swallowed with a debug log and ``None`` is returned so ``common_checks`` can still run against whatever limits are recorded @@ -2584,6 +2584,14 @@ async def _run_post_custom_auth_checks( user_api_key_cache=user_api_key_cache, proxy_logging_obj=proxy_logging_obj, ) + # common_checks() enforces the end-user budget, but the centralized + # gate skips it for custom-auth deployments unless + # custom_auth_run_common_checks is set. Enforce it here on that path + # so an over-budget end user can't keep making requests. + if end_user_object is not None and not general_settings.get( + "custom_auth_run_common_checks", False + ): + await _check_end_user_budget(end_user_obj=end_user_object, route=route) # 2. Check token expiry if valid_token.expires is not None: diff --git a/litellm/router.py b/litellm/router.py index d60c39ca402..7aaf989919c 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -9100,7 +9100,10 @@ class Router: except Exception: pass + # Three mutually exclusive scenarios for the model's metadata: if custom_model_info is not None and litellm_model_name_model_info is not None: + # (1) It has both custom model_info set and exists in the built-in map + # merge with custom overriding built-in model_info = cast( ModelInfo, _update_dictionary( @@ -9109,7 +9112,12 @@ class Router: ), ) elif litellm_model_name_model_info is not None: + # (2) Built-in only — no custom pricing to merge model_info = litellm_model_name_model_info + elif custom_model_info is not None: + # (3) Custom only — model not in built-in cost map yet + # custom_model_info already includes base_model defaults at this point, if applicable + model_info = cast(ModelInfo, custom_model_info) return model_info diff --git a/litellm/types/images/main.py b/litellm/types/images/main.py index 819f4954589..80e55297c42 100644 --- a/litellm/types/images/main.py +++ b/litellm/types/images/main.py @@ -20,6 +20,7 @@ class ImageEditOptionalRequestParams(TypedDict, total=False): response_format: Optional[Literal["url", "b64_json"]] size: Optional[str] user: Optional[str] + imageConfig: Optional[Dict[str, Any]] class ImageEditRequestParams(ImageEditOptionalRequestParams, total=False): diff --git a/litellm/types/llms/gemini.py b/litellm/types/llms/gemini.py index 8763544facc..38e6d533449 100644 --- a/litellm/types/llms/gemini.py +++ b/litellm/types/llms/gemini.py @@ -1,7 +1,7 @@ from enum import Enum -from typing import Any, Dict, Iterable, List, Literal, Optional, Union +from typing import Any, Dict, List, Literal, Optional -from typing_extensions import Required, TypedDict +from typing_extensions import TypedDict from .vertex_ai import ( GenerationConfig, @@ -171,6 +171,9 @@ class GeminiImageGenerationParameters(BaseModel): aspectRatio: Optional[str] = None """Aspect ratio for generated images (e.g., '1:1', '16:9', '9:16', '4:3', '3:4')""" + imageSize: Optional[str] = None + """Image size for generated images (e.g., '1K', '2K')""" + personGeneration: Optional[str] = None """Controls person generation in images""" diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index 346909f14eb..51e408f8c63 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -1084,6 +1084,7 @@ OpenAIImageGenerationOptionalParams = Literal[ "image_url", "image_prompt_strength", "aspect_ratio", + "imageConfig", ] OpenAIImageEditOptionalParams = Literal[ diff --git a/litellm/types/llms/vertex_ai.py b/litellm/types/llms/vertex_ai.py index a1d53978761..b972ff3c538 100644 --- a/litellm/types/llms/vertex_ai.py +++ b/litellm/types/llms/vertex_ai.py @@ -20,6 +20,7 @@ class FunctionResponse(TypedDict, total=False): id: str name: Required[str] response: Optional[dict] + parts: List["FunctionResponsePartType"] class FunctionCall(TypedDict, total=False): @@ -40,6 +41,11 @@ class BlobType(TypedDict, total=False): data: Required[str] +class FunctionResponsePartType(TypedDict, total=False): + inline_data: BlobType + file_data: FileDataType + + class PartType(TypedDict, total=False): text: str inline_data: BlobType diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 63c2513aed2..3dcff2be689 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -148,6 +148,7 @@ class ProviderSpecificModelInfo(TypedDict, total=False): supports_xhigh_reasoning_effort: Optional[bool] supports_max_reasoning_effort: Optional[bool] supports_output_config: Optional[bool] + supports_image_size: Optional[bool] bedrock_output_config_effort_ceiling: Optional[ Literal["low", "medium", "high", "max", "xhigh"] ] @@ -3300,6 +3301,8 @@ class LlmProviders(str, Enum): V0 = "v0" MORPH = "morph" LAMBDA_AI = "lambda_ai" + INCEPTION = "inception" + TEXT_COMPLETION_INCEPTION = "text-completion-inception" DEEPSEEK = "deepseek" SAMBANOVA = "sambanova" MARITALK = "maritalk" diff --git a/litellm/utils.py b/litellm/utils.py index 6188206148f..7cac830b2c2 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -3147,6 +3147,7 @@ def get_optional_params_image_gen( size: Optional[str] = None, style: Optional[str] = None, user: Optional[str] = None, + imageConfig: Optional[dict] = None, custom_llm_provider: Optional[str] = None, additional_drop_params: Optional[list] = None, provider_config: Optional[BaseImageGenerationConfig] = None, @@ -3183,6 +3184,7 @@ def get_optional_params_image_gen( "size": None, "style": None, "user": None, + "imageConfig": None, } non_default_params = _get_non_default_params( @@ -4547,6 +4549,18 @@ def get_optional_params( # noqa: PLR0915 ), ) + elif custom_llm_provider == "text-completion-inception": + optional_params = litellm.InceptionTextCompletionConfig().map_openai_params( + non_default_params=non_default_params, + optional_params=optional_params, + model=model, + drop_params=( + drop_params + if drop_params is not None and isinstance(drop_params, bool) + else False + ), + ) + elif custom_llm_provider == "databricks": optional_params = litellm.DatabricksConfig().map_openai_params( non_default_params=non_default_params, @@ -6083,6 +6097,7 @@ def _get_model_info_helper( # noqa: PLR0915 "provider_specific_entry", None ), uses_embed_content=_model_info.get("uses_embed_content", None), + supports_image_size=_model_info.get("supports_image_size", None), ) except Exception as e: verbose_logger.debug(f"Error getting model info: {e}") @@ -6637,6 +6652,14 @@ def validate_environment( # noqa: PLR0915 keys_in_environment = True else: missing_keys.append("CODESTRAL_API_KEY") + elif ( + custom_llm_provider == "inception" + or custom_llm_provider == "text-completion-inception" + ): + if "INCEPTION_API_KEY" in os.environ: + keys_in_environment = True + else: + missing_keys.append("INCEPTION_API_KEY") elif custom_llm_provider == "deepseek": if "DEEPSEEK_API_KEY" in os.environ: keys_in_environment = True @@ -8291,6 +8314,7 @@ class ProviderConfigManager: LlmProviders.XAI: (lambda: litellm.XAIChatConfig(), False), LlmProviders.ZAI: (lambda: litellm.ZAIChatConfig(), False), LlmProviders.LAMBDA_AI: (lambda: litellm.LambdaAIChatConfig(), False), + LlmProviders.INCEPTION: (lambda: litellm.InceptionChatConfig(), False), LlmProviders.LLAMA: (lambda: litellm.LlamaAPIConfig(), False), LlmProviders.TEXT_COMPLETION_OPENAI: ( lambda: litellm.OpenAITextCompletionConfig(), @@ -8356,6 +8380,10 @@ class ProviderConfigManager: lambda: litellm.CodestralTextCompletionConfig(), False, ), + LlmProviders.TEXT_COMPLETION_INCEPTION: ( + lambda: litellm.InceptionTextCompletionConfig(), + False, + ), LlmProviders.SAMBANOVA: (lambda: litellm.SambanovaConfig(), False), LlmProviders.MARITALK: (lambda: litellm.MaritalkConfig(), False), LlmProviders.VLLM: (lambda: litellm.VLLMConfig(), False), @@ -8928,6 +8956,8 @@ class ProviderConfigManager: return litellm.FireworksAITextCompletionConfig() elif LlmProviders.TOGETHER_AI == provider: return litellm.TogetherAITextCompletionConfig() + elif LlmProviders.TEXT_COMPLETION_INCEPTION == provider: + return litellm.InceptionTextCompletionConfig() return litellm.OpenAITextCompletionConfig() @staticmethod diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index b253235ec2a..ed6de4fa6b7 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -1075,6 +1075,7 @@ }, "eu.anthropic.claude-opus-4-6-v1": { "cache_creation_input_token_cost": 6.875e-06, + "cache_creation_input_token_cost_above_1hr": 1.1e-05, "cache_read_input_token_cost": 5.5e-07, "input_cost_per_token": 5.5e-06, "litellm_provider": "bedrock_converse", @@ -1104,6 +1105,7 @@ }, "au.anthropic.claude-opus-4-6-v1": { "cache_creation_input_token_cost": 6.875e-06, + "cache_creation_input_token_cost_above_1hr": 1.1e-05, "cache_read_input_token_cost": 5.5e-07, "input_cost_per_token": 5.5e-06, "litellm_provider": "bedrock_converse", @@ -1241,6 +1243,7 @@ }, "eu.anthropic.claude-opus-4-7": { "cache_creation_input_token_cost": 6.875e-06, + "cache_creation_input_token_cost_above_1hr": 1.1e-05, "cache_read_input_token_cost": 5.5e-07, "input_cost_per_token": 5.5e-06, "litellm_provider": "bedrock_converse", @@ -1271,6 +1274,7 @@ }, "au.anthropic.claude-opus-4-7": { "cache_creation_input_token_cost": 6.875e-06, + "cache_creation_input_token_cost_above_1hr": 1.1e-05, "cache_read_input_token_cost": 5.5e-07, "input_cost_per_token": 5.5e-06, "litellm_provider": "bedrock_converse", @@ -1543,6 +1547,7 @@ }, "eu.anthropic.claude-sonnet-4-6": { "cache_creation_input_token_cost": 4.125e-06, + "cache_creation_input_token_cost_above_1hr": 6.6e-06, "cache_read_input_token_cost": 3.3e-07, "input_cost_per_token": 3.3e-06, "litellm_provider": "bedrock_converse", @@ -1571,6 +1576,7 @@ }, "au.anthropic.claude-sonnet-4-6": { "cache_creation_input_token_cost": 4.125e-06, + "cache_creation_input_token_cost_above_1hr": 6.6e-06, "cache_read_input_token_cost": 3.3e-07, "input_cost_per_token": 3.3e-06, "litellm_provider": "bedrock_converse", @@ -1599,6 +1605,7 @@ }, "jp.anthropic.claude-sonnet-4-6": { "cache_creation_input_token_cost": 4.125e-06, + "cache_creation_input_token_cost_above_1hr": 6.6e-06, "cache_read_input_token_cost": 3.3e-07, "input_cost_per_token": 3.3e-06, "litellm_provider": "bedrock_converse", @@ -1995,11 +2002,13 @@ }, "au.anthropic.claude-sonnet-4-5-20250929-v1:0": { "cache_creation_input_token_cost": 4.125e-06, + "cache_creation_input_token_cost_above_1hr": 6.6e-06, "cache_read_input_token_cost": 3.3e-07, "input_cost_per_token": 3.3e-06, "input_cost_per_token_above_200k_tokens": 6.6e-06, "output_cost_per_token_above_200k_tokens": 2.475e-05, "cache_creation_input_token_cost_above_200k_tokens": 8.25e-06, + "cache_creation_input_token_cost_above_1hr_above_200k_tokens": 1.32e-05, "cache_read_input_token_cost_above_200k_tokens": 6.6e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 200000, @@ -7494,6 +7503,27 @@ "supports_video_input": true, "supports_vision": true }, + "azure_ai/kimi-k2.6": { + "input_cost_per_token": 9.5e-07, + "litellm_provider": "azure_ai", + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 4e-06, + "source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/introducing-kimi-k2-6-in-microsoft-foundry/4513125", + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true + }, "azure_ai/ministral-3b": { "input_cost_per_token": 4e-08, "litellm_provider": "azure_ai", @@ -12682,7 +12712,8 @@ "litellm_provider": "deepinfra", "mode": "chat", "supports_tool_choice": true, - "supports_function_calling": true + "supports_function_calling": true, + "supports_image_size": false }, "deepinfra/google/gemini-2.5-pro": { "max_tokens": 1000000, @@ -13583,6 +13614,7 @@ }, "eu.anthropic.claude-haiku-4-5-20251001-v1:0": { "cache_creation_input_token_cost": 1.375e-06, + "cache_creation_input_token_cost_above_1hr": 2.2e-06, "cache_read_input_token_cost": 1.1e-07, "input_cost_per_token": 1.1e-06, "deprecation_date": "2026-10-15", @@ -13787,11 +13819,13 @@ }, "eu.anthropic.claude-sonnet-4-5-20250929-v1:0": { "cache_creation_input_token_cost": 4.125e-06, + "cache_creation_input_token_cost_above_1hr": 6.6e-06, "cache_read_input_token_cost": 3.3e-07, "input_cost_per_token": 3.3e-06, "input_cost_per_token_above_200k_tokens": 6.6e-06, "output_cost_per_token_above_200k_tokens": 2.475e-05, "cache_creation_input_token_cost_above_200k_tokens": 8.25e-06, + "cache_creation_input_token_cost_above_1hr_above_200k_tokens": 1.32e-05, "cache_read_input_token_cost_above_200k_tokens": 6.6e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 200000, @@ -15006,7 +15040,8 @@ "search_context_size_medium": 0.035, "search_context_size_high": 0.035 }, - "supports_service_tier": true + "supports_service_tier": true, + "supports_image_size": false }, "gemini-2.5-flash-image": { "cache_read_input_token_cost": 3e-08, @@ -15056,7 +15091,8 @@ "supports_vision": true, "supports_web_search": false, "tpm": 8000000, - "supports_service_tier": true + "supports_service_tier": true, + "supports_image_size": false }, "gemini-3-pro-image-preview": { "input_cost_per_image": 0.0011, @@ -15345,7 +15381,8 @@ "search_context_size_medium": 0.035, "search_context_size_high": 0.035 }, - "supports_service_tier": true + "supports_service_tier": true, + "supports_image_size": false }, "gemini-2.5-flash-lite-preview-09-2025": { "cache_read_input_token_cost": 1e-08, @@ -15395,7 +15432,8 @@ "search_context_size_low": 0.035, "search_context_size_medium": 0.035, "search_context_size_high": 0.035 - } + }, + "supports_image_size": false }, "gemini-2.5-flash-preview-09-2025": { "cache_read_input_token_cost": 7.5e-08, @@ -15445,7 +15483,8 @@ "search_context_size_low": 0.035, "search_context_size_medium": 0.035, "search_context_size_high": 0.035 - } + }, + "supports_image_size": false }, "gemini-live-2.5-flash-preview-native-audio-09-2025": { "cache_read_input_token_cost": 7.5e-08, @@ -15596,7 +15635,8 @@ "search_context_size_low": 0.035, "search_context_size_medium": 0.035, "search_context_size_high": 0.035 - } + }, + "supports_image_size": false }, "gemini-2.5-pro": { "cache_read_input_token_cost": 1.25e-07, @@ -16606,7 +16646,8 @@ "search_context_size_medium": 0.035, "search_context_size_high": 0.035 }, - "supports_service_tier": true + "supports_service_tier": true, + "supports_image_size": false }, "gemini/gemini-2.5-flash-image": { "cache_read_input_token_cost": 3e-08, @@ -16662,7 +16703,8 @@ "search_context_size_medium": 0.035, "search_context_size_high": 0.035 }, - "supports_service_tier": true + "supports_service_tier": true, + "supports_image_size": false }, "gemini/gemini-3-pro-image-preview": { "input_cost_per_image": 0.0011, @@ -16841,7 +16883,8 @@ "search_context_size_medium": 0.035, "search_context_size_high": 0.035 }, - "supports_service_tier": true + "supports_service_tier": true, + "supports_image_size": false }, "gemini/gemini-2.5-flash-lite-preview-09-2025": { "cache_read_input_token_cost": 1e-08, @@ -16893,7 +16936,8 @@ "search_context_size_low": 0.035, "search_context_size_medium": 0.035, "search_context_size_high": 0.035 - } + }, + "supports_image_size": false }, "gemini/gemini-2.5-flash-preview-09-2025": { "cache_read_input_token_cost": 7.5e-08, @@ -16945,7 +16989,8 @@ "search_context_size_low": 0.035, "search_context_size_medium": 0.035, "search_context_size_high": 0.035 - } + }, + "supports_image_size": false }, "gemini/gemini-flash-latest": { "cache_read_input_token_cost": 7.5e-08, @@ -17102,7 +17147,8 @@ "search_context_size_low": 0.035, "search_context_size_medium": 0.035, "search_context_size_high": 0.035 - } + }, + "supports_image_size": false }, "gemini/gemini-2.5-flash-preview-tts": { "input_cost_per_token": 3e-07, @@ -23086,11 +23132,13 @@ }, "jp.anthropic.claude-sonnet-4-5-20250929-v1:0": { "cache_creation_input_token_cost": 4.125e-06, + "cache_creation_input_token_cost_above_1hr": 6.6e-06, "cache_read_input_token_cost": 3.3e-07, "input_cost_per_token": 3.3e-06, "input_cost_per_token_above_200k_tokens": 6.6e-06, "output_cost_per_token_above_200k_tokens": 2.475e-05, "cache_creation_input_token_cost_above_200k_tokens": 8.25e-06, + "cache_creation_input_token_cost_above_1hr_above_200k_tokens": 1.32e-05, "cache_read_input_token_cost_above_200k_tokens": 6.6e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 200000, @@ -23116,6 +23164,7 @@ }, "jp.anthropic.claude-haiku-4-5-20251001-v1:0": { "cache_creation_input_token_cost": 1.375e-06, + "cache_creation_input_token_cost_above_1hr": 2.2e-06, "cache_read_input_token_cost": 1.1e-07, "input_cost_per_token": 1.1e-06, "litellm_provider": "bedrock_converse", @@ -23228,6 +23277,31 @@ "supports_system_messages": true, "supports_tool_choice": true }, + "inception/mercury-2": { + "cache_read_input_token_cost": 2.5e-08, + "input_cost_per_token": 2.5e-07, + "litellm_provider": "inception", + "max_input_tokens": 128000, + "max_output_tokens": 50000, + "max_tokens": 50000, + "mode": "chat", + "output_cost_per_token": 7.5e-07, + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "text-completion-inception/mercury-edit-2": { + "cache_read_input_token_cost": 2.5e-08, + "input_cost_per_token": 2.5e-07, + "litellm_provider": "text-completion-inception", + "max_input_tokens": 32000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "completion", + "output_cost_per_token": 7.5e-07 + }, "lambda_ai/deepseek-llama3.3-70b": { "input_cost_per_token": 2e-07, "litellm_provider": "lambda_ai", @@ -26449,7 +26523,8 @@ "supports_function_calling": true, "supports_response_schema": true, "supports_vision": true, - "supports_native_streaming": true + "supports_native_streaming": true, + "supports_image_size": false }, "oci/google.gemini-2.5-pro": { "input_cost_per_token": 1.25e-06, @@ -26477,7 +26552,8 @@ "supports_function_calling": true, "supports_response_schema": true, "supports_vision": true, - "supports_native_streaming": true + "supports_native_streaming": true, + "supports_image_size": false }, "oci/cohere.command-a-vision": { "input_cost_per_token": 1.56e-06, @@ -27513,7 +27589,8 @@ "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "supports_image_size": false }, "openrouter/google/gemini-2.5-pro": { "input_cost_per_audio_token": 7e-07, @@ -29383,7 +29460,8 @@ "mode": "responses", "supports_web_search": true, "supports_reasoning": false, - "supports_function_calling": true + "supports_function_calling": true, + "supports_image_size": false }, "perplexity/xai/grok-4-1-fast-non-reasoning": { "litellm_provider": "perplexity", @@ -29965,7 +30043,8 @@ "supports_vision": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_response_schema": true + "supports_response_schema": true, + "supports_image_size": false }, "replicate/openai/gpt-oss-120b": { "input_cost_per_token": 1.8e-07, @@ -31784,6 +31863,7 @@ }, "au.anthropic.claude-haiku-4-5-20251001-v1:0": { "cache_creation_input_token_cost": 1.375e-06, + "cache_creation_input_token_cost_above_1hr": 2.2e-06, "cache_read_input_token_cost": 1.1e-07, "input_cost_per_token": 1.1e-06, "litellm_provider": "bedrock_converse", @@ -32663,7 +32743,8 @@ "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, - "supports_response_schema": true + "supports_response_schema": true, + "supports_image_size": false }, "vercel_ai_gateway/google/gemini-2.5-pro": { "input_cost_per_token": 2.5e-06, @@ -33430,6 +33511,7 @@ }, "vertex_ai/claude-haiku-4-5": { "cache_creation_input_token_cost": 1.25e-06, + "cache_creation_input_token_cost_above_1hr": 2e-06, "cache_read_input_token_cost": 1e-07, "input_cost_per_token": 1e-06, "litellm_provider": "vertex_ai-anthropic_models", @@ -33451,6 +33533,7 @@ }, "vertex_ai/claude-haiku-4-5@20251001": { "cache_creation_input_token_cost": 1.25e-06, + "cache_creation_input_token_cost_above_1hr": 2e-06, "cache_read_input_token_cost": 1e-07, "input_cost_per_token": 1e-06, "litellm_provider": "vertex_ai-anthropic_models", @@ -33501,6 +33584,7 @@ }, "vertex_ai/claude-3-7-sonnet@20250219": { "cache_creation_input_token_cost": 3.75e-06, + "cache_creation_input_token_cost_above_1hr": 6e-06, "cache_read_input_token_cost": 3e-07, "deprecation_date": "2026-05-11", "input_cost_per_token": 3e-06, @@ -33600,6 +33684,7 @@ }, "vertex_ai/claude-opus-4": { "cache_creation_input_token_cost": 1.875e-05, + "cache_creation_input_token_cost_above_1hr": 3e-05, "cache_read_input_token_cost": 1.5e-06, "input_cost_per_token": 1.5e-05, "litellm_provider": "vertex_ai-anthropic_models", @@ -33625,6 +33710,7 @@ }, "vertex_ai/claude-opus-4-1": { "cache_creation_input_token_cost": 1.875e-05, + "cache_creation_input_token_cost_above_1hr": 3e-05, "cache_read_input_token_cost": 1.5e-06, "input_cost_per_token": 1.5e-05, "input_cost_per_token_batches": 7.5e-06, @@ -33642,6 +33728,7 @@ }, "vertex_ai/claude-opus-4-1@20250805": { "cache_creation_input_token_cost": 1.875e-05, + "cache_creation_input_token_cost_above_1hr": 3e-05, "cache_read_input_token_cost": 1.5e-06, "input_cost_per_token": 1.5e-05, "input_cost_per_token_batches": 7.5e-06, @@ -33659,6 +33746,7 @@ }, "vertex_ai/claude-opus-4-5": { "cache_creation_input_token_cost": 6.25e-06, + "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, "input_cost_per_token": 5e-06, "litellm_provider": "vertex_ai-anthropic_models", @@ -33685,6 +33773,7 @@ }, "vertex_ai/claude-opus-4-5@20251101": { "cache_creation_input_token_cost": 6.25e-06, + "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, "input_cost_per_token": 5e-06, "litellm_provider": "vertex_ai-anthropic_models", @@ -33712,6 +33801,7 @@ }, "vertex_ai/claude-opus-4-6": { "cache_creation_input_token_cost": 6.25e-06, + "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, "input_cost_per_token": 5e-06, "litellm_provider": "vertex_ai-anthropic_models", @@ -33739,6 +33829,7 @@ }, "vertex_ai/claude-opus-4-6@default": { "cache_creation_input_token_cost": 6.25e-06, + "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, "input_cost_per_token": 5e-06, "litellm_provider": "vertex_ai-anthropic_models", @@ -33766,6 +33857,7 @@ }, "vertex_ai/claude-opus-4-7": { "cache_creation_input_token_cost": 6.25e-06, + "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, "input_cost_per_token": 5e-06, "litellm_provider": "vertex_ai-anthropic_models", @@ -33793,6 +33885,7 @@ }, "vertex_ai/claude-opus-4-7@default": { "cache_creation_input_token_cost": 6.25e-06, + "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, "input_cost_per_token": 5e-06, "litellm_provider": "vertex_ai-anthropic_models", @@ -33876,6 +33969,7 @@ }, "vertex_ai/claude-sonnet-4-5": { "cache_creation_input_token_cost": 3.75e-06, + "cache_creation_input_token_cost_above_1hr": 6e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_token": 3e-06, "input_cost_per_token_above_200k_tokens": 6e-06, @@ -33902,6 +33996,7 @@ }, "vertex_ai/claude-sonnet-4-6": { "cache_creation_input_token_cost": 3.75e-06, + "cache_creation_input_token_cost_above_1hr": 6e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_token": 3e-06, "litellm_provider": "vertex_ai-anthropic_models", @@ -33929,6 +34024,7 @@ }, "vertex_ai/claude-sonnet-4-5@20250929": { "cache_creation_input_token_cost": 3.75e-06, + "cache_creation_input_token_cost_above_1hr": 6e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_token": 3e-06, "input_cost_per_token_above_200k_tokens": 6e-06, @@ -33956,6 +34052,7 @@ }, "vertex_ai/claude-opus-4@20250514": { "cache_creation_input_token_cost": 1.875e-05, + "cache_creation_input_token_cost_above_1hr": 3e-05, "cache_read_input_token_cost": 1.5e-06, "input_cost_per_token": 1.5e-05, "litellm_provider": "vertex_ai-anthropic_models", @@ -33981,6 +34078,7 @@ }, "vertex_ai/claude-sonnet-4": { "cache_creation_input_token_cost": 3.75e-06, + "cache_creation_input_token_cost_above_1hr": 6e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_token": 3e-06, "input_cost_per_token_above_200k_tokens": 6e-06, @@ -34010,6 +34108,7 @@ }, "vertex_ai/claude-sonnet-4@20250514": { "cache_creation_input_token_cost": 3.75e-06, + "cache_creation_input_token_cost_above_1hr": 6e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_token": 3e-06, "input_cost_per_token_above_200k_tokens": 6e-06, @@ -34217,7 +34316,8 @@ "supports_url_context": true, "supports_vision": true, "supports_web_search": false, - "tpm": 8000000 + "tpm": 8000000, + "supports_image_size": false }, "vertex_ai/gemini-3-pro-image-preview": { "input_cost_per_image": 0.0011, @@ -41035,6 +41135,7 @@ }, "vertex_ai/claude-sonnet-4-6@default": { "cache_creation_input_token_cost": 3.75e-06, + "cache_creation_input_token_cost_above_1hr": 6e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_token": 3e-06, "litellm_provider": "vertex_ai-anthropic_models", diff --git a/provider_endpoints_support.json b/provider_endpoints_support.json index 3a01541060f..b4f782f9c3e 100644 --- a/provider_endpoints_support.json +++ b/provider_endpoints_support.json @@ -1273,6 +1273,24 @@ "interactions": true } }, + "inception": { + "display_name": "Inception (`inception`)", + "url": "https://docs.litellm.ai/docs/providers/inception", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, "infinity": { "display_name": "Infinity (`infinity`)", "url": "https://docs.litellm.ai/docs/providers/infinity", diff --git a/tests/llm_translation/test_gemini.py b/tests/llm_translation/test_gemini.py index bd340aa63be..0a5aebdf91b 100644 --- a/tests/llm_translation/test_gemini.py +++ b/tests/llm_translation/test_gemini.py @@ -17,6 +17,66 @@ from litellm import completion import json +GEMINI_3_IMAGE_SIZE_MAPPINGS = [ + ("512x512", "1:1", "512"), + ("1024x1024", "1:1", "1K"), + ("2048x2048", "1:1", "2K"), + ("4096x4096", "1:1", "4K"), + ("256x1024", "1:4", "512"), + ("512x2048", "1:4", "1K"), + ("1024x4096", "1:4", "2K"), + ("2048x8192", "1:4", "4K"), + ("192x1536", "1:8", "512"), + ("384x3072", "1:8", "1K"), + ("768x6144", "1:8", "2K"), + ("1536x12288", "1:8", "4K"), + ("424x632", "2:3", "512"), + ("848x1264", "2:3", "1K"), + ("1696x2528", "2:3", "2K"), + ("3392x5056", "2:3", "4K"), + ("632x424", "3:2", "512"), + ("1264x848", "3:2", "1K"), + ("2528x1696", "3:2", "2K"), + ("5056x3392", "3:2", "4K"), + ("448x600", "3:4", "512"), + ("896x1200", "3:4", "1K"), + ("1792x2400", "3:4", "2K"), + ("3584x4800", "3:4", "4K"), + ("1024x256", "4:1", "512"), + ("2048x512", "4:1", "1K"), + ("4096x1024", "4:1", "2K"), + ("8192x2048", "4:1", "4K"), + ("600x448", "4:3", "512"), + ("1200x896", "4:3", "1K"), + ("2400x1792", "4:3", "2K"), + ("4800x3584", "4:3", "4K"), + ("464x576", "4:5", "512"), + ("928x1152", "4:5", "1K"), + ("1856x2304", "4:5", "2K"), + ("3712x4608", "4:5", "4K"), + ("576x464", "5:4", "512"), + ("1152x928", "5:4", "1K"), + ("2304x1856", "5:4", "2K"), + ("4608x3712", "5:4", "4K"), + ("1536x192", "8:1", "512"), + ("3072x384", "8:1", "1K"), + ("6144x768", "8:1", "2K"), + ("12288x1536", "8:1", "4K"), + ("384x688", "9:16", "512"), + ("768x1376", "9:16", "1K"), + ("1536x2752", "9:16", "2K"), + ("3072x5504", "9:16", "4K"), + ("688x384", "16:9", "512"), + ("1376x768", "16:9", "1K"), + ("2752x1536", "16:9", "2K"), + ("5504x3072", "16:9", "4K"), + ("792x336", "21:9", "512"), + ("1584x672", "21:9", "1K"), + ("3168x1344", "21:9", "2K"), + ("6336x2688", "21:9", "4K"), +] + + class TestGoogleAIStudioGemini(BaseLLMChatTest): def get_base_completion_call_args(self) -> dict: return {"model": "gemini/gemini-2.5-flash"} @@ -365,6 +425,143 @@ def test_gemini_flash_image_preview_models(model_name: str): ] +@pytest.mark.parametrize( + "model, kwargs, expected_image_config", + [ + ( + "gemini/gemini-3-pro-image-preview", + {"imageConfig": {"aspectRatio": "16:9", "imageSize": "512px"}}, + {"aspectRatio": "16:9", "imageSize": "512px"}, + ), + ( + "gemini/gemini-2.5-flash-image", + {"size": "2048x2048"}, + {"aspectRatio": "1:1"}, + ), + ], +) +def test_gemini_image_generation_forwards_image_config( + model: str, kwargs: dict, expected_image_config: dict +): + from unittest.mock import patch, MagicMock + + with patch( + "litellm.llms.custom_httpx.llm_http_handler.HTTPHandler.post" + ) as mock_post: + mock_http_response = MagicMock() + mock_http_response.json.return_value = { + "candidates": [ + { + "content": { + "parts": [{"inlineData": {"data": "test_base64_image_data"}}] + } + } + ] + } + mock_http_response.status_code = 200 + mock_post.return_value = mock_http_response + + litellm.image_generation( + model=model, + prompt="Generate a simple test image", + api_key="test_api_key", + **kwargs, + ) + + request_data = mock_post.call_args.kwargs.get("json", {}) + assert request_data["generationConfig"]["imageConfig"] == expected_image_config + + +def test_gemini_image_generation_image_config_takes_precedence_over_size(): + from litellm.llms.gemini.image_generation.transformation import GoogleImageGenConfig + + explicit_image_config = {"aspectRatio": "16:9", "imageSize": "2K"} + + mapped_params = GoogleImageGenConfig().map_openai_params( + non_default_params={ + "imageConfig": explicit_image_config, + "size": "768x1376", + }, + optional_params={}, + model="gemini-3-pro-image-preview", + drop_params=False, + ) + + assert mapped_params["imageConfig"] == explicit_image_config + + +def test_gemini_image_generation_ignores_non_dict_image_config(): + from litellm.llms.gemini.image_generation.transformation import GoogleImageGenConfig + + mapped_params = GoogleImageGenConfig().map_openai_params( + non_default_params={ + "size": "768x1376", + "imageConfig": "not-a-dict", + }, + optional_params={}, + model="gemini-3-pro-image-preview", + drop_params=False, + ) + + assert mapped_params["imageConfig"] == {"aspectRatio": "9:16", "imageSize": "1K"} + + +@pytest.mark.parametrize( + "size, expected_aspect_ratio, expected_image_size", + GEMINI_3_IMAGE_SIZE_MAPPINGS, +) +def test_gemini_image_generation_openai_size_maps_to_google_table( + size: str, expected_aspect_ratio: str, expected_image_size: str +): + from litellm.llms.gemini.common_utils import ( + map_openai_size_to_gemini_image_config, + ) + + assert map_openai_size_to_gemini_image_config( + size, "gemini-3-pro-image-preview" + ) == { + "aspectRatio": expected_aspect_ratio, + "imageSize": expected_image_size, + } + + +@pytest.mark.parametrize( + "size, expected_aspect_ratio, expected_image_size", + [ + ("1000x1800", "9:16", "1K"), + ("1800x1000", "16:9", "1K"), + ("3000x3000", "1:1", "2K"), + ("500x500", "1:1", "512"), + ("1280x896", "4:3", "1K"), + ("896x1280", "3:4", "1K"), + ], +) +def test_gemini_image_generation_openai_size_snaps_to_nearest_option( + size: str, expected_aspect_ratio: str, expected_image_size: str +): + from litellm.llms.gemini.common_utils import ( + map_openai_size_to_gemini_image_config, + ) + + assert map_openai_size_to_gemini_image_config( + size, "gemini-3-pro-image-preview" + ) == { + "aspectRatio": expected_aspect_ratio, + "imageSize": expected_image_size, + } + + +@pytest.mark.parametrize("size", ["auto", "invalid", "0x1024", "1024x0"]) +def test_gemini_image_generation_openai_size_auto_uses_google_defaults(size: str): + from litellm.llms.gemini.common_utils import ( + map_openai_size_to_gemini_image_config, + ) + + assert map_openai_size_to_gemini_image_config( + size, "gemini-3-pro-image-preview" + ) is None + + def test_gemini_imagen_models_use_predict_endpoint(): """ Test that Imagen models still use :predict endpoint (not broken by gemini-2.5-flash-image-preview fix) @@ -387,6 +584,7 @@ def test_gemini_imagen_models_use_predict_endpoint(): response = litellm.image_generation( model="gemini/imagen-3.0-generate-001", prompt="Generate a simple test image", + size="1280x896", api_key="test_api_key", ) @@ -410,6 +608,9 @@ def test_gemini_imagen_models_use_predict_endpoint(): request_data = call_args.kwargs.get("json", {}) assert "instances" in request_data assert "parameters" in request_data + assert request_data["parameters"]["aspectRatio"] == "4:3" + assert request_data["parameters"]["imageSize"] == "1K" + assert "imageConfig" not in request_data["parameters"] def test_gemini_thinking(): diff --git a/tests/proxy_unit_tests/test_auth_checks.py b/tests/proxy_unit_tests/test_auth_checks.py index d9f4a6e56b8..e7136ecb195 100644 --- a/tests/proxy_unit_tests/test_auth_checks.py +++ b/tests/proxy_unit_tests/test_auth_checks.py @@ -38,8 +38,12 @@ from litellm.proxy.utils import CallInfo @pytest.mark.asyncio async def test_get_end_user_object(customer_spend, customer_budget): """ - Scenario 1: normal - Scenario 2: user over budget + Scenario 1: normal - get_end_user_object returns the cached user + Scenario 2: user over budget - NOTE: budget enforcement now happens in + common_checks() via _check_end_user_budget(), not in get_end_user_object() + + This test verifies that get_end_user_object correctly retrieves the end user + from cache. Budget enforcement is tested separately in test_check_end_user_budget(). """ end_user_id = "my-test-customer" _budget = LiteLLM_BudgetTable(max_budget=customer_budget) @@ -58,31 +62,62 @@ async def test_get_end_user_object(customer_spend, customer_budget): value=end_user_obj, model_type=LiteLLM_EndUserTable, ) + # get_end_user_object only fetches data - it no longer enforces budget + # Budget enforcement happens in common_checks() via _check_end_user_budget() + result = await get_end_user_object( + end_user_id=end_user_id, + prisma_client="RANDOM VALUE", # type: ignore + user_api_key_cache=_cache, + route="/v1/chat/completions", + ) + assert result is not None + assert result.user_id == end_user_id + + +@pytest.mark.parametrize("customer_spend, customer_budget", [(0, 10), (10, 0)]) +@pytest.mark.asyncio +async def test_check_end_user_budget(customer_spend, customer_budget): + """ + Test _check_end_user_budget enforcement: + - Scenario 1: customer_spend=0, customer_budget=10 - should pass (under budget) + - Scenario 2: customer_spend=10, customer_budget=0 - should fail (over budget) + + Note: Budget enforcement for end users happens in common_checks() via + _check_end_user_budget(), not in get_end_user_object(). + """ + from litellm.proxy.auth.auth_checks import _check_end_user_budget + + _budget = LiteLLM_BudgetTable(max_budget=customer_budget) + end_user_obj = LiteLLM_EndUserTable( + user_id="my-test-customer", + spend=customer_spend, + litellm_budget_table=_budget, + blocked=False, + ) + + should_exceed = customer_spend > customer_budget + try: - await get_end_user_object( - end_user_id=end_user_id, - prisma_client="RANDOM VALUE", # type: ignore - user_api_key_cache=_cache, + await _check_end_user_budget( + end_user_obj=end_user_obj, route="/v1/chat/completions", ) - if customer_spend > customer_budget: + if should_exceed: pytest.fail( - "Expected call to fail. Customer Spend={}, Customer Budget={}".format( + "Expected BudgetExceededError. Customer Spend={}, Customer Budget={}".format( customer_spend, customer_budget ) ) - except Exception as e: - if ( - isinstance(e, litellm.BudgetExceededError) - and customer_spend > customer_budget - ): - pass - else: + except litellm.BudgetExceededError as e: + if not should_exceed: pytest.fail( - "Expected call to work. Customer Spend={}, Customer Budget={}, Error={}".format( + "Unexpected BudgetExceededError. Customer Spend={}, Customer Budget={}, Error={}".format( customer_spend, customer_budget, str(e) ) ) + # Verify the error has correct info + assert e.current_cost == customer_spend + assert e.max_budget == customer_budget @pytest.mark.parametrize( diff --git a/tests/proxy_unit_tests/test_default_end_user_budget_simple.py b/tests/proxy_unit_tests/test_default_end_user_budget_simple.py index 970a7ab4718..6170b0a972e 100644 --- a/tests/proxy_unit_tests/test_default_end_user_budget_simple.py +++ b/tests/proxy_unit_tests/test_default_end_user_budget_simple.py @@ -134,9 +134,14 @@ async def test_explicit_budget_not_overridden_by_default(): @pytest.mark.asyncio async def test_budget_enforcement_blocks_over_budget_users(): """ - Core scenario: Budget limits are actually enforced. + Core scenario: Budget limits are actually enforced via _check_end_user_budget. Users who exceed their budget should be blocked. + + Note: Budget enforcement happens in common_checks() via _check_end_user_budget(), + not in get_end_user_object(). get_end_user_object only fetches the user data. """ + from litellm.proxy.auth.auth_checks import _check_end_user_budget + end_user_id = f"test_user_{uuid.uuid4().hex}" default_budget_id = str(uuid.uuid4()) litellm.max_end_user_budget_id = default_budget_id @@ -170,12 +175,23 @@ async def test_budget_enforcement_blocks_over_budget_users(): mock_cache.async_get_cache = AsyncMock(return_value=None) mock_cache.async_set_cache = AsyncMock() - # Should raise BudgetExceededError + # First, get the end user object (this just fetches data, doesn't enforce budget) + result = await get_end_user_object( + end_user_id=end_user_id, + prisma_client=mock_prisma_client, + user_api_key_cache=mock_cache, + route="/chat/completions", + ) + + # Verify user was fetched with default budget applied + assert result is not None + assert result.litellm_budget_table is not None + assert result.litellm_budget_table.max_budget == 10.0 + + # Now test budget enforcement separately via _check_end_user_budget with pytest.raises(litellm.BudgetExceededError) as exc_info: - await get_end_user_object( - end_user_id=end_user_id, - prisma_client=mock_prisma_client, - user_api_key_cache=mock_cache, + await _check_end_user_budget( + end_user_obj=result, route="/chat/completions", ) diff --git a/tests/proxy_unit_tests/test_proxy_server.py b/tests/proxy_unit_tests/test_proxy_server.py index 9c08175767d..6fdabe64e25 100644 --- a/tests/proxy_unit_tests/test_proxy_server.py +++ b/tests/proxy_unit_tests/test_proxy_server.py @@ -804,6 +804,7 @@ def test_img_gen(mock_aimage_generation, client_no_auth): "prompt": "A cute baby sea otter", "n": 1, "size": "1024x1024", + "imageConfig": {"aspectRatio": "9:16", "imageSize": "1K"}, } response = client_no_auth.post("/v1/images/generations", json=test_data) @@ -813,6 +814,7 @@ def test_img_gen(mock_aimage_generation, client_no_auth): prompt="A cute baby sea otter", n=1, size="1024x1024", + imageConfig={"aspectRatio": "9:16", "imageSize": "1K"}, metadata=mock.ANY, proxy_server_request=mock.ANY, secret_fields=mock.ANY, diff --git a/tests/test_litellm/completion_extras/__init__.py b/tests/test_litellm/completion_extras/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/test_litellm/completion_extras/test_responses_bridge_provider_propagation.py b/tests/test_litellm/completion_extras/test_responses_bridge_provider_propagation.py new file mode 100644 index 00000000000..b41dbd54b85 --- /dev/null +++ b/tests/test_litellm/completion_extras/test_responses_bridge_provider_propagation.py @@ -0,0 +1,116 @@ +""" +Regression test for https://github.com/BerriAI/litellm/issues/28505 - +the Responses API bridge double-strips the provider prefix from the +model name when a Chat Completions request has both `tools` and +`reasoning_effort`. + +Root cause: the bridge handler called `litellm.responses()` / +`litellm.aresponses()` without passing the already-resolved +`custom_llm_provider`. The downstream call then re-invoked +`get_llm_provider()` with `custom_llm_provider=None`, which stripped +a second provider prefix from a `provider/provider/model` deployment +string. + +This test pins both the sync and async bridge handler call sites: +the resolved `custom_llm_provider` must be forwarded to the underlying +`responses` / `aresponses` call so the provider isn't re-detected. +""" + +from unittest.mock import MagicMock, patch + +import pytest + +from litellm.completion_extras.litellm_responses_transformation.handler import ( + ResponsesToCompletionBridgeHandler, +) + + +def _validated_kwargs(): + return { + "model": "openai/openai/openai/gpt-5.5", + "messages": [{"role": "user", "content": "hi"}], + "optional_params": {}, + "litellm_params": {}, + "headers": {}, + "model_response": MagicMock(), + "logging_obj": MagicMock(), + "custom_llm_provider": "openai", + } + + +def test_sync_completion_forwards_custom_llm_provider(): + handler = ResponsesToCompletionBridgeHandler() + handler.transformation_handler = MagicMock() + handler.transformation_handler.transform_request.return_value = { + "model": "openai/openai/openai/gpt-5.5", + "input": [], + # `_build_sanitized_litellm_params` spreads `custom_llm_provider` from + # `litellm_params` into request_data on the real bridge path. Seed + # it here so the test exercises the overwrite (not an explicit kwarg + # that would TypeError against an already-present key). + "custom_llm_provider": "should-be-overwritten", + } + handler.transformation_handler.transform_response.return_value = ( + _validated_kwargs()["model_response"] + ) + with ( + patch.object( + handler, "validate_input_kwargs", return_value=_validated_kwargs() + ), + patch( + "litellm.responses", + return_value=MagicMock(spec=[]), + ) as mock_responses, + ): + # The handler routes ResponsesAPIResponse through transform_response. + # We just want to verify the kwargs going INTO responses(). + try: + handler.completion(acompletion=False) + except Exception: + # Downstream handling (transform_response, type checks) is not + # the subject of this test. + pass + assert mock_responses.called + kwargs = mock_responses.call_args.kwargs + assert kwargs.get("custom_llm_provider") == "openai", ( + "sync bridge must forward custom_llm_provider to litellm.responses() " + "so the downstream get_llm_provider() call does not re-strip the " + "provider prefix on a provider/provider/model deployment string" + ) + + +@pytest.mark.asyncio +async def test_async_completion_forwards_custom_llm_provider(): + handler = ResponsesToCompletionBridgeHandler() + handler.transformation_handler = MagicMock() + handler.transformation_handler.transform_request.return_value = { + "model": "openai/openai/openai/gpt-5.5", + "input": [], + # `_build_sanitized_litellm_params` spreads `custom_llm_provider` from + # `litellm_params` into request_data on the real bridge path. Seed + # it here so the test exercises the overwrite (not an explicit kwarg + # that would TypeError against an already-present key). + "custom_llm_provider": "should-be-overwritten", + } + + async def _fake_aresponses(**kwargs): + _fake_aresponses.kwargs = kwargs + return MagicMock(spec=[]) + + _fake_aresponses.kwargs = {} + + with ( + patch.object( + handler, "validate_input_kwargs", return_value=_validated_kwargs() + ), + patch("litellm.aresponses", _fake_aresponses), + ): + try: + await handler.acompletion() + except Exception: + pass + assert _fake_aresponses.kwargs.get("custom_llm_provider") == "openai", ( + "async bridge must forward custom_llm_provider to litellm.aresponses() " + "so the downstream get_llm_provider() call does not re-strip the " + "provider prefix on a provider/provider/model deployment string" + ) diff --git a/tests/test_litellm/integrations/opik/test_opik_extractors.py b/tests/test_litellm/integrations/opik/test_opik_extractors.py new file mode 100644 index 00000000000..6f85a1c6090 --- /dev/null +++ b/tests/test_litellm/integrations/opik/test_opik_extractors.py @@ -0,0 +1,84 @@ +from litellm.integrations.opik.opik_payload_builder.extractors import ( + extract_opik_metadata, +) + + +def test_extract_opik_metadata_fills_missing_keys_from_auth_metadata(): + litellm_metadata = {"opik": {"project_name": "my-proj"}} + standard_logging_metadata = { + "user_api_key_auth_metadata": { + "opik": { + "workspace": "auth-workspace", + "project_name": "auth-project", + } + } + } + + result = extract_opik_metadata( + litellm_metadata=litellm_metadata, + standard_logging_metadata=standard_logging_metadata, + ) + + assert result == { + "project_name": "my-proj", + "workspace": "auth-workspace", + } + + +def test_extract_opik_metadata_request_metadata_overrides_auth_metadata(): + litellm_metadata = { + "opik": { + "workspace": "request-workspace", + "thread_id": "request-thread", + } + } + standard_logging_metadata = { + "user_api_key_auth_metadata": { + "opik": { + "workspace": "auth-workspace", + "thread_id": "auth-thread", + "project_name": "auth-project", + } + } + } + + result = extract_opik_metadata( + litellm_metadata=litellm_metadata, + standard_logging_metadata=standard_logging_metadata, + ) + + assert result == { + "workspace": "request-workspace", + "thread_id": "request-thread", + "project_name": "auth-project", + } + + +def test_extract_opik_metadata_requester_metadata_overrides_all_other_sources(): + litellm_metadata = {"opik": {"project_name": "request-project"}} + standard_logging_metadata = { + "user_api_key_auth_metadata": { + "opik": { + "workspace": "auth-workspace", + "project_name": "auth-project", + } + }, + "requester_metadata": { + "opik": { + "workspace": "requester-workspace", + "thread_id": "requester-thread", + "project_name": "requester-project", + } + }, + } + + result = extract_opik_metadata( + litellm_metadata=litellm_metadata, + standard_logging_metadata=standard_logging_metadata, + ) + + assert result == { + "project_name": "requester-project", + "workspace": "requester-workspace", + "thread_id": "requester-thread", + } diff --git a/tests/test_litellm/integrations/test_opentelemetry.py b/tests/test_litellm/integrations/test_opentelemetry.py index f6dee9b64c9..0601f9c0eef 100644 --- a/tests/test_litellm/integrations/test_opentelemetry.py +++ b/tests/test_litellm/integrations/test_opentelemetry.py @@ -1263,7 +1263,6 @@ class TestOpenTelemetry(unittest.TestCase): ) as mock_get_headers, patch.object(otel, "_get_tracer_with_dynamic_headers") as mock_get_tracer, ): - # Test case 1: With dynamic headers mock_get_headers.return_value = { "arize-space-id": "test-space", @@ -2668,7 +2667,7 @@ class TestOpenTelemetryExternalSpan(unittest.TestCase): # Verify parent span is still recording after each call self.assertTrue( parent_span.is_recording(), - f"External span should still be recording after completion #{i+1}", + f"External span should still be recording after completion #{i + 1}", ) # Verify all spans have the same trace_id @@ -5170,6 +5169,138 @@ class TestOpenTelemetryPreprocessingDuration(unittest.TestCase): assert "litellm.preprocessing.duration_ms" not in self._attr(span, exp) +class TestGetSpanContextLitellmMetadataFallback(unittest.TestCase): + """ + Tests for _get_span_context() falling back to litellm_metadata. + + On /v1/messages (Anthropic Messages API) and other LITELLM_METADATA_ROUTES, + litellm_parent_otel_span is stored in litellm_params["litellm_metadata"] + instead of litellm_params["metadata"]. _get_span_context() must check + both locations. + + Fixes: https://github.com/BerriAI/litellm/issues/27934 + """ + + def test_span_context_from_metadata(self): + """Parent span is found when stored in litellm_params['metadata'] (OpenAI path).""" + otel = OpenTelemetry() + mock_span = MagicMock() + mock_span.get_span_context.return_value = MagicMock(is_valid=True) + + kwargs = { + "litellm_params": { + "metadata": {"litellm_parent_otel_span": mock_span}, + } + } + + ctx, detected_span = otel._get_span_context(kwargs) + self.assertIsNotNone(ctx) + # Should NOT fall through to "no parent context" path + self.assertIsNone(detected_span) + + def test_span_context_from_litellm_metadata_fallback(self): + """Parent span is found when stored in litellm_params['litellm_metadata'] (Anthropic path).""" + otel = OpenTelemetry() + mock_span = MagicMock() + mock_span.get_span_context.return_value = MagicMock(is_valid=True) + + kwargs = { + "litellm_params": { + "metadata": { + "user_id": "test-user" + }, # Anthropic native metadata, no span + "litellm_metadata": {"litellm_parent_otel_span": mock_span}, + } + } + + ctx, detected_span = otel._get_span_context(kwargs) + self.assertIsNotNone(ctx) + self.assertIsNone(detected_span) + + def test_span_context_metadata_takes_priority(self): + """When both metadata and litellm_metadata have the span, metadata wins.""" + otel = OpenTelemetry() + span_from_metadata = MagicMock(name="span_from_metadata") + span_from_metadata.get_span_context.return_value = MagicMock(is_valid=True) + span_from_litellm_metadata = MagicMock(name="span_from_litellm_metadata") + span_from_litellm_metadata.get_span_context.return_value = MagicMock( + is_valid=True + ) + + kwargs = { + "litellm_params": { + "metadata": {"litellm_parent_otel_span": span_from_metadata}, + "litellm_metadata": { + "litellm_parent_otel_span": span_from_litellm_metadata + }, + } + } + + ctx, detected_span = otel._get_span_context(kwargs) + self.assertIsNotNone(ctx) + self.assertIsNone(detected_span) + # metadata span is found first, so get_span_context on the + # litellm_metadata span should never be called — proving + # metadata takes priority over litellm_metadata. + span_from_litellm_metadata.get_span_context.assert_not_called() + + def test_span_context_no_parent_when_neither_has_span(self): + """When neither metadata nor litellm_metadata has a span, returns (None, None).""" + otel = OpenTelemetry() + + kwargs = { + "litellm_params": { + "metadata": {"user_id": "test-user"}, + "litellm_metadata": {"some_key": "some_value"}, + } + } + + ctx, detected_span = otel._get_span_context(kwargs) + # No parent span in either metadata dict and no active span in test + # context, so both should be None. + self.assertIsNone(ctx) + self.assertIsNone(detected_span) + + +class TestEndProxySpanLitellmMetadataFallback(unittest.TestCase): + """ + Tests for _end_proxy_span_from_kwargs() falling back to litellm_metadata. + + Fixes: https://github.com/BerriAI/litellm/issues/27934 + """ + + def test_end_proxy_span_from_metadata(self): + """Proxy span is found and ended from litellm_params['metadata'].""" + otel = OpenTelemetry() + mock_span = MagicMock() + mock_span.name = "Received Proxy Server Request" + mock_span.is_recording.return_value = True + + kwargs = { + "litellm_params": { + "metadata": {"litellm_parent_otel_span": mock_span}, + } + } + + otel._end_proxy_span_from_kwargs(kwargs, end_time=datetime.now()) + mock_span.end.assert_called_once() + + def test_end_proxy_span_from_litellm_metadata(self): + """Proxy span is found and ended from litellm_params['litellm_metadata'] (fallback).""" + otel = OpenTelemetry() + mock_span = MagicMock() + mock_span.name = "Received Proxy Server Request" + mock_span.is_recording.return_value = True + + kwargs = { + "litellm_params": { + "metadata": {"user_id": "test-user"}, # No span here + "litellm_metadata": {"litellm_parent_otel_span": mock_span}, + } + } + + otel._end_proxy_span_from_kwargs(kwargs, end_time=datetime.now()) + mock_span.end.assert_called_once() class TestOpenTelemetryInferenceIdentityAttributes(unittest.TestCase): """team_metadata, http.route, and both model names (the user-facing model_group alias and the dispatched provider model) must land on the diff --git a/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py b/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py index 129ea237efe..91fd07dcffc 100644 --- a/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py +++ b/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py @@ -22,6 +22,18 @@ from litellm.litellm_core_utils.prompt_templates.factory import ( from litellm.types.llms.openai import ChatCompletionToolMessage +def _get_gemini_function_response_inline_data_parts(result): + assert isinstance(result, list), "expected Gemini parts list" + assert len(result) == 1, "multimodal function responses should stay in one part" + function_response_part = result[0] + assert ( + "inline_data" not in function_response_part + ), "inline_data should be nested under function_response.parts" + function_response = function_response_part["function_response"] + nested_parts = function_response["parts"] + return [part["inline_data"] for part in nested_parts if "inline_data" in part] + + def test_ollama_pt_simple_messages(): """Test basic functionality with simple text messages""" messages = [ @@ -615,8 +627,8 @@ def test_convert_gemini_tool_call_result_with_image_url(): message=message_str_format, last_message_with_tool_calls=last_message_with_tool_calls, ) - # Should have inline_data for the image - assert isinstance(result, list) and any("inline_data" in p for p in result) + inline_parts = _get_gemini_function_response_inline_data_parts(result) + assert len(inline_parts) == 1 # Test with dict image_url format (OpenAI standard) message_dict_format = ChatCompletionToolMessage( @@ -635,7 +647,8 @@ def test_convert_gemini_tool_call_result_with_image_url(): message=message_dict_format, last_message_with_tool_calls=last_message_with_tool_calls, ) - assert isinstance(result2, list) and any("inline_data" in p for p in result2) + inline_parts = _get_gemini_function_response_inline_data_parts(result2) + assert len(inline_parts) == 1 def test_convert_gemini_tool_call_result_with_anthropic_image_block(): @@ -677,11 +690,10 @@ def test_convert_gemini_tool_call_result_with_anthropic_image_block(): message=message, last_message_with_tool_calls=last_message_with_tool_calls, ) - assert isinstance(result, list), "expected a list of parts" - inline_parts = [p for p in result if "inline_data" in p] + inline_parts = _get_gemini_function_response_inline_data_parts(result) assert len(inline_parts) == 1, "expected exactly one inline_data part" - assert inline_parts[0]["inline_data"]["mime_type"] == "image/png" - assert inline_parts[0]["inline_data"]["data"] == tiny_png_b64 + assert inline_parts[0]["mime_type"] == "image/png" + assert inline_parts[0]["data"] == tiny_png_b64 def test_convert_gemini_tool_call_result_with_multiple_anthropic_image_blocks(): @@ -734,12 +746,11 @@ def test_convert_gemini_tool_call_result_with_multiple_anthropic_image_blocks(): message=message, last_message_with_tool_calls=last_message_with_tool_calls, ) - assert isinstance(result, list), "expected a list of parts" - inline_parts = [p for p in result if "inline_data" in p] + inline_parts = _get_gemini_function_response_inline_data_parts(result) assert ( len(inline_parts) == 2 ), f"expected 2 inline_data parts, got {len(inline_parts)}" - mime_types = {p["inline_data"]["mime_type"] for p in inline_parts} + mime_types = {p["mime_type"] for p in inline_parts} assert mime_types == {"image/png", "image/jpeg"} @@ -773,13 +784,12 @@ def test_convert_gemini_tool_call_result_with_data_url_string(): message=message, last_message_with_tool_calls=last_message_with_tool_calls, ) - assert isinstance(result, list), "expected a list of parts" - inline_parts = [p for p in result if "inline_data" in p] + inline_parts = _get_gemini_function_response_inline_data_parts(result) assert ( len(inline_parts) == 1 ), "data-URL image string was not converted to inline_data" - assert inline_parts[0]["inline_data"]["mime_type"] == "image/png" - assert inline_parts[0]["inline_data"]["data"] == tiny_png_b64 + assert inline_parts[0]["mime_type"] == "image/png" + assert inline_parts[0]["data"] == tiny_png_b64 def test_convert_gemini_tool_call_result_with_data_url_extra_params(): @@ -811,12 +821,11 @@ def test_convert_gemini_tool_call_result_with_data_url_extra_params(): message=message, last_message_with_tool_calls=last_message_with_tool_calls, ) - assert isinstance(result, list), "expected a list of parts" - inline_parts = [p for p in result if "inline_data" in p] + inline_parts = _get_gemini_function_response_inline_data_parts(result) assert len(inline_parts) == 1 assert ( - inline_parts[0]["inline_data"]["mime_type"] == "image/png" - ), f"expected clean 'image/png', got '{inline_parts[0]['inline_data']['mime_type']}'" + inline_parts[0]["mime_type"] == "image/png" + ), f"expected clean 'image/png', got '{inline_parts[0]['mime_type']}'" def test_bedrock_tools_unpack_defs(): diff --git a/tests/test_litellm/llms/azure_ai/test_azure_ai_kimi_k26_metadata.py b/tests/test_litellm/llms/azure_ai/test_azure_ai_kimi_k26_metadata.py new file mode 100644 index 00000000000..812b9288ca8 --- /dev/null +++ b/tests/test_litellm/llms/azure_ai/test_azure_ai_kimi_k26_metadata.py @@ -0,0 +1,76 @@ +""" +Test Azure AI Kimi K2.6 model metadata. +""" + +import json +from importlib.resources import files + +import pytest + + +@pytest.fixture(scope="module") +def use_local_model_cost_map(): + monkeypatch = pytest.MonkeyPatch() + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") + + import litellm + from litellm.utils import _invalidate_model_cost_lowercase_map + + original_model_cost = litellm.model_cost + litellm.model_cost = json.loads( + files("litellm") + .joinpath("model_prices_and_context_window_backup.json") + .read_text(encoding="utf-8") + ) + litellm.get_model_info.cache_clear() + _invalidate_model_cost_lowercase_map() + try: + yield litellm + finally: + litellm.model_cost = original_model_cost + litellm.get_model_info.cache_clear() + _invalidate_model_cost_lowercase_map() + monkeypatch.undo() + + +def test_azure_ai_kimi_k26_model_info(use_local_model_cost_map): + model_info = use_local_model_cost_map.get_model_info(model="azure_ai/kimi-k2.6") + + assert model_info["litellm_provider"] == "azure_ai" + assert model_info["mode"] == "chat" + assert model_info["max_input_tokens"] == 262144 + assert model_info["max_output_tokens"] == 262144 + assert model_info["max_tokens"] == 262144 + assert model_info["input_cost_per_token"] == pytest.approx(9.5e-07) + assert model_info["output_cost_per_token"] == pytest.approx(4e-06) + assert model_info["supports_function_calling"] is True + assert model_info["supports_reasoning"] is True + assert model_info["supports_tool_choice"] is True + assert model_info["supports_vision"] is True + + +def test_azure_ai_kimi_k26_raw_model_cost_entry(use_local_model_cost_map): + model_info = use_local_model_cost_map.model_cost["azure_ai/kimi-k2.6"] + + assert model_info["supported_modalities"] == ["text", "image"] + assert model_info["supported_output_modalities"] == ["text"] + assert model_info["supports_function_calling"] is True + assert model_info["supports_reasoning"] is True + assert model_info["supports_tool_choice"] is True + assert model_info["supports_vision"] is True + + +def test_azure_ai_kimi_k26_cost_per_token(use_local_model_cost_map): + from litellm.llms.azure_ai.cost_calculator import cost_per_token + from litellm.types.utils import Usage + + usage = Usage( + prompt_tokens=1_000_000, + completion_tokens=1_000_000, + total_tokens=2_000_000, + ) + + prompt_cost, completion_cost = cost_per_token(model="kimi-k2.6", usage=usage) + + assert prompt_cost == pytest.approx(0.95) + assert completion_cost == pytest.approx(4.0) diff --git a/tests/test_litellm/llms/gemini/image_edit/test_gemini_image_edit_transformation.py b/tests/test_litellm/llms/gemini/image_edit/test_gemini_image_edit_transformation.py index 682df923693..9b57e1991de 100644 --- a/tests/test_litellm/llms/gemini/image_edit/test_gemini_image_edit_transformation.py +++ b/tests/test_litellm/llms/gemini/image_edit/test_gemini_image_edit_transformation.py @@ -7,6 +7,8 @@ from unittest.mock import MagicMock import httpx import pytest +import litellm +from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup from litellm.llms.gemini.image_edit.transformation import GeminiImageEditConfig @@ -19,6 +21,7 @@ class TestGeminiImageEditTransformation: def test_map_openai_params(self) -> None: optional_params: Dict[str, object] = { + "n": 2, "size": "1792x1024", "response_format": "b64_json", "quality": "high", @@ -30,20 +33,77 @@ class TestGeminiImageEditTransformation: drop_params=False, ) - assert mapped["aspectRatio"] == "16:9" + assert mapped["imageConfig"] == {"aspectRatio": "16:9"} + assert mapped["sampleCount"] == 2 assert "response_format" not in mapped assert "quality" not in mapped + def test_map_openai_params_with_image_size_for_gemini_3(self) -> None: + optional_params: Dict[str, object] = { + "size": "768x1376", + } + + mapped = self.config.map_openai_params( + image_edit_optional_params=optional_params, # type: ignore[arg-type] + model="gemini-3-pro-image-preview", + drop_params=False, + ) + + assert mapped["imageConfig"] == {"aspectRatio": "9:16", "imageSize": "1K"} + + def test_map_openai_params_forwards_image_config_as_is(self) -> None: + optional_params: Dict[str, object] = { + "size": "1024x1024", + "imageConfig": {"aspectRatio": "16:9", "imageSize": "512px"}, + } + + mapped = self.config.map_openai_params( + image_edit_optional_params=optional_params, # type: ignore[arg-type] + model="gemini-3-pro-image-preview", + drop_params=False, + ) + + assert mapped["imageConfig"] == {"aspectRatio": "16:9", "imageSize": "512px"} + + def test_map_openai_params_parses_form_image_config_json(self) -> None: + optional_params: Dict[str, object] = { + "imageConfig": '{"aspectRatio":"16:9","imageSize":"1K"}', + } + + mapped = self.config.map_openai_params( + image_edit_optional_params=optional_params, # type: ignore[arg-type] + model="gemini-3-pro-image-preview", + drop_params=False, + ) + + assert mapped["imageConfig"] == {"aspectRatio": "16:9", "imageSize": "1K"} + + def test_map_openai_params_rejects_malformed_form_image_config_json( + self, + ) -> None: + optional_params: Dict[str, object] = { + "imageConfig": "{bad", + } + + with pytest.raises(litellm.UnsupportedParamsError) as exc_info: + self.config.map_openai_params( + image_edit_optional_params=optional_params, # type: ignore[arg-type] + model="gemini-3-pro-image-preview", + drop_params=False, + ) + + assert "`imageConfig` must be valid JSON" in str(exc_info.value) + def test_transform_image_edit_request(self) -> None: image_bytes = b"fake_image_data" image = BytesIO(image_bytes) optional_params = { "sampleCount": 2, - "aspectRatio": "16:9", + "imageConfig": {"aspectRatio": "16:9", "imageSize": "2K"}, } request_body, files = self.config.transform_image_edit_request( - model=self.model, + model="gemini-3-pro-image-preview", prompt=self.prompt, image=[image], # Gemini pipeline passes list of images image_edit_optional_request_params=optional_params, @@ -61,7 +121,28 @@ class TestGeminiImageEditTransformation: assert base64.b64decode(inline_data["data"]) == image_bytes generation_config = request_body["generationConfig"] + assert generation_config["candidateCount"] == 2 assert generation_config["imageConfig"]["aspectRatio"] == "16:9" + assert generation_config["imageConfig"]["imageSize"] == "2K" + + def test_transform_image_edit_request_omits_image_size_for_gemini_25(self) -> None: + image = BytesIO(b"fake_image_data") + optional_params = { + "imageConfig": {"aspectRatio": "16:9", "imageSize": "2K"}, + } + + request_body, _ = self.config.transform_image_edit_request( + model=self.model, + prompt=self.prompt, + image=[image], + image_edit_optional_request_params=optional_params, + litellm_params=MagicMock(), + headers={}, + ) + + assert request_body["generationConfig"]["imageConfig"] == { + "aspectRatio": "16:9" + } def test_transform_image_edit_request_multiple_images(self) -> None: image_one = BytesIO(b"image_one") @@ -115,7 +196,16 @@ class TestGeminiImageEditTransformation: ] } }, - ] + ], + "usageMetadata": { + "promptTokenCount": 35, + "candidatesTokenCount": 1716, + "totalTokenCount": 1751, + "promptTokensDetails": [ + {"modality": "TEXT", "tokenCount": 30}, + {"modality": "IMAGE", "tokenCount": 5}, + ], + }, } mock_response = MagicMock(spec=httpx.Response) @@ -138,6 +228,19 @@ class TestGeminiImageEditTransformation: "utf-8" ) + usage = image_response.model_dump()["usage"] + assert usage["input_tokens"] == 35 + assert usage["output_tokens"] == 1716 + assert usage["prompt_tokens"] == 35 + assert usage["completion_tokens"] == 1716 + assert usage["prompt_tokens_details"]["image_tokens"] == 5 + assert usage["completion_tokens_details"]["image_tokens"] == 1716 + + logging_usage = StandardLoggingPayloadSetup.get_usage_as_dict( + response_obj=image_response.model_dump() + ) + assert logging_usage["completion_tokens_details"]["image_tokens"] == 1716 + def test_transform_image_edit_request_without_image_raises(self) -> None: optional_params = {} diff --git a/tests/test_litellm/llms/gemini/test_cost_calculator.py b/tests/test_litellm/llms/gemini/test_cost_calculator.py index 9bb83aa7cff..6d51bcd2c88 100644 --- a/tests/test_litellm/llms/gemini/test_cost_calculator.py +++ b/tests/test_litellm/llms/gemini/test_cost_calculator.py @@ -1,7 +1,23 @@ +import os + import pytest +import litellm from litellm.llms.gemini.cost_calculator import cost_per_web_search_request -from litellm.types.utils import PromptTokensDetailsWrapper, Usage +from litellm.llms.gemini.image_edit.cost_calculator import ( + cost_calculator as gemini_image_edit_cost_calculator, +) +from litellm.llms.gemini.image_generation.cost_calculator import ( + cost_calculator as gemini_image_generation_cost_calculator, +) +from litellm.types.utils import ( + ImageObject, + ImageResponse, + ImageUsage, + ImageUsageInputTokensDetails, + PromptTokensDetailsWrapper, + Usage, +) def _make_usage(web_search_requests: int) -> Usage: @@ -63,3 +79,171 @@ def test_no_usage_details(): usage = Usage(prompt_tokens=100, completion_tokens=50, total_tokens=150) cost = cost_per_web_search_request(usage=usage, model_info=model_info) assert cost == 0.0 + + +def test_gemini_image_edit_cost_prefers_token_usage_metadata(): + os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" + litellm.model_cost = litellm.get_model_cost_map(url="") + model = "gemini/gemini-3-pro-image-preview" + model_info = litellm.get_model_info(model=model, custom_llm_provider="gemini") + + input_text_tokens = 20 + input_image_tokens = 1120 + output_image_tokens = 1120 + prompt_tokens = input_text_tokens + input_image_tokens + image_response = ImageResponse( + data=[ImageObject(b64_json="img1"), ImageObject(b64_json="img2")], + usage=ImageUsage( + input_tokens=prompt_tokens, + input_tokens_details=ImageUsageInputTokensDetails( + text_tokens=input_text_tokens, + image_tokens=input_image_tokens, + ), + output_tokens=output_image_tokens, + total_tokens=prompt_tokens + output_image_tokens, + ), + ) + + cost = gemini_image_edit_cost_calculator( + model=model, + image_response=image_response, + ) + + expected_cost = ( + prompt_tokens * model_info["input_cost_per_token"] + + output_image_tokens * model_info["output_cost_per_image_token"] + ) + flat_image_cost = ( + len(image_response.data or []) * model_info["output_cost_per_image"] + ) + assert round(cost, 10) == round(expected_cost, 10) + assert cost != flat_image_cost + + +def test_gemini_image_edit_cost_uses_output_token_details(): + os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" + litellm.model_cost = litellm.get_model_cost_map(url="") + model = "gemini/gemini-3-pro-image-preview" + model_info = litellm.get_model_info(model=model, custom_llm_provider="gemini") + + input_text_tokens = 20 + output_text_tokens = 213 + output_image_tokens = 1120 + output_tokens = output_text_tokens + output_image_tokens + image_response = ImageResponse( + data=[ImageObject(b64_json="img1")], + usage=ImageUsage( + input_tokens=input_text_tokens, + input_tokens_details=ImageUsageInputTokensDetails( + text_tokens=input_text_tokens, + image_tokens=0, + ), + output_tokens=output_tokens, + total_tokens=input_text_tokens + output_tokens, + prompt_tokens=input_text_tokens, + completion_tokens=output_tokens, + prompt_tokens_details={ + "text_tokens": input_text_tokens, + "image_tokens": 0, + }, + completion_tokens_details={ + "text_tokens": output_text_tokens, + "image_tokens": output_image_tokens, + }, + output_tokens_details={ + "text_tokens": output_text_tokens, + "image_tokens": output_image_tokens, + }, + ), + ) + + cost = gemini_image_edit_cost_calculator( + model=model, + image_response=image_response, + ) + + expected_cost = ( + input_text_tokens * model_info["input_cost_per_token"] + + output_text_tokens * model_info["output_cost_per_token"] + + output_image_tokens * model_info["output_cost_per_image_token"] + ) + all_output_as_image_cost = ( + input_text_tokens * model_info["input_cost_per_token"] + + (output_text_tokens + output_image_tokens) + * model_info["output_cost_per_image_token"] + ) + assert round(cost, 10) == round(expected_cost, 10) + assert cost != all_output_as_image_cost + + +def test_gemini_image_generation_cost_uses_output_token_details(): + os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" + litellm.model_cost = litellm.get_model_cost_map(url="") + model = "gemini/gemini-3-pro-image-preview" + model_info = litellm.get_model_info(model=model, custom_llm_provider="gemini") + + input_text_tokens = 20 + output_text_tokens = 213 + output_image_tokens = 1120 + output_tokens = output_text_tokens + output_image_tokens + image_response = ImageResponse( + data=[ImageObject(b64_json="img1")], + usage=ImageUsage( + input_tokens=input_text_tokens, + input_tokens_details=ImageUsageInputTokensDetails( + text_tokens=input_text_tokens, + image_tokens=0, + ), + output_tokens=output_tokens, + total_tokens=input_text_tokens + output_tokens, + prompt_tokens=input_text_tokens, + completion_tokens=output_tokens, + prompt_tokens_details={ + "text_tokens": input_text_tokens, + "image_tokens": 0, + }, + completion_tokens_details={ + "text_tokens": output_text_tokens, + "image_tokens": output_image_tokens, + }, + output_tokens_details={ + "text_tokens": output_text_tokens, + "image_tokens": output_image_tokens, + }, + ), + ) + + cost = gemini_image_generation_cost_calculator( + model=model, + image_response=image_response, + ) + + expected_cost = ( + input_text_tokens * model_info["input_cost_per_token"] + + output_text_tokens * model_info["output_cost_per_token"] + + output_image_tokens * model_info["output_cost_per_image_token"] + ) + all_output_as_image_cost = ( + input_text_tokens * model_info["input_cost_per_token"] + + (output_text_tokens + output_image_tokens) + * model_info["output_cost_per_image_token"] + ) + assert round(cost, 10) == round(expected_cost, 10) + assert cost != all_output_as_image_cost + + +def test_gemini_image_edit_cost_falls_back_to_flat_image_pricing(): + os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" + litellm.model_cost = litellm.get_model_cost_map(url="") + model = "gemini/gemini-3-pro-image-preview" + model_info = litellm.get_model_info(model=model, custom_llm_provider="gemini") + image_response = ImageResponse( + data=[ImageObject(b64_json="img1"), ImageObject(b64_json="img2")] + ) + + cost = gemini_image_edit_cost_calculator( + model=model, + image_response=image_response, + ) + + assert cost == len(image_response.data or []) * model_info["output_cost_per_image"] diff --git a/tests/test_litellm/llms/gemini/test_gemini_image_generation_transformation.py b/tests/test_litellm/llms/gemini/test_gemini_image_generation_transformation.py new file mode 100644 index 00000000000..4610d1b99bf --- /dev/null +++ b/tests/test_litellm/llms/gemini/test_gemini_image_generation_transformation.py @@ -0,0 +1,240 @@ +import httpx + +from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup +from litellm.llms.gemini.image_generation.transformation import GoogleImageGenConfig +from litellm.types.utils import ImageResponse + + +def test_gemini_image_generation_request_uses_shared_generation_config(): + config = GoogleImageGenConfig() + + request = config.transform_image_generation_request( + model="gemini-3.1-flash-image-preview", + prompt="Generate a simple app icon", + optional_params={ + "sampleCount": 2, + "imageConfig": {"aspectRatio": "16:9", "imageSize": "2K"}, + }, + litellm_params={}, + headers={}, + ) + + assert request["contents"][0]["parts"] == [{"text": "Generate a simple app icon"}] + assert request["generationConfig"] == { + "response_modalities": ["IMAGE", "TEXT"], + "imageConfig": {"aspectRatio": "16:9", "imageSize": "2K"}, + "candidateCount": 2, + } + + +def test_gemini_image_generation_map_openai_params_maps_n_size_and_image_config(): + config = GoogleImageGenConfig() + + mapped = config.map_openai_params( + non_default_params={ + "n": 2, + "size": "768x1376", + "imageConfig": {"aspectRatio": "1:1", "imageSize": "512"}, + }, + optional_params={}, + model="gemini-3.1-flash-image-preview", + drop_params=False, + ) + + assert mapped == { + "sampleCount": 2, + "imageConfig": {"aspectRatio": "1:1", "imageSize": "512"}, + } + + +def test_imagen_generation_with_provider_prefix_uses_imagen_params_and_response(): + config = GoogleImageGenConfig() + + mapped = config.map_openai_params( + non_default_params={ + "n": 1, + "size": "1024x1024", + }, + optional_params={}, + model="gemini/imagen-4.0-generate-001", + drop_params=False, + ) + assert mapped == { + "sampleCount": 1, + "aspectRatio": "1:1", + "imageSize": "1K", + } + + request = config.transform_image_generation_request( + model="gemini/imagen-4.0-generate-001", + prompt="Generate a simple app icon", + optional_params=mapped, + litellm_params={}, + headers={}, + ) + assert request == { + "instances": [{"prompt": "Generate a simple app icon"}], + "parameters": { + "sampleCount": 1, + "aspectRatio": "1:1", + "imageSize": "1K", + }, + } + + result = config.transform_image_generation_response( + model="gemini/imagen-4.0-generate-001", + raw_response=httpx.Response( + status_code=200, + json={ + "predictions": [ + { + "bytesBase64Encoded": "fake-imagen-image", + } + ] + }, + ), + model_response=ImageResponse(data=[]), + logging_obj=None, + request_data={}, + optional_params={}, + litellm_params={}, + encoding=None, + ) + assert result.data is not None + assert result.data[0].b64_json == "fake-imagen-image" + + +def test_imagen_generation_forwards_mapped_openai_size_image_size(): + config = GoogleImageGenConfig() + + mapped = config.map_openai_params( + non_default_params={ + "size": "512x512", + }, + optional_params={}, + model="gemini/imagen-4.0-generate-001", + drop_params=False, + ) + assert mapped == {"aspectRatio": "1:1", "imageSize": "512"} + + request = config.transform_image_generation_request( + model="gemini/imagen-4.0-generate-001", + prompt="Generate a simple app icon", + optional_params=mapped, + litellm_params={}, + headers={}, + ) + + assert request == { + "instances": [{"prompt": "Generate a simple app icon"}], + "parameters": {"aspectRatio": "1:1", "imageSize": "512"}, + } + + +def test_gemini_image_generation_usage_includes_chat_token_details(): + config = GoogleImageGenConfig() + raw_response = httpx.Response( + status_code=200, + json={ + "candidates": [ + { + "content": { + "parts": [ + { + "inlineData": { + "mimeType": "image/png", + "data": "fake-image", + } + } + ] + } + } + ], + "usageMetadata": { + "promptTokenCount": 35, + "candidatesTokenCount": 1716, + "totalTokenCount": 1751, + "promptTokensDetails": [ + {"modality": "TEXT", "tokenCount": 30}, + {"modality": "IMAGE", "tokenCount": 5}, + ], + "candidatesTokensDetails": [ + {"modality": "TEXT", "tokenCount": 213}, + {"modality": "IMAGE", "tokenCount": 1120}, + ], + }, + }, + ) + + result = config.transform_image_generation_response( + model="gemini-3.1-flash-image-preview", + raw_response=raw_response, + model_response=ImageResponse(data=[]), + logging_obj=None, + request_data={}, + optional_params={}, + litellm_params={}, + encoding=None, + ) + + usage = result.model_dump()["usage"] + + assert usage["input_tokens"] == 35 + assert usage["output_tokens"] == 1716 + assert usage["prompt_tokens"] == 35 + assert usage["completion_tokens"] == 1716 + assert usage["prompt_tokens_details"]["image_tokens"] == 5 + assert usage["completion_tokens_details"]["text_tokens"] == 596 + assert usage["completion_tokens_details"]["image_tokens"] == 1120 + assert usage["output_tokens_details"]["text_tokens"] == 596 + assert usage["output_tokens_details"]["image_tokens"] == 1120 + + logging_usage = StandardLoggingPayloadSetup.get_usage_as_dict( + response_obj=result.model_dump() + ) + assert logging_usage["completion_tokens_details"]["text_tokens"] == 596 + assert logging_usage["completion_tokens_details"]["image_tokens"] == 1120 + + +def test_gemini_image_generation_usage_without_output_details_treats_output_as_image(): + config = GoogleImageGenConfig() + raw_response = httpx.Response( + status_code=200, + json={ + "candidates": [ + { + "content": { + "parts": [ + { + "inlineData": { + "mimeType": "image/png", + "data": "fake-image", + } + } + ] + } + } + ], + "usageMetadata": { + "promptTokenCount": 35, + "candidatesTokenCount": 1716, + "totalTokenCount": 1751, + "promptTokensDetails": [{"modality": "TEXT", "tokenCount": 35}], + }, + }, + ) + + result = config.transform_image_generation_response( + model="gemini-3.1-flash-image-preview", + raw_response=raw_response, + model_response=ImageResponse(data=[]), + logging_obj=None, + request_data={}, + optional_params={}, + litellm_params={}, + encoding=None, + ) + + usage = result.model_dump()["usage"] + assert usage["completion_tokens_details"]["text_tokens"] == 0 + assert usage["completion_tokens_details"]["image_tokens"] == 1716 diff --git a/tests/test_litellm/llms/inception/__init__.py b/tests/test_litellm/llms/inception/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/test_litellm/llms/inception/test_inception_chat_transformation.py b/tests/test_litellm/llms/inception/test_inception_chat_transformation.py new file mode 100644 index 00000000000..0750fb9e405 --- /dev/null +++ b/tests/test_litellm/llms/inception/test_inception_chat_transformation.py @@ -0,0 +1,326 @@ +""" +Tests for Inception (Mercury) chat provider integration +""" + +import json +import os +from unittest import mock + +import httpx + +import litellm +from litellm.llms.inception.chat.transformation import InceptionChatConfig + + +def test_inception_config_initialization(): + config = InceptionChatConfig() + assert config.custom_llm_provider == "inception" + + +def test_inception_chat_supports_diffusion_params(): + """The chat config must expose Inception's diffusion-LLM request controls""" + params = InceptionChatConfig().get_supported_openai_params("mercury-2") + for p in ( + "reasoning_effort", + "reasoning_summary", + "reasoning_summary_wait", + "diffusing", + "realtime", + "tools", + "tool_choice", + "response_format", + ): + assert p in params, f"{p} should be a supported chat param" + + +def test_inception_chat_sends_diffusion_params_in_body(): + """reasoning_effort (incl. `instant`) and the diffusion flags reach the request body""" + + captured = {} + + def fake_send(self, request, **kwargs): + captured["body"] = json.loads(request.content.decode()) + return httpx.Response( + status_code=200, + request=request, + headers={"content-type": "application/json"}, + content=json.dumps( + { + "id": "c-1", + "object": "chat.completion", + "created": 1, + "model": "mercury-2", + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "hi"}, + "finish_reason": "stop", + } + ], + "usage": { + "prompt_tokens": 5, + "completion_tokens": 1, + "total_tokens": 6, + }, + } + ).encode(), + ) + + with mock.patch("httpx.Client.send", new=fake_send): + litellm.completion( + model="inception/mercury-2", + messages=[{"role": "user", "content": "hi"}], + api_key="sk-x", + reasoning_effort="instant", + reasoning_summary=True, + reasoning_summary_wait=True, + diffusing=True, + realtime=True, + max_completion_tokens=128, + ) + + body = captured["body"] + assert body["reasoning_effort"] == "instant" + assert body["reasoning_summary"] is True + assert body["reasoning_summary_wait"] is True + assert body["diffusing"] is True + assert body["realtime"] is True + assert body["max_tokens"] == 128 # max_completion_tokens mapped to max_tokens + + +def test_inception_chat_response_surfaces_reasoning_and_usage(): + """reasoning_summary / warning survive, and reasoning_tokens maps to usage details""" + + def fake_send(self, request, **kwargs): + return httpx.Response( + status_code=200, + request=request, + headers={"content-type": "application/json"}, + content=json.dumps( + { + "id": "c-1", + "object": "chat.completion", + "created": 1, + "model": "mercury-2", + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "answer"}, + "finish_reason": "stop", + } + ], + "usage": { + "prompt_tokens": 5, + "completion_tokens": 2, + "total_tokens": 7, + "reasoning_tokens": 4, + "cached_input_tokens": 3, + }, + "reasoning_summary": { + "content": "step by step", + "status": "complete", + }, + "warning": "heads up", + } + ).encode(), + ) + + with mock.patch("httpx.Client.send", new=fake_send): + r = litellm.completion( + model="inception/mercury-2", + messages=[{"role": "user", "content": "hi"}], + api_key="sk-x", + ) + + assert r.reasoning_summary == {"content": "step by step", "status": "complete"} + assert r.warning == "heads up" + assert r.usage.completion_tokens_details.reasoning_tokens == 4 + assert r.usage.model_extra.get("cached_input_tokens") == 3 + + +def test_inception_get_openai_compatible_provider_info(): + config = InceptionChatConfig() + + with mock.patch.dict(os.environ, {}, clear=True): + with mock.patch.object(litellm, "inception_key", None): + api_base, api_key = config._get_openai_compatible_provider_info(None, None) + assert api_base == "https://api.inceptionlabs.ai/v1" + assert api_key is None + + with mock.patch.dict( + os.environ, + { + "INCEPTION_API_KEY": "test-key", + "INCEPTION_API_BASE": "https://custom.inceptionlabs.ai/v1", + }, + ): + api_base, api_key = config._get_openai_compatible_provider_info(None, None) + assert api_base == "https://custom.inceptionlabs.ai/v1" + assert api_key == "test-key" + + with mock.patch.dict( + os.environ, + { + "INCEPTION_API_KEY": "env-key", + "INCEPTION_API_BASE": "https://env.inceptionlabs.ai/v1", + }, + ): + api_base, api_key = config._get_openai_compatible_provider_info( + "https://param.inceptionlabs.ai/v1", "param-key" + ) + assert api_base == "https://param.inceptionlabs.ai/v1" + assert api_key == "param-key" + + +def test_inception_key_module_attr_fallback(): + """litellm.inception_key is used when no param/env key is provided""" + config = InceptionChatConfig() + with mock.patch.dict(os.environ, {}, clear=True): + with mock.patch.object(litellm, "inception_key", "module-attr-key"): + _, api_key = config._get_openai_compatible_provider_info(None, None) + assert api_key == "module-attr-key" + + +def test_inception_does_not_leak_key_to_caller_api_base(): + """ + The server-managed Inception key must not be forwarded to a caller-supplied + api_base. It is only resolved for the default/server base, or when the + caller also supplies their own key. + """ + config = InceptionChatConfig() + with mock.patch.dict( + os.environ, {"INCEPTION_API_KEY": "server-secret"}, clear=True + ): + with mock.patch.object(litellm, "inception_key", "module-secret"): + # caller overrides api_base without a key -> server key withheld + api_base, api_key = config._get_openai_compatible_provider_info( + "https://attacker.example/v1", None + ) + assert api_base == "https://attacker.example/v1" + assert api_key is None + + # caller overrides api_base AND supplies their own key -> used as-is + _, api_key = config._get_openai_compatible_provider_info( + "https://attacker.example/v1", "caller-key" + ) + assert api_key == "caller-key" + + # default/server base -> server-managed key resolved + _, api_key = config._get_openai_compatible_provider_info(None, None) + assert api_key == "module-secret" + + +def test_get_llm_provider_inception(): + from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider + + model, provider, _, _ = get_llm_provider("inception/mercury-2") + assert model == "mercury-2" + assert provider == "inception" + + model, provider, _, api_base = get_llm_provider( + "mercury-2", api_base="https://api.inceptionlabs.ai/v1" + ) + assert model == "mercury-2" + assert provider == "inception" + assert api_base == "https://api.inceptionlabs.ai/v1" + + +def test_inception_in_provider_lists(): + assert "inception" in litellm.openai_compatible_providers + assert "inception" in litellm.provider_list + assert "https://api.inceptionlabs.ai/v1" in litellm.openai_compatible_endpoints + + +def test_inception_model_configuration(): + from litellm import get_model_info + + os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" + litellm.model_cost = litellm.get_model_cost_map(url="") + litellm.inception_models = set() + litellm.add_known_models() + + info = get_model_info("inception/mercury-2") + assert info.get("litellm_provider") == "inception" + assert info.get("mode") == "chat" + assert info.get("max_input_tokens") == 128000 + assert info.get("input_cost_per_token") == 2.5e-07 + assert info.get("output_cost_per_token") == 7.5e-07 + assert info.get("cache_read_input_token_cost") == 2.5e-08 + assert info.get("supports_function_calling") is True + assert info.get("supports_tool_choice") is True + assert info.get("supports_response_schema") is True + + +def test_inception_model_list_populated(): + os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" + litellm.model_cost = litellm.get_model_cost_map(url="") + litellm.inception_models = set() + litellm.add_known_models() + + assert "inception/mercury-2" in litellm.inception_models + for model in litellm.inception_models: + assert model.startswith("inception/") + + +def test_inception_completion_targets_inception_endpoint(): + """ + End-to-end: a completion routed through the inception provider must hit + Inception's base URL and path, send a Bearer token, strip the + `inception/` prefix from the model name, and forward tool_choice. + """ + + captured = {} + + def fake_send(self, request, **kwargs): + captured["url"] = str(request.url) + captured["auth"] = request.headers.get("authorization") + captured["body"] = json.loads(request.content.decode()) + return httpx.Response( + status_code=200, + request=request, + headers={"content-type": "application/json"}, + content=json.dumps( + { + "id": "cmpl-1", + "object": "chat.completion", + "created": 1, + "model": "mercury-2", + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "hi"}, + "finish_reason": "stop", + } + ], + "usage": { + "prompt_tokens": 5, + "completion_tokens": 1, + "total_tokens": 6, + }, + } + ).encode(), + ) + + tools = [ + { + "type": "function", + "function": { + "name": "f", + "parameters": {"type": "object", "properties": {}}, + }, + } + ] + with mock.patch("httpx.Client.send", new=fake_send): + response = litellm.completion( + model="inception/mercury-2", + messages=[{"role": "user", "content": "hello"}], + api_key="sk-test-fake-123", + tools=tools, + tool_choice="auto", + ) + + assert captured["url"] == "https://api.inceptionlabs.ai/v1/chat/completions" + assert captured["auth"] == "Bearer sk-test-fake-123" + assert captured["body"]["model"] == "mercury-2" + assert captured["body"]["tool_choice"] == "auto" + assert response.choices[0].message.content == "hi" diff --git a/tests/test_litellm/llms/inception/test_inception_completion_transformation.py b/tests/test_litellm/llms/inception/test_inception_completion_transformation.py new file mode 100644 index 00000000000..9b7c8dd3742 --- /dev/null +++ b/tests/test_litellm/llms/inception/test_inception_completion_transformation.py @@ -0,0 +1,300 @@ +""" +Tests for Inception (Mercury) fill-in-the-middle (FIM) provider integration +""" + +import json +import os +from unittest import mock + +import httpx +import pytest + +import litellm +from litellm.llms.inception.completion.transformation import ( + InceptionTextCompletionConfig, +) + + +def _fim_response_bytes(): + return json.dumps( + { + "id": "fim-1", + "object": "text_completion", + "created": 1, + "model": "mercury-edit-2", + "choices": [ + {"text": "a + b", "index": 0, "finish_reason": "stop", "logprobs": None} + ], + "usage": {"prompt_tokens": 5, "completion_tokens": 3, "total_tokens": 8}, + } + ).encode() + + +def test_inception_fim_supports_suffix_param(): + """The FIM config must keep `suffix` (otherwise FIM requests lose context)""" + config = InceptionTextCompletionConfig() + assert "suffix" in config.get_supported_openai_params("mercury-edit-2") + + mapped = config.map_openai_params( + non_default_params={"suffix": "\n return x", "max_completion_tokens": 50}, + optional_params={}, + model="mercury-edit-2", + drop_params=False, + ) + assert mapped["suffix"] == "\n return x" + assert mapped["max_tokens"] == 50 + + +def test_inception_fim_supported_params_match_schema(): + """FIM exposes the OpenAI subset of Inception's FIMCompletionRequest only""" + params = InceptionTextCompletionConfig().get_supported_openai_params( + "mercury-edit-2" + ) + for p in ("suffix", "top_p", "frequency_penalty", "presence_penalty", "stop"): + assert p in params + # Chat-only sampling controls are not part of Inception's FIM schema + for p in ("temperature", "seed", "logprobs", "n", "user"): + assert p not in params + + +def test_text_completion_inception_in_provider_lists(): + from litellm.types.utils import LlmProviders + + assert LlmProviders.TEXT_COMPLETION_INCEPTION == "text-completion-inception" + assert "text-completion-inception" in litellm.provider_list + + +def test_inception_get_supported_openai_params_dispatch(): + """litellm.get_supported_openai_params routes the FIM provider to our config""" + params = litellm.get_supported_openai_params( + model="mercury-edit-2", custom_llm_provider="text-completion-inception" + ) + assert "suffix" in params + assert "temperature" not in params + + +@pytest.mark.parametrize("provider", ["inception", "text-completion-inception"]) +def test_inception_validate_environment(provider): + model = ( + "inception/mercury-2" + if provider == "inception" + else "text-completion-inception/mercury-edit-2" + ) + + with mock.patch.dict(os.environ, {}, clear=True): + result = litellm.validate_environment(model) + assert result["keys_in_environment"] is False + assert "INCEPTION_API_KEY" in result["missing_keys"] + + with mock.patch.dict(os.environ, {"INCEPTION_API_KEY": "sk-x"}, clear=True): + result = litellm.validate_environment(model) + assert result["keys_in_environment"] is True + + +def test_inception_completion_endpoint_returns_chat_object(): + """ + Calling chat `completion()` with the FIM provider converts the text + completion result into a chat-shaped ModelResponse. + """ + + def fake_send(self, request, **kwargs): + return httpx.Response( + status_code=200, + request=request, + headers={"content-type": "application/json"}, + content=_fim_response_bytes(), + ) + + with mock.patch("httpx.Client.send", new=fake_send): + r = litellm.completion( + model="text-completion-inception/mercury-edit-2", + messages=[{"role": "user", "content": "def add(a, b): return "}], + api_key="sk-x", + ) + + assert r.choices[0].message.content == "a + b" + + +@pytest.mark.asyncio +async def test_inception_fim_async(): + """async FIM path (acompletion) hits Inception's /v1/fim/completions""" + + captured = {} + + async def fake_asend(self, request, **kwargs): + captured["url"] = str(request.url) + return httpx.Response( + status_code=200, + request=request, + headers={"content-type": "application/json"}, + content=_fim_response_bytes(), + ) + + with mock.patch("httpx.AsyncClient.send", new=fake_asend): + r = await litellm.atext_completion( + model="text-completion-inception/mercury-edit-2", + prompt="def add(a, b): return ", + suffix="\n", + api_key="sk-x", + max_tokens=10, + ) + + assert captured["url"] == "https://api.inceptionlabs.ai/v1/fim/completions" + assert r.choices[0].text == "a + b" + + +def test_inception_fim_model_configuration(): + from litellm import get_model_info + + os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" + litellm.model_cost = litellm.get_model_cost_map(url="") + litellm.text_completion_inception_models = set() + litellm.add_known_models() + + assert ( + "text-completion-inception/mercury-edit-2" + in litellm.text_completion_inception_models + ) + info = get_model_info("text-completion-inception/mercury-edit-2") + assert info.get("litellm_provider") == "text-completion-inception" + assert info.get("mode") == "completion" + assert info.get("max_input_tokens") == 32000 + + +def test_inception_fim_targets_fim_endpoint(): + """ + End-to-end: a FIM request must hit `/v1/fim/completions` (NOT + `/v1/completions`), carry the `suffix`, and parse the standard `text` field. + """ + + captured = {} + + def fake_send(self, request, **kwargs): + captured["url"] = str(request.url) + captured["auth"] = request.headers.get("authorization") + captured["body"] = json.loads(request.content.decode()) + return httpx.Response( + status_code=200, + request=request, + headers={"content-type": "application/json"}, + content=json.dumps( + { + "id": "fim-1", + "object": "text_completion", + "created": 1, + "model": "mercury-edit-2", + "choices": [ + { + "text": "a + b", + "index": 0, + "finish_reason": "stop", + "logprobs": None, + } + ], + "usage": { + "prompt_tokens": 5, + "completion_tokens": 3, + "total_tokens": 8, + }, + } + ).encode(), + ) + + with mock.patch("httpx.Client.send", new=fake_send): + response = litellm.text_completion( + model="text-completion-inception/mercury-edit-2", + prompt="def add(a, b):\n return ", + suffix="\n", + api_key="sk-fim-fake", + max_tokens=20, + ) + + assert captured["url"] == "https://api.inceptionlabs.ai/v1/fim/completions" + assert captured["auth"] == "Bearer sk-fim-fake" + assert captured["body"]["model"] == "mercury-edit-2" + assert captured["body"]["suffix"] == "\n" + assert "prompt" in captured["body"] + assert response.choices[0].text == "a + b" + + +def test_inception_fim_does_not_leak_global_api_key(): + """ + Regression: the global litellm.api_key (commonly an OpenAI key) must not be + forwarded to Inception. Only an Inception-specific key (param, + litellm.inception_key, or INCEPTION_API_KEY) may be sent to the Inception base. + """ + + captured = {} + + def fake_send(self, request, **kwargs): + captured["auth"] = request.headers.get("authorization") + return httpx.Response( + status_code=200, + request=request, + headers={"content-type": "application/json"}, + content=_fim_response_bytes(), + ) + + with mock.patch.dict( + os.environ, {"INCEPTION_API_KEY": "sk-inception-correct"}, clear=True + ): + with mock.patch.object(litellm, "inception_key", None): + with mock.patch.object(litellm, "api_key", "sk-global-should-not-leak"): + with mock.patch("httpx.Client.send", new=fake_send): + litellm.text_completion( + model="text-completion-inception/mercury-edit-2", + prompt="def add(a, b): return ", + max_tokens=10, + ) + + assert captured["auth"] == "Bearer sk-inception-correct" + + +def test_inception_fim_extra_body_forwards_vllm_params(): + """top_k / repetition_penalty are reachable via extra_body (not OpenAI params)""" + + captured = {} + + def fake_send(self, request, **kwargs): + captured["body"] = json.loads(request.content.decode()) + return httpx.Response( + status_code=200, + request=request, + headers={"content-type": "application/json"}, + content=json.dumps( + { + "id": "f-1", + "object": "text_completion", + "created": 1, + "model": "mercury-edit-2", + "choices": [ + { + "text": "x", + "index": 0, + "finish_reason": "stop", + "logprobs": None, + } + ], + "usage": { + "prompt_tokens": 2, + "completion_tokens": 1, + "total_tokens": 3, + }, + } + ).encode(), + ) + + with mock.patch("httpx.Client.send", new=fake_send): + litellm.text_completion( + model="text-completion-inception/mercury-edit-2", + prompt="def f(", + suffix=")", + api_key="sk-x", + top_p=0.9, + extra_body={"top_k": 40, "repetition_penalty": 1.1}, + ) + + body = captured["body"] + assert body["top_p"] == 0.9 + assert body["top_k"] == 40 + assert body["repetition_penalty"] == 1.1 diff --git a/tests/test_litellm/llms/vertex_ai/gemini/test_tool_call_followed_by_text_assistant.py b/tests/test_litellm/llms/vertex_ai/gemini/test_tool_call_followed_by_text_assistant.py new file mode 100644 index 00000000000..bbd12e25f43 --- /dev/null +++ b/tests/test_litellm/llms/vertex_ai/gemini/test_tool_call_followed_by_text_assistant.py @@ -0,0 +1,57 @@ +""" +Regression test for tool-call / tool-result matching in the Gemini message converter. + +When an assistant message that contains tool_calls is followed by a *second* assistant +message that has no tool_calls (e.g. the model emits a short narration turn after the +tool call but before the tool result), the converter used to overwrite its +`last_message_with_tool_calls` reference with the text-only assistant message. The +subsequent tool result could then no longer be matched to its tool call, and conversion +failed with: + + Exception: Missing corresponding tool call for tool response message. + +This happens for any OpenAI-style history with that shape, independent of provider/model. +""" + +import pytest + +from litellm.llms.vertex_ai.gemini.transformation import ( + _gemini_convert_messages_with_history, +) + + +def _messages_with_text_assistant_between_tool_call_and_result(): + return [ + {"role": "user", "content": "list the files"}, + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_abc123", + "type": "function", + "function": {"name": "shell", "arguments": '{"command": ["ls"]}'}, + } + ], + }, + # text-only assistant message in between (no tool_calls) + {"role": "assistant", "content": "Running the command now."}, + {"role": "tool", "tool_call_id": "call_abc123", "content": "math.py"}, + ] + + +def test_tool_result_matches_tool_call_with_text_assistant_in_between(): + messages = _messages_with_text_assistant_between_tool_call_and_result() + + # Should not raise "Missing corresponding tool call for tool response message". + contents = _gemini_convert_messages_with_history(messages=messages) + + # The function response must be present and carry the correct tool name. + function_responses = [ + part["function_response"] + for content in contents + for part in content["parts"] + if isinstance(part, dict) and part.get("function_response") + ] + assert function_responses, f"expected a functionResponse part, got: {contents}" + assert function_responses[0]["name"] == "shell" diff --git a/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py b/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py index 263fb1c6e65..628a6ed4cba 100644 --- a/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py +++ b/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py @@ -1154,44 +1154,82 @@ def test_convert_tool_response_with_base64_image(): ] } - # Convert tool response (returns list when image is present) + # Convert tool response with nested multimodal functionResponse.parts. result = convert_to_gemini_tool_call_result( tool_message, last_message_with_tool_calls ) - # Verify results - should be a list with 2 parts (function_response + inline_data) - assert isinstance( - result, list - ), f"Expected list when image present, got {type(result)}" - assert len(result) == 2, f"Expected 2 parts, got {len(result)}" - - # Find function_response part and inline_data part - function_response_part = None - inline_data_part = None - for part in result: - if "function_response" in part: - function_response_part = part - elif "inline_data" in part: - inline_data_part = part - - # Check function_response exists - assert function_response_part is not None, "Missing function_response part" - function_response = function_response_part["function_response"] + assert isinstance(result, list), "Should return a parts list when media is present" + assert len(result) == 1, "Should return one function_response part" + result_part = result[0] + assert "function_response" in result_part + assert "inline_data" not in result_part + function_response = result_part["function_response"] assert function_response["name"] == "click_at" assert "response" in function_response # Verify JSON response is parsed correctly assert "url" in function_response["response"] assert function_response["response"]["url"] == "https://example.com" - # Check inline_data exists - assert inline_data_part is not None, "Missing inline_data part" - inline_data: BlobType = inline_data_part["inline_data"] + # Check inline_data is nested under functionResponse.parts. + assert "parts" in function_response + assert len(function_response["parts"]) == 1 + inline_data: BlobType = function_response["parts"][0]["inline_data"] assert "data" in inline_data assert "mime_type" in inline_data assert inline_data["mime_type"] == "image/png" assert inline_data["data"] == test_image_base64 +def test_gemini_history_nests_multimodal_tool_response_parts(): + """Full history conversion should not emit sibling inline_data tool result parts.""" + test_image_base64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==" + messages = [ + {"role": "user", "content": "Get me an image"}, + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_get_image", + "type": "function", + "function": {"name": "get_image", "arguments": "{}"}, + } + ], + }, + { + "role": "tool", + "tool_call_id": "call_get_image", + "content": [ + {"type": "text", "text": '{"image_ref": "inline"}'}, + { + "type": "image", + "source": { + "type": "base64", + "media_type": "image/png", + "data": test_image_base64, + }, + }, + ], + }, + ] + + contents = _gemini_convert_messages_with_history(messages=messages) + + tool_response_parts = contents[-1]["parts"] + assert len(tool_response_parts) == 1 + assert "inline_data" not in tool_response_parts[0] + function_response = tool_response_parts[0]["function_response"] + assert function_response["parts"] == [ + { + "inline_data": { + "data": test_image_base64, + "mime_type": "image/png", + } + } + ] + + def test_convert_tool_response_with_url_image(): """Test tool response with HTTP URL image (will download and convert).""" import pytest @@ -1225,24 +1263,20 @@ def test_convert_tool_response_with_url_image(): tool_message, last_message_with_tool_calls ) - # Should be a list with 2 parts when image is present assert isinstance( result, list - ), f"Expected list when image present, got {type(result)}" - assert len(result) == 2, f"Expected 2 parts, got {len(result)}" - - # Find parts - function_response_part = next(p for p in result if "function_response" in p) - inline_data_part = next(p for p in result if "inline_data" in p) - - # Check function_response exists - assert function_response_part is not None, "Missing function_response part" - function_response = function_response_part["function_response"] + ), "Should return a parts list when media is present" + assert len(result) == 1, "Should return one function_response part" + result_part = result[0] + assert "function_response" in result_part + assert "inline_data" not in result_part + function_response = result_part["function_response"] assert function_response["name"] == "type_text_at" - # Check inline_data exists (URL should be downloaded and converted) - assert inline_data_part is not None, "Missing inline_data part" - inline_data: BlobType = inline_data_part["inline_data"] + # Check inline_data is nested under functionResponse.parts. + assert "parts" in function_response + assert len(function_response["parts"]) == 1 + inline_data: BlobType = function_response["parts"][0]["inline_data"] assert "data" in inline_data assert "mime_type" in inline_data except Exception as e: @@ -1558,38 +1592,27 @@ def test_convert_tool_response_with_pdf_file(): ] } - # Convert tool response (returns list when file is present) + # Convert tool response with nested multimodal functionResponse.parts. result = convert_to_gemini_tool_call_result( tool_message, last_message_with_tool_calls ) - # Verify results - should be a list with 2 parts (function_response + inline_data) - assert isinstance( - result, list - ), f"Expected list when file present, got {type(result)}" - assert len(result) == 2, f"Expected 2 parts, got {len(result)}" - - # Find function_response part and inline_data part - function_response_part = None - inline_data_part = None - for part in result: - if "function_response" in part: - function_response_part = part - elif "inline_data" in part: - inline_data_part = part - - # Check function_response exists - assert function_response_part is not None, "Missing function_response part" - function_response = function_response_part["function_response"] + assert isinstance(result, list), "Should return a parts list when media is present" + assert len(result) == 1, "Should return one function_response part" + result_part = result[0] + assert "function_response" in result_part + assert "inline_data" not in result_part + function_response = result_part["function_response"] assert function_response["name"] == "analyze_document" assert "response" in function_response # Verify JSON response is parsed correctly assert "status" in function_response["response"] assert function_response["response"]["status"] == "success" - # Check inline_data exists - assert inline_data_part is not None, "Missing inline_data part" - inline_data: BlobType = inline_data_part["inline_data"] + # Check inline_data is nested under functionResponse.parts. + assert "parts" in function_response + assert len(function_response["parts"]) == 1 + inline_data: BlobType = function_response["parts"][0]["inline_data"] assert "data" in inline_data assert "mime_type" in inline_data assert inline_data["mime_type"] == "application/pdf" @@ -1624,21 +1647,13 @@ def test_convert_tool_response_with_input_file_type(): tool_message, last_message_with_tool_calls ) - # Verify results - assert isinstance( - result, list - ), f"Expected list when file present, got {type(result)}" - assert len(result) == 2, f"Expected 2 parts, got {len(result)}" - - # Find inline_data part - inline_data_part = None - for part in result: - if "inline_data" in part: - inline_data_part = part - - # Check inline_data exists - assert inline_data_part is not None, "Missing inline_data part" - assert inline_data_part["inline_data"]["mime_type"] == "application/pdf" + # Check inline_data is nested under functionResponse.parts. + assert isinstance(result, list), "Should return a parts list when media is present" + assert len(result) == 1, "Should return one function_response part" + function_response = result[0]["function_response"] + assert ( + function_response["parts"][0]["inline_data"]["mime_type"] == "application/pdf" + ) def test_convert_tool_response_with_nested_file_object(): @@ -1669,21 +1684,11 @@ def test_convert_tool_response_with_nested_file_object(): tool_message, last_message_with_tool_calls ) - # Verify results - should be a list with 2 parts - assert isinstance( - result, list - ), f"Expected list when file present, got {type(result)}" - assert len(result) == 2, f"Expected 2 parts, got {len(result)}" - - # Find inline_data part - inline_data_part = None - for part in result: - if "inline_data" in part: - inline_data_part = part - - # Check inline_data exists - assert inline_data_part is not None, "Missing inline_data part" - inline_data: BlobType = inline_data_part["inline_data"] + # Check inline_data is nested under functionResponse.parts. + assert isinstance(result, list), "Should return a parts list when media is present" + assert len(result) == 1, "Should return one function_response part" + function_response = result[0]["function_response"] + inline_data: BlobType = function_response["parts"][0]["inline_data"] assert "data" in inline_data assert "mime_type" in inline_data assert inline_data["mime_type"] == "application/pdf" diff --git a/tests/test_litellm/proxy/auth/test_auth_checks.py b/tests/test_litellm/proxy/auth/test_auth_checks.py index 6f6c4508333..d5043c775b3 100644 --- a/tests/test_litellm/proxy/auth/test_auth_checks.py +++ b/tests/test_litellm/proxy/auth/test_auth_checks.py @@ -3391,30 +3391,44 @@ async def test_resolve_end_user_swallows_db_errors_and_returns_none( @pytest.mark.asyncio -async def test_resolve_end_user_reraises_budget_exceeded( +async def test_resolve_end_user( _validate_flag_on, monkeypatch ): - """BudgetExceededError from get_end_user_object must bubble up so the - auth path enforces spend limits instead of silently dropping the id.""" - import litellm + """Verify that resolve_and_validate_end_user_id does NOT raise BudgetExceededError. + + Note: As of the refactor that moved _check_end_user_budget out of + get_end_user_object, budget enforcement now happens in common_checks(). + + The end-user validation path should return the user ID regardless of budget status. + Budget enforcement for end users happens later in common_checks() via + _check_end_user_budget(), which respects skip_budget_checks for zero-cost models. + + This test verifies that even when get_end_user_object returns a user with a budget, + resolve_and_validate_end_user_id does not block the request - budget enforcement + is deferred to common_checks() where skip_budget_checks logic can be applied. + """ from litellm.proxy.auth import auth_checks from litellm.proxy.auth.auth_checks import resolve_and_validate_end_user_id + # Mock get_end_user_object to return a user with budget info + # (simulating a user who may have exceeded their budget) + mock_end_user = MagicMock() + mock_end_user.user_id = "customer-over-budget" monkeypatch.setattr( auth_checks, "get_end_user_object", - AsyncMock( - side_effect=litellm.BudgetExceededError(current_cost=10.0, max_budget=5.0) - ), + AsyncMock(return_value=mock_end_user), ) cache = _validation_cache() - with pytest.raises(litellm.BudgetExceededError): - await resolve_and_validate_end_user_id( - raw_end_user_id="customer-over-budget", - prisma_client=MagicMock(), - user_api_key_cache=cache, - ) + # resolve_and_validate_end_user_id should return the user ID without raising + # BudgetExceededError - budget enforcement happens in common_checks() + result = await resolve_and_validate_end_user_id( + raw_end_user_id="customer-over-budget", + prisma_client=MagicMock(), + user_api_key_cache=cache, + ) + assert result == "customer-over-budget" @pytest.mark.asyncio diff --git a/tests/test_litellm/proxy/auth/test_custom_auth_end_user_budget.py b/tests/test_litellm/proxy/auth/test_custom_auth_end_user_budget.py index 4084fa4f3aa..68907de6f2d 100644 --- a/tests/test_litellm/proxy/auth/test_custom_auth_end_user_budget.py +++ b/tests/test_litellm/proxy/auth/test_custom_auth_end_user_budget.py @@ -5,7 +5,11 @@ from litellm.proxy.auth.user_api_key_auth import ( _run_post_custom_auth_checks, update_valid_token_with_end_user_params, ) -from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy._types import ( + LiteLLM_BudgetTable, + LiteLLM_EndUserTable, + UserAPIKeyAuth, +) @pytest.mark.asyncio @@ -88,6 +92,85 @@ async def test_custom_auth_run_post_custom_auth_checks_with_end_user_budget_exce mock_budget_check.assert_awaited_once() +@pytest.mark.asyncio +async def test_custom_auth_enforces_end_user_budget_when_common_checks_skipped(): + # custom-auth deployments with custom_auth_run_common_checks unset skip + # common_checks() (and its end-user budget enforcement) in the centralized + # gate, so the helper must enforce the end-user budget itself. Regression: + # an over-budget end user must be rejected on this path. + valid_token = UserAPIKeyAuth(token="test_token", end_user_id="customer-1") + over_budget_end_user = LiteLLM_EndUserTable( + user_id="customer-1", + blocked=False, + spend=0.0, + litellm_budget_table=LiteLLM_BudgetTable(max_budget=1.0), + ) + + async def mock_get_current_spend(counter_key, fallback_spend): + if counter_key == "spend:end_user:customer-1": + return 5.0 + return fallback_spend + + with ( + patch( + "litellm.proxy.auth.user_api_key_auth.get_end_user_object", + new_callable=AsyncMock, + return_value=over_budget_end_user, + ), + patch("litellm.proxy.proxy_server.get_current_spend", mock_get_current_spend), + patch("litellm.proxy.proxy_server.general_settings", {}), + ): + with pytest.raises(litellm.BudgetExceededError): + await _run_post_custom_auth_checks( + valid_token=valid_token, + request=None, + request_data={"model": "gpt-4"}, + route="/v1/chat/completions", + parent_otel_span=None, + ) + + +@pytest.mark.asyncio +async def test_custom_auth_defers_end_user_budget_to_common_checks_when_enabled(): + # With custom_auth_run_common_checks set, the wrapper's common_checks() + # enforces the end-user budget, so the helper must not double-enforce it. + valid_token = UserAPIKeyAuth(token="test_token", end_user_id="customer-1") + end_user_obj = LiteLLM_EndUserTable( + user_id="customer-1", + blocked=False, + spend=0.0, + litellm_budget_table=LiteLLM_BudgetTable(max_budget=1.0), + ) + + with ( + patch( + "litellm.proxy.auth.user_api_key_auth.get_end_user_object", + new_callable=AsyncMock, + return_value=end_user_obj, + ), + patch( + "litellm.proxy.auth.user_api_key_auth._check_end_user_budget", + new_callable=AsyncMock, + ) as mock_check, + patch( + "litellm.proxy.auth.user_api_key_auth._enforce_key_and_fallback_model_access", + new_callable=AsyncMock, + ), + patch( + "litellm.proxy.proxy_server.general_settings", + {"custom_auth_run_common_checks": True}, + ), + ): + await _run_post_custom_auth_checks( + valid_token=valid_token, + request=None, + request_data={"model": "gpt-4"}, + route="/v1/chat/completions", + parent_otel_span=None, + ) + mock_check.assert_not_awaited() + + def test_update_valid_token_does_not_override_custom_auth_values_with_none(): """ Greptile feedback: if custom auth sets end_user_model_max_budget on the token, diff --git a/tests/test_litellm/test_bedrock_anthropic_1hr_cache_pricing.py b/tests/test_litellm/test_bedrock_anthropic_1hr_cache_pricing.py index 69af35dfeae..983f60b0339 100644 --- a/tests/test_litellm/test_bedrock_anthropic_1hr_cache_pricing.py +++ b/tests/test_litellm/test_bedrock_anthropic_1hr_cache_pricing.py @@ -72,9 +72,40 @@ US_EXPECTED = [ ("us.anthropic.claude-haiku-4-5-20251001-v1:0", 2.2e-06, None), ] +# EU/AU/JP cross-region inference profiles carry the same +10% regional +# premium as US (per AWS Bedrock pricing). Coverage list filters to entries +# that actually exist in the pricing JSON - e.g. Opus 4.6 has no JP profile. +REGIONAL_EXPECTED = [ + # Opus 4.6 - $11.00 / MTok (eu/au only; no jp profile) + ("eu.anthropic.claude-opus-4-6-v1", 1.1e-05, None), + ("au.anthropic.claude-opus-4-6-v1", 1.1e-05, None), + # Opus 4.7 - $11.00 / MTok (eu/au; jp is added in #28567) + ("eu.anthropic.claude-opus-4-7", 1.1e-05, None), + ("au.anthropic.claude-opus-4-7", 1.1e-05, None), + # Sonnet 4.6 - $6.60 / MTok + ("eu.anthropic.claude-sonnet-4-6", 6.6e-06, None), + ("au.anthropic.claude-sonnet-4-6", 6.6e-06, None), + ("jp.anthropic.claude-sonnet-4-6", 6.6e-06, None), + # Sonnet 4.5 - $6.60 / MTok with $13.20 / MTok long-context tier + ("eu.anthropic.claude-sonnet-4-5-20250929-v1:0", 6.6e-06, 1.32e-05), + ("au.anthropic.claude-sonnet-4-5-20250929-v1:0", 6.6e-06, 1.32e-05), + ("jp.anthropic.claude-sonnet-4-5-20250929-v1:0", 6.6e-06, 1.32e-05), + # Haiku 4.5 - $2.20 / MTok + ("eu.anthropic.claude-haiku-4-5-20251001-v1:0", 2.2e-06, None), + ("au.anthropic.claude-haiku-4-5-20251001-v1:0", 2.2e-06, None), + ("jp.anthropic.claude-haiku-4-5-20251001-v1:0", 2.2e-06, None), + # Note: eu.anthropic.claude-opus-4-5-20251101-v1:0 is intentionally NOT + # in this list. The existing entry carries base/global 5m rates + # (5e-06 / 6.25e-06) instead of the +10% regional premium (5.5e-06 / + # 6.875e-06), which would make the 1.6x 5m-to-1h invariant fail. + # Fixing the EU 5m rates first is left to a follow-up so this PR + # stays scoped to the 1-hour cache tier addition. +] + @pytest.mark.parametrize( - "model_key, expected_1hr, expected_1hr_lc", GLOBAL_EXPECTED + US_EXPECTED + "model_key, expected_1hr, expected_1hr_lc", + GLOBAL_EXPECTED + US_EXPECTED + REGIONAL_EXPECTED, ) def test_bedrock_anthropic_1hr_cache_write_pricing( model_data, model_key, expected_1hr, expected_1hr_lc diff --git a/tests/test_litellm/test_cost_calculator.py b/tests/test_litellm/test_cost_calculator.py index 1a9bf5a9428..d973f8b4542 100644 --- a/tests/test_litellm/test_cost_calculator.py +++ b/tests/test_litellm/test_cost_calculator.py @@ -2120,11 +2120,11 @@ def test_gemini_3_1_flash_lite_pricing(): ): model_info = litellm.model_cost.get(model_name) assert model_info is not None, f"Missing model pricing entry: {model_name}" - assert model_info["input_cost_per_token"] == 4.5e-07 - assert model_info["input_cost_per_audio_token"] == 9e-07 - assert model_info["output_cost_per_token"] == 2.7e-06 - assert model_info["output_cost_per_reasoning_token"] == 2.7e-06 - assert model_info["cache_read_input_token_cost"] == 4.5e-08 + assert model_info["input_cost_per_token"] == 2.5e-07 + assert model_info["input_cost_per_audio_token"] == 5e-07 + assert model_info["output_cost_per_token"] == 1.5e-06 + assert model_info["output_cost_per_reasoning_token"] == 1.5e-06 + assert model_info["cache_read_input_token_cost"] == 2.5e-08 assert model_info["max_input_tokens"] == 1048576 diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index 5e636b86ed6..e9287a95438 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -2376,6 +2376,74 @@ def test_get_deployment_model_info_base_model_flow(): # Should return None when no model info is found assert result is None + # Test Case 6: custom_model_info present but litellm_model_name_model_info is None + # (model has custom pricing in config but is not in built-in model_prices_and_context_window.json) + mock_custom_pricing_only = { + "input_cost_per_token": 1.74e-06, + "output_cost_per_token": 3.48e-06, + "cache_read_input_token_cost": 1.45e-08, + "mode": "chat", + } + + with patch.object( + litellm, + "model_cost", + {"custom-model-id": mock_custom_pricing_only}, + ): + with patch.object(litellm, "get_model_info") as mock_get_model_info: + # Model NOT in built-in cost map — raise exception + mock_get_model_info.side_effect = Exception("Model not in cost map") + + result = router.get_deployment_model_info( + model_id="custom-model-id", model_name="unknown-model" + ) + + # Should return custom_model_info even when litellm_model_name_model_info is None + assert result is not None + assert result["input_cost_per_token"] == 1.74e-06 + assert result["output_cost_per_token"] == 3.48e-06 + assert result["cache_read_input_token_cost"] == 1.45e-08 + assert result["mode"] == "chat" + + # Test Case 7: custom_model_info with base_model but litellm_model_name_model_info None + mock_custom_with_base = { + "base_model": "some-base-model", + "input_cost_per_token": 0.01, + "output_cost_per_token": 0.02, + } + mock_base_info = { + "key": "some-base-model", + "max_tokens": 8192, + "mode": "chat", + "litellm_provider": "openai", + } + + with patch.object( + litellm, + "model_cost", + {"custom-with-base": mock_custom_with_base}, + ): + with patch.object(litellm, "get_model_info") as mock_get_model_info: + + def get_info_side_effect(model): + if model == "some-base-model": + return mock_base_info + raise Exception("Model not in cost map") + + mock_get_model_info.side_effect = get_info_side_effect + + result = router.get_deployment_model_info( + model_id="custom-with-base", model_name="unknown-model" + ) + + # Should return custom_model_info merged with base model info + assert result is not None + assert ( + result["input_cost_per_token"] == 0.01 + ) # From custom (overrides base) + assert result["max_tokens"] == 8192 # From base model + assert result["litellm_provider"] == "openai" # From base model + print("✓ All base model flow test cases passed!") diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index 6a78653ec99..2d75671f1cb 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -928,6 +928,7 @@ def test_aaamodel_prices_and_context_window_json_is_valid(): }, }, "supports_native_streaming": {"type": "boolean"}, + "supports_image_size": {"type": "boolean"}, "supports_native_structured_output": {"type": "boolean"}, "tiered_pricing": { "type": "array", diff --git a/ui/litellm-dashboard/src/app/layout.tsx b/ui/litellm-dashboard/src/app/layout.tsx index f79b7eb7028..a73921ce35b 100644 --- a/ui/litellm-dashboard/src/app/layout.tsx +++ b/ui/litellm-dashboard/src/app/layout.tsx @@ -11,7 +11,7 @@ const inter = Inter({ subsets: ["latin"] }); export const metadata: Metadata = { title: "LiteLLM Dashboard", description: "LiteLLM Proxy Admin UI", - icons: { icon: "./favicon.ico" }, + icons: { icon: "/get_favicon" }, }; export default function RootLayout({ From 48c9fabb26d948ab3f4f5e4906bec9af8daa6d11 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Wed, 3 Jun 2026 23:44:15 +0530 Subject: [PATCH 4/7] Fix : a2a bugs 030626 (#29566) * Fix error code and context id injection bug * Add support for all A2A methods * Add logging * address greptile review: relay upstream JSON-RPC errors, move _PASCAL_TO_WIRE to module level, add error path tests * fix(a2a): run pre_call_hook for tasks/resubscribe SSE path to enforce guardrails tasks/resubscribe was returning the raw SSE stream without calling proxy_logging_obj.pre_call_hook, silently bypassing any guardrails configured on the agent. This patch calls pre_call_hook before streaming begins and wires post_call_failure_hook into the SSE generator so errors are logged. Adds a regression test verifying the hook is called. * fix(a2a): use get_async_httpx_client instead of creating httpx clients per request Creating httpx.AsyncClient instances per-request adds ~500ms latency. Switch _forward_jsonrpc and _forward_jsonrpc_sse to use the shared client from get_async_httpx_client(httpxSpecialProvider.A2A). * fix(a2a): forward caller identity headers on task ops; validate push notification URL Two security fixes for task management methods: 1. All task operations (tasks/get, tasks/list, tasks/cancel, tasks/resubscribe, push notification config methods) now forward X-LiteLLM-User-Id and X-LiteLLM-Team-Id headers to the upstream agent, so the agent can scope task access to the authenticated caller. 2. tasks/pushNotificationConfig/set validates the callback URL before forwarding: requires HTTPS and rejects private/loopback/reserved IP ranges and localhost hostnames to prevent SSRF. * Fix A2A task hook and push URL handling * fix(a2a): fix mypy type errors for request_id and header_name dict key types * Fix A2A request id and params forwarding * Forward trace IDs for A2A task calls * fix(a2a): strip client-forwarded X-LiteLLM-* headers before applying authenticated identity A client could send x-a2a--x-litellm-user-id in their request and have it forwarded to the upstream agent as an authenticated identity header. Fix: sanitize any X-LiteLLM-* headers from agent_extra_headers before merging, then apply the authenticated identity headers last so they always override client-supplied values. * Fix A2A SSE fallback JSON-RPC error code * Fix A2A SSE error id backfill * fix(a2a): validate both push notification url fields to close SSRF bypass * fix(a2a): widen request_id annotation to match JSON-RPC id call sites * fix(a2a): run post-call streaming hook for tasks/resubscribe so agent guardrails apply tasks/resubscribe returned the raw upstream SSE stream without routing events through the post-call streaming hook, so output guardrails configured on the agent were silently skipped for streaming task subscriptions while every other task method and message/stream applied them. Parse upstream JSON-RPC SSE events and feed them through async_streaming_data_generator, matching message/stream, so guardrails inspect the streamed task content. Adds a regression test that fails when the streamed events bypass the guardrail hook. --------- Co-authored-by: Cursor Agent Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- litellm/a2a_protocol/main.py | 17 +- .../proxy/agent_endpoints/a2a_endpoints.py | 367 ++++- litellm/types/agents.py | 38 +- .../test_send_message_response.py | 43 + .../agent_endpoints/test_a2a_endpoints.py | 1212 +++++++++++++++++ 5 files changed, 1639 insertions(+), 38 deletions(-) create mode 100644 tests/test_litellm/a2a_protocol/test_send_message_response.py diff --git a/litellm/a2a_protocol/main.py b/litellm/a2a_protocol/main.py index 3ad5485dea1..6979e1ac659 100644 --- a/litellm/a2a_protocol/main.py +++ b/litellm/a2a_protocol/main.py @@ -159,7 +159,9 @@ async def _send_message_via_completion_bridge( api_base=api_base, ) - return LiteLLMSendMessageResponse.from_dict(response_dict) + return LiteLLMSendMessageResponse.from_dict( + response_dict, request_id=str(request.id) + ) async def _execute_a2a_send_with_retry( @@ -317,15 +319,6 @@ async def asend_message( ) card_url = getattr(agent_card, "url", None) if agent_card else None - context_id = trace_id or str(uuid.uuid4()) - message = request.params.message - if isinstance(message, dict): - if message.get("context_id") is None: - message["context_id"] = context_id - else: - if getattr(message, "context_id", None) is None: - message.context_id = context_id - a2a_response = await _execute_a2a_send_with_retry( a2a_client=a2a_client, request=request, @@ -338,7 +331,9 @@ async def asend_message( verbose_logger.info(f"A2A send_message completed, request_id={request.id}") # Wrap in LiteLLM response type for _hidden_params support - response = LiteLLMSendMessageResponse.from_a2a_response(a2a_response) + response = LiteLLMSendMessageResponse.from_a2a_response( + a2a_response, request_id=str(request.id) + ) # Calculate token usage from request and response response_dict = a2a_response.model_dump(mode="json", exclude_none=True) diff --git a/litellm/proxy/agent_endpoints/a2a_endpoints.py b/litellm/proxy/agent_endpoints/a2a_endpoints.py index 7b56155982c..9f1403d4328 100644 --- a/litellm/proxy/agent_endpoints/a2a_endpoints.py +++ b/litellm/proxy/agent_endpoints/a2a_endpoints.py @@ -6,12 +6,14 @@ The A2A SDK can point to LiteLLM's URL and invoke agents registered with LiteLLM """ import json -from typing import Any, Dict, List, Optional +from typing import Any, AsyncGenerator, Dict, List, Optional +from urllib.parse import urlparse from fastapi import APIRouter, Depends, HTTPException, Request, Response from fastapi.responses import JSONResponse, StreamingResponse from litellm._logging import verbose_proxy_logger +from litellm.litellm_core_utils.url_utils import SSRFError, validate_url from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.agent_endpoints.utils import merge_agent_headers from litellm.proxy.auth.user_api_key_auth import user_api_key_auth @@ -19,9 +21,66 @@ from litellm.types.utils import all_litellm_params router = APIRouter() +_PASCAL_TO_WIRE: Dict[str, str] = { + "GetTask": "tasks/get", + "ListTasks": "tasks/list", + "CancelTask": "tasks/cancel", + "SubscribeToTask": "tasks/resubscribe", + "CreateTaskPushNotificationConfig": "tasks/pushNotificationConfig/set", + "GetTaskPushNotificationConfig": "tasks/pushNotificationConfig/get", + "ListTaskPushNotificationConfigs": "tasks/pushNotificationConfig/list", + "DeleteTaskPushNotificationConfig": "tasks/pushNotificationConfig/delete", + "GetExtendedAgentCard": "agent/getAuthenticatedExtendedCard", +} + + +def _validate_push_notification_url(url: str) -> None: + parsed = urlparse(url) + if parsed.scheme != "https": + raise HTTPException( + status_code=400, + detail="Push notification URL must use HTTPS", + ) + try: + validate_url(url) + except (SSRFError, ValueError) as e: + raise HTTPException(status_code=400, detail=str(e)) from e + + +def _caller_identity_headers(user_api_key_dict: UserAPIKeyAuth) -> Dict[str, str]: + headers: Dict[str, str] = {} + if user_api_key_dict.user_id: + headers["X-LiteLLM-User-Id"] = user_api_key_dict.user_id + if user_api_key_dict.team_id: + headers["X-LiteLLM-Team-Id"] = user_api_key_dict.team_id + return headers + + +def _forwarding_headers( + user_api_key_dict: UserAPIKeyAuth, + request_data: dict, + agent_extra_headers: Optional[Dict[str, str]], +) -> Optional[Dict[str, str]]: + sanitized = ( + { + k: v + for k, v in agent_extra_headers.items() + if not k.lower().startswith("x-litellm-") + } + if agent_extra_headers + else None + ) + merged = merge_agent_headers(dynamic_headers=sanitized, static_headers=None) or {} + identity = _caller_identity_headers(user_api_key_dict) + trace_id = request_data.get("litellm_trace_id") + if trace_id: + identity["X-LiteLLM-Trace-Id"] = str(trace_id) + merged.update(identity) + return merged or None + def _jsonrpc_error( - request_id: Optional[str], + request_id: Optional[Any], code: int, message: str, status_code: int = 400, @@ -67,9 +126,158 @@ def _enforce_inbound_trace_id(agent: Any, request: Request) -> None: ) +async def _forward_jsonrpc( + agent_url: str, + body: dict, + extra_headers: Optional[Dict[str, str]] = None, +) -> dict: + from litellm.llms.custom_httpx.http_handler import get_async_httpx_client + from litellm.types.llms.custom_http import httpxSpecialProvider + + headers = {"Content-Type": "application/json", **(extra_headers or {})} + handler = get_async_httpx_client( + llm_provider=httpxSpecialProvider.A2A, + params={"timeout": 60.0}, + ) + resp = await handler.post(agent_url, json=body, headers=headers) + try: + result = resp.json() + except Exception: + resp.raise_for_status() + raise + if not resp.is_success and "error" not in result: + resp.raise_for_status() + return result + + +async def _a2a_sse_event_source( + agent_url: str, + body: dict, + request_id: Optional[Any] = None, + extra_headers: Optional[Dict[str, str]] = None, +) -> AsyncGenerator[dict, None]: + """Stream an upstream A2A SSE response as parsed JSON-RPC event dicts. + + Upstream HTTP/JSON-RPC errors are surfaced as a single JSON-RPC error event + so the caller can relay them instead of breaking the stream. + """ + from litellm.llms.custom_httpx.http_handler import get_async_httpx_client + from litellm.types.agents import _normalize_a2a_jsonrpc_response + from litellm.types.llms.custom_http import httpxSpecialProvider + + headers = { + "Content-Type": "application/json", + "Accept": "text/event-stream", + **(extra_headers or {}), + } + handler = get_async_httpx_client( + llm_provider=httpxSpecialProvider.A2A, + params={"timeout": None}, + ) + async_client = handler.client + req = async_client.build_request("POST", agent_url, json=body, headers=headers) + resp = await async_client.send(req, stream=True) + try: + if not resp.is_success: + error_body = await resp.aread() + error_event: Optional[dict] = None + try: + parsed = json.loads(error_body) + if isinstance(parsed, dict) and "error" in parsed: + error_event = _normalize_a2a_jsonrpc_response( + parsed, request_id=request_id + ) + except Exception: + error_event = None + yield error_event or { + "jsonrpc": "2.0", + "id": request_id, + "error": {"code": -32603, "message": resp.reason_phrase}, + } + return + async for line in resp.aiter_lines(): + stripped = line.strip() + if not stripped.startswith("data:"): + continue + payload = stripped[len("data:") :].strip() + if not payload: + continue + try: + yield json.loads(payload) + except Exception: + continue + finally: + await resp.aclose() + + +async def _forward_jsonrpc_sse( + agent_url: str, + body: dict, + request_id: Optional[Any] = None, + extra_headers: Optional[Dict[str, str]] = None, + proxy_logging_obj: Optional[Any] = None, + user_api_key_dict: Optional[Any] = None, + request_data: Optional[dict] = None, +) -> StreamingResponse: + event_source = _a2a_sse_event_source( + agent_url, body, request_id=request_id, extra_headers=extra_headers + ) + + def _serialize_chunk(chunk: Any) -> str: + return f"data: {json.dumps(chunk)}\n\n" + + def _serialize_error(proxy_exc: Any) -> str: + return ( + "data: " + + json.dumps( + { + "jsonrpc": "2.0", + "id": request_id, + "error": { + "code": -32603, + "message": getattr(proxy_exc, "message", str(proxy_exc)), + }, + } + ) + + "\n\n" + ) + + if ( + proxy_logging_obj is not None + and user_api_key_dict is not None + and request_data is not None + ): + # Route streamed events through the shared streaming generator so the + # post-call streaming hook (and therefore agent guardrails) inspects + # tasks/resubscribe output the same way message/stream does. + from litellm.proxy.common_request_processing import ( + ProxyBaseLLMRequestProcessing, + ) + + generator: AsyncGenerator[str, None] = ( + ProxyBaseLLMRequestProcessing.async_streaming_data_generator( + response=event_source, + user_api_key_dict=user_api_key_dict, + request_data=request_data, + proxy_logging_obj=proxy_logging_obj, + serialize_chunk=_serialize_chunk, + serialize_error=_serialize_error, + ) + ) + else: + + async def _passthrough() -> AsyncGenerator[str, None]: + async for chunk in event_source: + yield _serialize_chunk(chunk) + + generator = _passthrough() + + return StreamingResponse(generator, media_type="text/event-stream") + + async def _handle_stream_message( api_base: Optional[str], - request_id: str, + request_id: Any, params: dict, litellm_params: Optional[dict] = None, agent_id: Optional[str] = None, @@ -310,8 +518,6 @@ async def invoke_agent_a2a( # noqa: PLR0915 - message/send: Send a message and get a response - message/stream: Send a message and stream the response """ - from litellm.a2a_protocol import asend_message - from litellm.a2a_protocol.main import A2A_SDK_AVAILABLE from litellm.proxy.agent_endpoints.auth.agent_permission_handler import ( AgentRequestHandler, ) @@ -322,9 +528,11 @@ async def invoke_agent_a2a( # noqa: PLR0915 version, ) - body = {} + body: Dict[str, Any] = {} + request_data: Dict[str, Any] = body try: body = await request.json() + request_data = body verbose_proxy_logger.debug(f"A2A request for agent '{agent_id}': {body}") @@ -334,11 +542,14 @@ async def invoke_agent_a2a( # noqa: PLR0915 body.get("id"), -32600, "Invalid Request: jsonrpc must be '2.0'" ) - request_id = body.get("id") - method = body.get("method") + request_id: Optional[Any] = body.get("id") + method: Optional[str] = body.get("method") params = body.get("params", {}) - if params: + if method: + method = _PASCAL_TO_WIRE.get(method, method) + + if isinstance(params, dict): # extract any litellm params from the params - eg. 'guardrails' # ``metadata`` is intentionally excluded: it's a first-class A2A # ``MessageSendParams`` field that the completion bridge forwards @@ -347,20 +558,12 @@ async def invoke_agent_a2a( # noqa: PLR0915 # silently drop the caller's A2A request-level metadata. params_to_remove = [] for key, value in params.items(): - if key in all_litellm_params and key != "metadata": + if key in all_litellm_params and key not in {"id", "metadata"}: params_to_remove.append(key) body[key] = value for key in params_to_remove: params.pop(key) - if not A2A_SDK_AVAILABLE: - return _jsonrpc_error( - request_id, - -32603, - "Server error: 'a2a' package not installed. Please install 'a2a-sdk'.", - 500, - ) - # Find the agent agent = _get_agent(agent_id) if agent is None: @@ -441,6 +644,7 @@ async def invoke_agent_a2a( # noqa: PLR0915 route_type="asend_message", version=version, ) + request_data = data # Build merged headers for the backend agent static_headers: Dict[str, str] = dict(agent.static_headers or {}) @@ -453,9 +657,10 @@ async def invoke_agent_a2a( # noqa: PLR0915 # 1. Admin-configured extra_headers: forward named headers from client request if agent.extra_headers: for header_name in agent.extra_headers: - val = normalized.get(header_name.lower()) + header_name_str = str(header_name) + val = normalized.get(header_name_str.lower()) if val is not None: - dynamic_headers[header_name] = val + dynamic_headers[header_name_str] = val # 2. Convention-based forwarding: x-a2a-{agent_id_or_name}-{header_name} # Matches both agent_id (UUID) and agent_name (alias), case-insensitive. @@ -489,10 +694,20 @@ async def invoke_agent_a2a( # noqa: PLR0915 # Route through SDK functions if method == "message/send": + from litellm.a2a_protocol import asend_message + from litellm.a2a_protocol.main import A2A_SDK_AVAILABLE + + if not A2A_SDK_AVAILABLE: + return _jsonrpc_error( + request_id, + -32603, + "Server error: 'a2a' package not installed. Please install 'a2a-sdk'.", + 500, + ) from a2a.types import MessageSendParams, SendMessageRequest a2a_request = SendMessageRequest( - id=request_id, + id=request_id if request_id is not None else "", params=MessageSendParams(**params), ) # Defer spend-log until after post_call_success_hook so guardrail @@ -532,7 +747,7 @@ async def invoke_agent_a2a( # noqa: PLR0915 elif method == "message/stream": return await _handle_stream_message( api_base=agent_url, - request_id=request_id, + request_id=request_id if request_id is not None else "", params=params, litellm_params=litellm_params, agent_id=agent.agent_id, @@ -543,6 +758,106 @@ async def invoke_agent_a2a( # noqa: PLR0915 request_data=data, proxy_logging_obj=proxy_logging_obj, ) + elif method in { + "tasks/get", + "tasks/list", + "tasks/cancel", + "tasks/pushNotificationConfig/set", + "tasks/pushNotificationConfig/get", + "tasks/pushNotificationConfig/list", + "tasks/pushNotificationConfig/delete", + "agent/getAuthenticatedExtendedCard", + }: + if not agent_url: + return _jsonrpc_error( + request_id, -32000, f"Agent '{agent_id}' has no URL configured", 500 + ) + if method == "tasks/pushNotificationConfig/set": + if not isinstance(params, dict): + raise HTTPException( + status_code=400, + detail="params must be an object", + ) + push_config = params.get("pushNotificationConfig", {}) + if "pushNotificationConfig" in params and not isinstance( + push_config, dict + ): + raise HTTPException( + status_code=400, + detail="pushNotificationConfig must be an object", + ) + for callback_url in (params.get("url"), push_config.get("url")): + if not callback_url: + continue + if not isinstance(callback_url, str): + raise HTTPException( + status_code=400, + detail="Push notification URL must be a string", + ) + _validate_push_notification_url(callback_url) + forward_body = { + "jsonrpc": "2.0", + "id": request_id, + "method": method, + "params": params, + } + caller_headers = _forwarding_headers( + user_api_key_dict=user_api_key_dict, + request_data=data, + agent_extra_headers=agent_extra_headers, + ) + result = await _forward_jsonrpc( + agent_url, forward_body, extra_headers=caller_headers + ) + if method == "agent/getAuthenticatedExtendedCard": + if isinstance(result.get("result"), dict) and "url" in result["result"]: + result["result"][ + "url" + ] = f"{str(request.base_url).rstrip('/')}/a2a/{agent_id}" + from litellm.types.agents import LiteLLMSendMessageResponse + + response = LiteLLMSendMessageResponse.from_dict( + result, request_id=request_id + ) + response = await proxy_logging_obj.post_call_success_hook( + user_api_key_dict=user_api_key_dict, + data=data, + response=response, + ) + return JSONResponse( + content=( + response.model_dump(mode="json", exclude_none=True) + if hasattr(response, "model_dump") + else response + ) + ) + + elif method == "tasks/resubscribe": + if not agent_url: + return _jsonrpc_error( + request_id, -32000, f"Agent '{agent_id}' has no URL configured", 500 + ) + forward_body = { + "jsonrpc": "2.0", + "id": request_id, + "method": method, + "params": params, + } + sse_caller_headers = _forwarding_headers( + user_api_key_dict=user_api_key_dict, + request_data=data, + agent_extra_headers=agent_extra_headers, + ) + return await _forward_jsonrpc_sse( + agent_url, + forward_body, + request_id=request_id, + extra_headers=sse_caller_headers, + proxy_logging_obj=proxy_logging_obj, + user_api_key_dict=user_api_key_dict, + request_data=data, + ) + else: return _jsonrpc_error(request_id, -32601, f"Method '{method}' not found") @@ -550,4 +865,12 @@ async def invoke_agent_a2a( # noqa: PLR0915 raise except Exception as e: verbose_proxy_logger.exception(f"Error invoking agent: {e}") + try: + await proxy_logging_obj.post_call_failure_hook( + user_api_key_dict=user_api_key_dict, + original_exception=e, + request_data=request_data, + ) + except Exception: + pass return _jsonrpc_error(body.get("id"), -32603, f"Internal error: {str(e)}", 500) diff --git a/litellm/types/agents.py b/litellm/types/agents.py index 8556b6bac93..f34631b5600 100644 --- a/litellm/types/agents.py +++ b/litellm/types/agents.py @@ -298,6 +298,23 @@ class MakeAgentsPublicRequest(BaseModel): agent_ids: List[str] +def _normalize_a2a_jsonrpc_response( + response_dict: Dict[str, Any], + request_id: Optional[Any] = None, +) -> Dict[str, Any]: + """ + Ensure JSON-RPC responses include ``id`` when the caller supplied one. + + The a2a SDK may omit ``id`` on error payloads even when the upstream agent + returned it. Backfill from the outbound request id so LiteLLM can surface the + agent error instead of failing Pydantic validation. + """ + normalized = dict(response_dict) + if normalized.get("id") is None and request_id is not None: + normalized["id"] = str(request_id) + return normalized + + class LiteLLMSendMessageResponse(LiteLLMPydanticObjectBase): """ LiteLLM wrapper for A2A SendMessageResponse. @@ -322,31 +339,42 @@ class LiteLLMSendMessageResponse(LiteLLMPydanticObjectBase): @classmethod def from_a2a_response( - cls, response: "SendMessageResponse" + cls, + response: "SendMessageResponse", + request_id: Optional[Any] = None, ) -> "LiteLLMSendMessageResponse": """ Create a LiteLLMSendMessageResponse from an a2a SDK SendMessageResponse. Args: response: The a2a SDK SendMessageResponse + request_id: JSON-RPC request id to backfill when the SDK omits it on errors Returns: LiteLLMSendMessageResponse with _hidden_params support """ - # Convert the a2a response to a dict response_dict = response.model_dump(mode="json", exclude_none=True) - + response_dict = _normalize_a2a_jsonrpc_response( + response_dict, request_id=request_id + ) return cls(**response_dict) @classmethod - def from_dict(cls, response_dict: Dict[str, Any]) -> "LiteLLMSendMessageResponse": + def from_dict( + cls, + response_dict: Dict[str, Any], + request_id: Optional[Any] = None, + ) -> "LiteLLMSendMessageResponse": """ Create a LiteLLMSendMessageResponse from a dict. Args: response_dict: Dict with A2A response structure + request_id: JSON-RPC request id to backfill when missing on error payloads Returns: LiteLLMSendMessageResponse with _hidden_params support """ - return cls(**response_dict) + return cls( + **_normalize_a2a_jsonrpc_response(response_dict, request_id=request_id) + ) diff --git a/tests/test_litellm/a2a_protocol/test_send_message_response.py b/tests/test_litellm/a2a_protocol/test_send_message_response.py new file mode 100644 index 00000000000..832aa288c7a --- /dev/null +++ b/tests/test_litellm/a2a_protocol/test_send_message_response.py @@ -0,0 +1,43 @@ +"""Tests for LiteLLMSendMessageResponse JSON-RPC normalization.""" + +from litellm.types.agents import LiteLLMSendMessageResponse + + +def test_from_dict_backfills_id_on_agent_error_response(): + agent_error = { + "jsonrpc": "2.0", + "error": {"code": -32054, "message": "Session not found"}, + } + + response = LiteLLMSendMessageResponse.from_dict( + agent_error, request_id="r1" + ) + + assert response.id == "r1" + assert response.error == {"code": -32054, "message": "Session not found"} + assert response.result is None + + +def test_from_dict_preserves_existing_id(): + payload = { + "id": "upstream-id", + "jsonrpc": "2.0", + "error": {"code": -32001, "message": "Task not found"}, + } + + response = LiteLLMSendMessageResponse.from_dict( + payload, request_id="r1" + ) + + assert response.id == "upstream-id" + + +def test_from_dict_without_request_id_still_requires_id(): + try: + LiteLLMSendMessageResponse.from_dict( + {"jsonrpc": "2.0", "error": {"code": -32054, "message": "x"}} + ) + except Exception as exc: + assert "id" in str(exc).lower() + else: + raise AssertionError("expected validation error when id and request_id missing") diff --git a/tests/test_litellm/proxy/agent_endpoints/test_a2a_endpoints.py b/tests/test_litellm/proxy/agent_endpoints/test_a2a_endpoints.py index a32f2eadb99..07e878401e0 100644 --- a/tests/test_litellm/proxy/agent_endpoints/test_a2a_endpoints.py +++ b/tests/test_litellm/proxy/agent_endpoints/test_a2a_endpoints.py @@ -5,7 +5,9 @@ Tests that invoke_agent_a2a properly integrates with add_litellm_data_to_request """ import json +import socket import sys +from contextlib import ExitStack from unittest.mock import AsyncMock, MagicMock, patch import pytest @@ -348,3 +350,1213 @@ async def test_invoke_agent_a2a_injects_authenticated_key_hash_for_bridge(): captured.get("litellm_params", {}).get(A2A_USER_API_KEY_HASH_PARAM) == mock_user_api_key_dict.api_key ), "authenticated key hash was not forwarded to the completion bridge" + + +def _make_agent_mock(url: str = "http://backend-agent:10001") -> MagicMock: + agent = MagicMock() + agent.agent_id = "test-agent" + agent.agent_name = "test-agent" + agent.agent_card_params = {"url": url, "name": "Test Agent"} + agent.litellm_params = {} + agent.static_headers = None + agent.extra_headers = None + return agent + + +def _make_request_mock( + method: str, params: dict, request_id: object = "req-1" +) -> MagicMock: + req = MagicMock() + req.headers = {} + req.json = AsyncMock( + return_value={ + "jsonrpc": "2.0", + "id": request_id, + "method": method, + "params": params, + } + ) + return req + + +def _base_patches(agent: MagicMock): + return [ + patch( + "litellm.proxy.agent_endpoints.a2a_endpoints._get_agent", + return_value=agent, + ), + patch( + "litellm.proxy.agent_endpoints.auth.agent_permission_handler.AgentRequestHandler.is_agent_allowed", + new=AsyncMock(return_value=True), + ), + patch( + "litellm.proxy.common_request_processing.add_litellm_data_to_request", + new=AsyncMock(side_effect=_add_proxy_data), + ), + patch("litellm.proxy.proxy_server.general_settings", {}), + patch("litellm.proxy.proxy_server.proxy_config", MagicMock()), + patch("litellm.proxy.proxy_server.version", "1.0.0"), + ] + + +async def _add_proxy_data(data, **kwargs): + data["proxy_server_request"] = { + "url": "http://localhost:4000", + "method": "POST", + "headers": {}, + "body": {}, + } + data.setdefault("metadata", {}) + return data + + +@pytest.mark.asyncio +@pytest.mark.parametrize("method", ["message/send", "message/stream"]) +async def test_message_methods_preserve_numeric_zero_request_id(method: str): + from fastapi.responses import JSONResponse + from litellm.proxy._types import UserAPIKeyAuth + + class MessageSendParams: + def __init__(self, **kwargs): + self.__dict__.update(kwargs) + + class SendMessageRequest: + def __init__(self, **kwargs): + self.__dict__.update(kwargs) + + agent = _make_agent_mock() + params = { + "message": { + "role": "user", + "parts": [{"kind": "text", "text": "Hello"}], + "messageId": "msg-123", + } + } + mock_request = _make_request_mock(method, params, request_id=0) + user_api_key_dict = UserAPIKeyAuth(api_key="sk-test", user_id="u1", team_id="t1") + captured = {} + + async def capture_asend_message(request, **kwargs): + captured["request_id"] = request.id + response = MagicMock() + response.model_dump.return_value = { + "jsonrpc": "2.0", + "id": request.id, + "result": {"status": "success"}, + } + return response + + async def capture_stream_message(**kwargs): + captured["request_id"] = kwargs["request_id"] + return JSONResponse({"jsonrpc": "2.0", "id": kwargs["request_id"]}) + + mock_a2a_types = MagicMock() + mock_a2a_types.MessageSendParams = MessageSendParams + mock_a2a_types.SendMessageRequest = SendMessageRequest + + with ExitStack() as stack: + for p in _base_patches(agent): + stack.enter_context(p) + stack.enter_context(patch("litellm.a2a_protocol.main.A2A_SDK_AVAILABLE", True)) + if method == "message/send": + stack.enter_context( + patch.dict( + sys.modules, + {"a2a": MagicMock(), "a2a.types": mock_a2a_types}, + ) + ) + stack.enter_context( + patch( + "litellm.a2a_protocol.asend_message", + new=AsyncMock(side_effect=capture_asend_message), + ) + ) + else: + stack.enter_context( + patch( + "litellm.proxy.agent_endpoints.a2a_endpoints._handle_stream_message", + new=AsyncMock(side_effect=capture_stream_message), + ) + ) + + from litellm.proxy.agent_endpoints.a2a_endpoints import invoke_agent_a2a + + await invoke_agent_a2a( + agent_id="test-agent", + request=mock_request, + fastapi_response=MagicMock(), + user_api_key_dict=user_api_key_dict, + ) + + assert captured["request_id"] == 0 + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "method,params", + [ + ("tasks/get", {"id": "task-1"}), + ("tasks/list", {"contextId": "ctx-1"}), + ("tasks/cancel", {"id": "task-1"}), + ( + "tasks/pushNotificationConfig/set", + {"taskId": "task-1", "url": "https://webhook.example.com"}, + ), + ("tasks/pushNotificationConfig/get", {"taskId": "task-1", "id": "cfg-1"}), + ("tasks/pushNotificationConfig/list", {"taskId": "task-1"}), + ("tasks/pushNotificationConfig/delete", {"taskId": "task-1", "id": "cfg-1"}), + ], +) +async def test_task_methods_forward_jsonrpc(method: str, params: dict): + from litellm.proxy._types import UserAPIKeyAuth + + upstream_response = { + "jsonrpc": "2.0", + "id": "req-1", + "result": {"id": "task-1", "status": {"state": "completed"}}, + } + agent = _make_agent_mock() + mock_request = _make_request_mock(method, params) + + mock_http_response = MagicMock() + mock_http_response.json.return_value = upstream_response + mock_http_response.is_success = True + mock_http_response.raise_for_status = MagicMock() + + mock_handler = MagicMock() + mock_handler.post = AsyncMock(return_value=mock_http_response) + mock_handler.client = MagicMock() + + user_api_key_dict = UserAPIKeyAuth(api_key="sk-test", user_id="u1", team_id="t1") + + with ExitStack() as stack: + for p in _base_patches(agent): + stack.enter_context(p) + stack.enter_context( + patch( + "litellm.llms.custom_httpx.http_handler.get_async_httpx_client", + return_value=mock_handler, + ) + ) + stack.enter_context( + patch( + "litellm.proxy.agent_endpoints.a2a_endpoints.validate_url", + return_value=("https://webhook.example.com", "webhook.example.com"), + ) + ) + + from litellm.proxy.agent_endpoints.a2a_endpoints import invoke_agent_a2a + + response = await invoke_agent_a2a( + agent_id="test-agent", + request=mock_request, + fastapi_response=MagicMock(), + user_api_key_dict=user_api_key_dict, + ) + + body = json.loads(response.body.decode()) + assert body["jsonrpc"] == "2.0" + assert body["result"]["id"] == "task-1" + + posted = mock_handler.post.call_args + assert posted is not None + forwarded_body = posted.kwargs.get("json") or posted.args[1] + assert forwarded_body["method"] == method + + +@pytest.mark.asyncio +@pytest.mark.parametrize("method", ["tasks/get", "tasks/resubscribe"]) +async def test_task_methods_extract_litellm_params_before_forwarding(method: str): + from litellm.proxy._types import UserAPIKeyAuth + + agent = _make_agent_mock() + params = { + "id": "task-1", + "guardrails": ["guardrail-1"], + "tags": ["tag-1"], + } + mock_request = _make_request_mock(method, params) + user_api_key_dict = UserAPIKeyAuth(api_key="sk-test", user_id="u1", team_id="t1") + captured_data = {} + + async def capture_proxy_data(data, **kwargs): + captured_data.update(data) + return await _add_proxy_data(data, **kwargs) + + upstream_response = { + "jsonrpc": "2.0", + "id": "req-1", + "result": {"id": "task-1", "status": {"state": "completed"}}, + } + mock_http_response = MagicMock() + mock_http_response.json.return_value = upstream_response + mock_http_response.is_success = True + + async def fake_aiter_lines(): + yield 'data: {"jsonrpc":"2.0","id":"req-1","result":{"taskId":"task-1"}}' + + mock_resp = AsyncMock() + mock_resp.is_success = True + mock_resp.aiter_lines = fake_aiter_lines + mock_resp.aclose = AsyncMock() + + mock_async_client = MagicMock() + mock_async_client.build_request = MagicMock(return_value=MagicMock()) + mock_async_client.send = AsyncMock(return_value=mock_resp) + + mock_handler = MagicMock() + mock_handler.post = AsyncMock(return_value=mock_http_response) + mock_handler.client = mock_async_client + + with ExitStack() as stack: + for p in _base_patches(agent): + stack.enter_context(p) + stack.enter_context( + patch( + "litellm.proxy.common_request_processing.add_litellm_data_to_request", + new=AsyncMock(side_effect=capture_proxy_data), + ) + ) + stack.enter_context( + patch( + "litellm.llms.custom_httpx.http_handler.get_async_httpx_client", + return_value=mock_handler, + ) + ) + + from litellm.proxy.agent_endpoints.a2a_endpoints import invoke_agent_a2a + + response = await invoke_agent_a2a( + agent_id="test-agent", + request=mock_request, + fastapi_response=MagicMock(), + user_api_key_dict=user_api_key_dict, + ) + if method == "tasks/resubscribe": + async for _ in response.body_iterator: + pass + + if method == "tasks/resubscribe": + forwarded_body = mock_async_client.build_request.call_args.kwargs["json"] + else: + forwarded_body = mock_handler.post.call_args.kwargs["json"] + assert forwarded_body["params"] == {"id": "task-1"} + assert captured_data["guardrails"] == ["guardrail-1"] + assert captured_data["tags"] == ["tag-1"] + + +@pytest.mark.asyncio +async def test_subscribe_to_task_returns_sse_stream(): + from litellm.proxy._types import UserAPIKeyAuth + + agent = _make_agent_mock() + mock_request = _make_request_mock("SubscribeToTask", {"id": "task-1"}) + user_api_key_dict = UserAPIKeyAuth(api_key="sk-test", user_id="u1", team_id="t1") + + sse_lines = [ + 'data: {"jsonrpc":"2.0","id":"req-1","result":{"taskId":"task-1","status":{"state":"working"}}}', + 'data: {"jsonrpc":"2.0","id":"req-1","result":{"taskId":"task-1","status":{"state":"completed"}}}', + ] + + async def fake_aiter_lines(): + for line in sse_lines: + yield line + + mock_resp = AsyncMock() + mock_resp.is_success = True + mock_resp.aiter_lines = fake_aiter_lines + mock_resp.aclose = AsyncMock() + + mock_async_client = MagicMock() + mock_async_client.build_request = MagicMock(return_value=MagicMock()) + mock_async_client.send = AsyncMock(return_value=mock_resp) + + mock_handler = MagicMock() + mock_handler.client = mock_async_client + mock_handler.post = AsyncMock() + + chunks = [] + with ExitStack() as stack: + for p in _base_patches(agent): + stack.enter_context(p) + stack.enter_context( + patch( + "litellm.llms.custom_httpx.http_handler.get_async_httpx_client", + return_value=mock_handler, + ) + ) + + from litellm.proxy.agent_endpoints.a2a_endpoints import invoke_agent_a2a + + response = await invoke_agent_a2a( + agent_id="test-agent", + request=mock_request, + fastapi_response=MagicMock(), + user_api_key_dict=user_api_key_dict, + ) + + assert response.media_type == "text/event-stream" + async for chunk in response.body_iterator: + chunks.append(chunk) + + full = "".join(chunks) + assert "working" in full + assert "completed" in full + + +@pytest.mark.asyncio +async def test_subscribe_to_task_calls_pre_call_hook(): + """tasks/resubscribe must run pre_call_hook so guardrails configured on + the agent are enforced before streaming begins.""" + from litellm.proxy._types import UserAPIKeyAuth + + agent = _make_agent_mock() + mock_request = _make_request_mock("tasks/resubscribe", {"id": "task-1"}) + user_api_key_dict = UserAPIKeyAuth(api_key="sk-test", user_id="u1", team_id="t1") + + async def fake_aiter_lines(): + yield 'data: {"jsonrpc":"2.0","id":"req-1","result":{"taskId":"task-1","status":{"state":"completed"}}}' + + mock_resp = AsyncMock() + mock_resp.is_success = True + mock_resp.aiter_lines = fake_aiter_lines + mock_resp.aclose = AsyncMock() + + mock_async_client = MagicMock() + mock_async_client.build_request = MagicMock(return_value=MagicMock()) + mock_async_client.send = AsyncMock(return_value=mock_resp) + + mock_handler = MagicMock() + mock_handler.client = mock_async_client + mock_handler.post = AsyncMock() + + async def _passthrough_iterator(response, **kwargs): + async for chunk in response: + yield chunk + + mock_proxy_logging = MagicMock() + mock_proxy_logging.pre_call_hook = AsyncMock( + side_effect=lambda user_api_key_dict, data, call_type: data + ) + mock_proxy_logging.async_post_call_streaming_iterator_hook = _passthrough_iterator + mock_proxy_logging.post_call_failure_hook = AsyncMock(return_value=None) + + with ExitStack() as stack: + for p in _base_patches(agent): + stack.enter_context(p) + stack.enter_context( + patch( + "litellm.llms.custom_httpx.http_handler.get_async_httpx_client", + return_value=mock_handler, + ) + ) + stack.enter_context( + patch( + "litellm.proxy.proxy_server.proxy_logging_obj", + mock_proxy_logging, + ) + ) + + from litellm.proxy.agent_endpoints.a2a_endpoints import invoke_agent_a2a + + response = await invoke_agent_a2a( + agent_id="test-agent", + request=mock_request, + fastapi_response=MagicMock(), + user_api_key_dict=user_api_key_dict, + ) + + assert response.media_type == "text/event-stream" + async for _ in response.body_iterator: + pass + + mock_proxy_logging.pre_call_hook.assert_awaited_once() + call_kwargs = mock_proxy_logging.pre_call_hook.await_args.kwargs + assert call_kwargs.get("call_type") == "asend_message" + assert call_kwargs.get("user_api_key_dict") == user_api_key_dict + + +@pytest.mark.asyncio +async def test_subscribe_to_task_runs_post_call_streaming_guardrail(): + """tasks/resubscribe must route streamed events through the post-call + streaming hook so output guardrails configured on the agent inspect the + streamed task content. Regression: the SSE path previously returned the raw + upstream stream and bypassed guardrails entirely.""" + import litellm + from litellm.integrations.custom_guardrail import CustomGuardrail + from litellm.proxy._types import UserAPIKeyAuth + + inspected: list = [] + + class _RecordingGuardrail(CustomGuardrail): + async def async_post_call_streaming_hook(self, user_api_key_dict, response): + inspected.append(response) + return response + + guardrail = _RecordingGuardrail( + guardrail_name="record-a2a", default_on=True, event_hook="post_call" + ) + + agent = _make_agent_mock() + mock_request = _make_request_mock("tasks/resubscribe", {"id": "task-1"}) + user_api_key_dict = UserAPIKeyAuth(api_key="sk-test", user_id="u1", team_id="t1") + + async def fake_aiter_lines(): + yield ( + 'data: {"jsonrpc":"2.0","id":"req-1","result":' + '{"kind":"message","parts":[{"kind":"text","text":"resubscribe-secret"}]}}' + ) + + mock_resp = AsyncMock() + mock_resp.is_success = True + mock_resp.aiter_lines = fake_aiter_lines + mock_resp.aclose = AsyncMock() + + mock_async_client = MagicMock() + mock_async_client.build_request = MagicMock(return_value=MagicMock()) + mock_async_client.send = AsyncMock(return_value=mock_resp) + + mock_handler = MagicMock() + mock_handler.client = mock_async_client + mock_handler.post = AsyncMock() + + with ExitStack() as stack: + for p in _base_patches(agent): + stack.enter_context(p) + stack.enter_context( + patch( + "litellm.llms.custom_httpx.http_handler.get_async_httpx_client", + return_value=mock_handler, + ) + ) + stack.enter_context(patch.object(litellm, "callbacks", [guardrail])) + + from litellm.proxy.agent_endpoints.a2a_endpoints import invoke_agent_a2a + + response = await invoke_agent_a2a( + agent_id="test-agent", + request=mock_request, + fastapi_response=MagicMock(), + user_api_key_dict=user_api_key_dict, + ) + + assert response.media_type == "text/event-stream" + async for _ in response.body_iterator: + pass + + assert any("resubscribe-secret" in str(r) for r in inspected), ( + "tasks/resubscribe streamed content was not passed to the post-call " + "streaming guardrail hook" + ) + + +@pytest.mark.asyncio +async def test_task_method_failure_hook_uses_enriched_request_data(): + from litellm.proxy._types import UserAPIKeyAuth + + agent = _make_agent_mock() + mock_request = _make_request_mock("tasks/get", {"id": "task-1"}) + user_api_key_dict = UserAPIKeyAuth(api_key="sk-test", user_id="u1", team_id="t1") + + async def add_proxy_data_copy(data, **kwargs): + enriched = dict(data) + enriched["proxy_server_request"] = { + "url": "http://localhost:4000", + "method": "POST", + "headers": {}, + "body": {}, + } + enriched.setdefault("metadata", {}) + return enriched + + mock_handler = MagicMock() + mock_handler.post = AsyncMock(side_effect=RuntimeError("upstream failed")) + + mock_proxy_logging = MagicMock() + mock_proxy_logging.pre_call_hook = AsyncMock( + side_effect=lambda user_api_key_dict, data, call_type: data + ) + mock_proxy_logging.post_call_failure_hook = AsyncMock(return_value=None) + + with ExitStack() as stack: + for p in _base_patches(agent): + stack.enter_context(p) + stack.enter_context( + patch( + "litellm.proxy.common_request_processing.add_litellm_data_to_request", + new=AsyncMock(side_effect=add_proxy_data_copy), + ) + ) + stack.enter_context( + patch( + "litellm.llms.custom_httpx.http_handler.get_async_httpx_client", + return_value=mock_handler, + ) + ) + stack.enter_context( + patch( + "litellm.proxy.proxy_server.proxy_logging_obj", + mock_proxy_logging, + ) + ) + + from litellm.proxy.agent_endpoints.a2a_endpoints import invoke_agent_a2a + + response = await invoke_agent_a2a( + agent_id="test-agent", + request=mock_request, + fastapi_response=MagicMock(), + user_api_key_dict=user_api_key_dict, + ) + + body = json.loads(response.body.decode()) + assert body["error"]["code"] == -32603 + failure_data = mock_proxy_logging.post_call_failure_hook.await_args.kwargs[ + "request_data" + ] + assert failure_data.get("litellm_call_id") + assert failure_data.get("agent_id") == "test-agent" + + +@pytest.mark.asyncio +async def test_get_extended_agent_card_rewrites_url(): + from litellm.proxy._types import UserAPIKeyAuth + + agent = _make_agent_mock() + mock_request = _make_request_mock("GetExtendedAgentCard", {}) + mock_request.base_url = "http://localhost:4000/" + user_api_key_dict = UserAPIKeyAuth(api_key="sk-test", user_id="u1", team_id="t1") + + upstream_card = { + "name": "Test Agent", + "url": "http://backend-agent:10001", + "description": "A test agent", + } + upstream_response = {"jsonrpc": "2.0", "id": "req-1", "result": upstream_card} + + mock_http_response = MagicMock() + mock_http_response.json.return_value = upstream_response + mock_http_response.is_success = True + mock_http_response.raise_for_status = MagicMock() + + mock_handler = MagicMock() + mock_handler.post = AsyncMock(return_value=mock_http_response) + mock_handler.client = MagicMock() + + with ExitStack() as stack: + for p in _base_patches(agent): + stack.enter_context(p) + stack.enter_context( + patch( + "litellm.llms.custom_httpx.http_handler.get_async_httpx_client", + return_value=mock_handler, + ) + ) + + from litellm.proxy.agent_endpoints.a2a_endpoints import invoke_agent_a2a + + response = await invoke_agent_a2a( + agent_id="test-agent", + request=mock_request, + fastapi_response=MagicMock(), + user_api_key_dict=user_api_key_dict, + ) + + body = json.loads(response.body.decode()) + assert body["result"]["url"] == "http://localhost:4000/a2a/test-agent" + assert body["result"]["name"] == "Test Agent" + + +@pytest.mark.asyncio +async def test_unknown_method_returns_jsonrpc_error(): + from litellm.proxy._types import UserAPIKeyAuth + + agent = _make_agent_mock() + mock_request = _make_request_mock("SomeUnknownMethod", {}) + user_api_key_dict = UserAPIKeyAuth(api_key="sk-test", user_id="u1", team_id="t1") + + with ExitStack() as stack: + for p in _base_patches(agent): + stack.enter_context(p) + + from litellm.proxy.agent_endpoints.a2a_endpoints import invoke_agent_a2a + + response = await invoke_agent_a2a( + agent_id="test-agent", + request=mock_request, + fastapi_response=MagicMock(), + user_api_key_dict=user_api_key_dict, + ) + + body = json.loads(response.body.decode()) + assert body["error"]["code"] == -32601 + assert "SomeUnknownMethod" in body["error"]["message"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "pascal_method,expected_wire_method", + [ + ("GetTask", "tasks/get"), + ("ListTasks", "tasks/list"), + ("CancelTask", "tasks/cancel"), + ("SubscribeToTask", "tasks/resubscribe"), + ("CreateTaskPushNotificationConfig", "tasks/pushNotificationConfig/set"), + ("GetTaskPushNotificationConfig", "tasks/pushNotificationConfig/get"), + ("ListTaskPushNotificationConfigs", "tasks/pushNotificationConfig/list"), + ("DeleteTaskPushNotificationConfig", "tasks/pushNotificationConfig/delete"), + ("GetExtendedAgentCard", "agent/getAuthenticatedExtendedCard"), + ], +) +async def test_pascal_method_names_normalize_to_wire_format( + pascal_method: str, expected_wire_method: str +): + from litellm.proxy._types import UserAPIKeyAuth + + agent = _make_agent_mock() + mock_request = _make_request_mock(pascal_method, {"id": "task-1"}) + user_api_key_dict = UserAPIKeyAuth(api_key="sk-test", user_id="u1", team_id="t1") + + upstream_response = {"jsonrpc": "2.0", "id": "req-1", "result": {"id": "task-1"}} + mock_http_response = MagicMock() + mock_http_response.json.return_value = upstream_response + mock_http_response.is_success = True + mock_http_response.raise_for_status = MagicMock() + + async def _empty_aiter_lines(): + return + yield # make it an async generator + + mock_sse_resp = AsyncMock() + mock_sse_resp.is_success = True + mock_sse_resp.aiter_lines = _empty_aiter_lines + mock_sse_resp.aclose = AsyncMock() + + mock_async_client = MagicMock() + mock_async_client.build_request = MagicMock(return_value=MagicMock()) + mock_async_client.send = AsyncMock(return_value=mock_sse_resp) + + mock_handler = MagicMock() + mock_handler.post = AsyncMock(return_value=mock_http_response) + mock_handler.client = mock_async_client + + with ExitStack() as stack: + for p in _base_patches(agent): + stack.enter_context(p) + stack.enter_context( + patch( + "litellm.llms.custom_httpx.http_handler.get_async_httpx_client", + return_value=mock_handler, + ) + ) + + from litellm.proxy.agent_endpoints.a2a_endpoints import invoke_agent_a2a + + response = await invoke_agent_a2a( + agent_id="test-agent", + request=mock_request, + fastapi_response=MagicMock(), + user_api_key_dict=user_api_key_dict, + ) + + if expected_wire_method == "tasks/resubscribe": + assert response.media_type == "text/event-stream" + async for _ in response.body_iterator: + pass + else: + body = json.loads(response.body.decode()) + assert "error" not in body, f"Got error: {body}" + + if expected_wire_method != "tasks/resubscribe": + posted = mock_handler.post.call_args + forwarded_body = posted.kwargs.get("json") or posted.args[1] + assert forwarded_body["method"] == expected_wire_method, ( + f"Expected '{expected_wire_method}' forwarded for PascalCase '{pascal_method}', " + f"but got '{forwarded_body['method']}'" + ) + + +@pytest.mark.asyncio +async def test_task_method_upstream_jsonrpc_error_on_http_4xx_is_relayed(): + """When upstream returns HTTP 4xx with a JSON-RPC error body, the error body + must be relayed to the client unchanged, not replaced with a generic string.""" + from litellm.proxy._types import UserAPIKeyAuth + + agent = _make_agent_mock() + mock_request = _make_request_mock("tasks/get", {"id": "nonexistent"}) + user_api_key_dict = UserAPIKeyAuth(api_key="sk-test", user_id="u1", team_id="t1") + + upstream_error = { + "jsonrpc": "2.0", + "id": "req-1", + "error": {"code": -32001, "message": "Task not found"}, + } + + mock_http_response = MagicMock() + mock_http_response.json.return_value = upstream_error + mock_http_response.is_success = False + mock_http_response.raise_for_status = MagicMock( + side_effect=Exception("404 Not Found") + ) + + mock_handler = MagicMock() + mock_handler.post = AsyncMock(return_value=mock_http_response) + mock_handler.client = MagicMock() + + with ExitStack() as stack: + for p in _base_patches(agent): + stack.enter_context(p) + stack.enter_context( + patch( + "litellm.llms.custom_httpx.http_handler.get_async_httpx_client", + return_value=mock_handler, + ) + ) + + from litellm.proxy.agent_endpoints.a2a_endpoints import invoke_agent_a2a + + response = await invoke_agent_a2a( + agent_id="test-agent", + request=mock_request, + fastapi_response=MagicMock(), + user_api_key_dict=user_api_key_dict, + ) + + body = json.loads(response.body.decode()) + assert body["error"]["code"] == -32001 + assert body["error"]["message"] == "Task not found" + + +@pytest.mark.asyncio +async def test_subscribe_to_task_upstream_error_yields_jsonrpc_error_event(): + """When upstream returns a non-2xx response for tasks/resubscribe, the SSE + stream must yield a JSON-RPC error event instead of silently breaking.""" + from litellm.proxy._types import UserAPIKeyAuth + + agent = _make_agent_mock() + mock_request = _make_request_mock("tasks/resubscribe", {"id": "task-1"}) + user_api_key_dict = UserAPIKeyAuth(api_key="sk-test", user_id="u1", team_id="t1") + + mock_resp = AsyncMock() + mock_resp.is_success = False + mock_resp.status_code = 404 + mock_resp.reason_phrase = "Not Found" + mock_resp.aread = AsyncMock( + return_value=b'{"jsonrpc":"2.0","error":{"code":-32001,"message":"Task not found"}}' + ) + mock_resp.aclose = AsyncMock() + + mock_async_client = MagicMock() + mock_async_client.build_request = MagicMock(return_value=MagicMock()) + mock_async_client.send = AsyncMock(return_value=mock_resp) + + mock_handler = MagicMock() + mock_handler.client = mock_async_client + mock_handler.post = AsyncMock() + + chunks = [] + with ExitStack() as stack: + for p in _base_patches(agent): + stack.enter_context(p) + stack.enter_context( + patch( + "litellm.llms.custom_httpx.http_handler.get_async_httpx_client", + return_value=mock_handler, + ) + ) + + from litellm.proxy.agent_endpoints.a2a_endpoints import invoke_agent_a2a + + response = await invoke_agent_a2a( + agent_id="test-agent", + request=mock_request, + fastapi_response=MagicMock(), + user_api_key_dict=user_api_key_dict, + ) + + assert response.media_type == "text/event-stream" + async for chunk in response.body_iterator: + chunks.append(chunk) + + full = "".join(chunks) + body = json.loads(full.removeprefix("data: ").strip()) + assert body["id"] == "req-1" + assert body["error"]["code"] == -32001 + assert body["error"]["message"] == "Task not found" + + +@pytest.mark.asyncio +async def test_forward_jsonrpc_sse_fallback_error_uses_jsonrpc_error_code(): + mock_resp = AsyncMock() + mock_resp.is_success = False + mock_resp.status_code = 503 + mock_resp.reason_phrase = "Service Unavailable" + mock_resp.aread = AsyncMock(return_value=b"upstream unavailable") + mock_resp.aclose = AsyncMock() + + mock_async_client = MagicMock() + mock_async_client.build_request = MagicMock(return_value=MagicMock()) + mock_async_client.send = AsyncMock(return_value=mock_resp) + + mock_handler = MagicMock() + mock_handler.client = mock_async_client + + with patch( + "litellm.llms.custom_httpx.http_handler.get_async_httpx_client", + return_value=mock_handler, + ): + from litellm.proxy.agent_endpoints.a2a_endpoints import _forward_jsonrpc_sse + + response = await _forward_jsonrpc_sse( + agent_url="http://backend-agent:10001", + body={"jsonrpc": "2.0", "id": "req-1", "method": "tasks/resubscribe"}, + request_id="req-1", + ) + + chunks = [] + async for chunk in response.body_iterator: + chunks.append(chunk) + + body = json.loads("".join(chunks).removeprefix("data: ").strip()) + assert body["error"]["code"] == -32603 + assert body["error"]["message"] == "Service Unavailable" + + +@pytest.mark.asyncio +async def test_task_methods_forward_caller_identity_headers(): + """Task operations must forward X-LiteLLM-User-Id and X-LiteLLM-Team-Id so the + upstream agent can scope resources to the authenticated caller.""" + from litellm.proxy._types import UserAPIKeyAuth + + upstream_response = { + "jsonrpc": "2.0", + "id": "req-1", + "result": {"id": "task-1", "status": {"state": "completed"}}, + } + agent = _make_agent_mock() + mock_request = _make_request_mock("tasks/get", {"id": "task-1"}) + user_api_key_dict = UserAPIKeyAuth( + api_key="sk-test", user_id="user-abc", team_id="team-xyz" + ) + + mock_http_response = MagicMock() + mock_http_response.json.return_value = upstream_response + mock_http_response.is_success = True + + mock_handler = MagicMock() + mock_handler.post = AsyncMock(return_value=mock_http_response) + + with ExitStack() as stack: + for p in _base_patches(agent): + stack.enter_context(p) + stack.enter_context( + patch( + "litellm.llms.custom_httpx.http_handler.get_async_httpx_client", + return_value=mock_handler, + ) + ) + + from litellm.proxy.agent_endpoints.a2a_endpoints import invoke_agent_a2a + + await invoke_agent_a2a( + agent_id="test-agent", + request=mock_request, + fastapi_response=MagicMock(), + user_api_key_dict=user_api_key_dict, + ) + + posted_headers = mock_handler.post.call_args.kwargs.get("headers") or {} + assert posted_headers.get("X-LiteLLM-User-Id") == "user-abc" + assert posted_headers.get("X-LiteLLM-Team-Id") == "team-xyz" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("method", ["tasks/get", "tasks/resubscribe"]) +async def test_task_methods_forward_trace_header(method: str): + from litellm.proxy._types import UserAPIKeyAuth + + agent = _make_agent_mock() + mock_request = _make_request_mock(method, {"id": "task-1"}) + user_api_key_dict = UserAPIKeyAuth(api_key="sk-test", user_id="u1", team_id="t1") + + async def add_proxy_data_with_trace(data, **kwargs): + data = await _add_proxy_data(data, **kwargs) + data["litellm_trace_id"] = "trace-123" + return data + + upstream_response = { + "jsonrpc": "2.0", + "id": "req-1", + "result": {"id": "task-1", "status": {"state": "completed"}}, + } + + mock_http_response = MagicMock() + mock_http_response.json.return_value = upstream_response + mock_http_response.is_success = True + + async def fake_aiter_lines(): + yield 'data: {"jsonrpc":"2.0","id":"req-1","result":{"taskId":"task-1"}}' + + mock_resp = AsyncMock() + mock_resp.is_success = True + mock_resp.aiter_lines = fake_aiter_lines + mock_resp.aclose = AsyncMock() + + mock_async_client = MagicMock() + mock_async_client.build_request = MagicMock(return_value=MagicMock()) + mock_async_client.send = AsyncMock(return_value=mock_resp) + + mock_handler = MagicMock() + mock_handler.post = AsyncMock(return_value=mock_http_response) + mock_handler.client = mock_async_client + + with ExitStack() as stack: + for p in _base_patches(agent): + stack.enter_context(p) + stack.enter_context( + patch( + "litellm.proxy.common_request_processing.add_litellm_data_to_request", + new=AsyncMock(side_effect=add_proxy_data_with_trace), + ) + ) + stack.enter_context( + patch( + "litellm.llms.custom_httpx.http_handler.get_async_httpx_client", + return_value=mock_handler, + ) + ) + + from litellm.proxy.agent_endpoints.a2a_endpoints import invoke_agent_a2a + + response = await invoke_agent_a2a( + agent_id="test-agent", + request=mock_request, + fastapi_response=MagicMock(), + user_api_key_dict=user_api_key_dict, + ) + if method == "tasks/resubscribe": + async for _ in response.body_iterator: + pass + + if method == "tasks/resubscribe": + forwarded_headers = mock_async_client.build_request.call_args.kwargs["headers"] + else: + forwarded_headers = mock_handler.post.call_args.kwargs["headers"] + assert forwarded_headers.get("X-LiteLLM-Trace-Id") == "trace-123" + + +@pytest.mark.asyncio +async def test_push_notification_config_set_rejects_http_url(): + """tasks/pushNotificationConfig/set must reject non-HTTPS callback URLs to prevent SSRF.""" + from fastapi import HTTPException + + from litellm.proxy._types import UserAPIKeyAuth + + agent = _make_agent_mock() + mock_request = _make_request_mock( + "tasks/pushNotificationConfig/set", + {"taskId": "task-1", "url": "http://internal-webhook.example.com/hook"}, + ) + user_api_key_dict = UserAPIKeyAuth(api_key="sk-test", user_id="u1", team_id="t1") + + with ExitStack() as stack: + for p in _base_patches(agent): + stack.enter_context(p) + + from litellm.proxy.agent_endpoints.a2a_endpoints import invoke_agent_a2a + + with pytest.raises(HTTPException) as exc_info: + await invoke_agent_a2a( + agent_id="test-agent", + request=mock_request, + fastapi_response=MagicMock(), + user_api_key_dict=user_api_key_dict, + ) + + assert exc_info.value.status_code == 400 + assert "HTTPS" in exc_info.value.detail + + +@pytest.mark.asyncio +async def test_push_notification_config_set_rejects_private_ip(): + """tasks/pushNotificationConfig/set must reject callback URLs pointing to private IP ranges.""" + from fastapi import HTTPException + + from litellm.proxy._types import UserAPIKeyAuth + + agent = _make_agent_mock() + mock_request = _make_request_mock( + "tasks/pushNotificationConfig/set", + {"taskId": "task-1", "url": "https://192.168.1.100/hook"}, + ) + user_api_key_dict = UserAPIKeyAuth(api_key="sk-test", user_id="u1", team_id="t1") + + with ExitStack() as stack: + for p in _base_patches(agent): + stack.enter_context(p) + + from litellm.proxy.agent_endpoints.a2a_endpoints import invoke_agent_a2a + + with pytest.raises(HTTPException) as exc_info: + await invoke_agent_a2a( + agent_id="test-agent", + request=mock_request, + fastapi_response=MagicMock(), + user_api_key_dict=user_api_key_dict, + ) + + assert exc_info.value.status_code == 400 + assert "blocked address" in exc_info.value.detail.lower() + + +@pytest.mark.asyncio +async def test_push_notification_config_set_validates_nested_url_when_top_level_present(): + """A safe top-level params.url must not let a private pushNotificationConfig.url bypass SSRF checks. + + Both URL-bearing fields are forwarded to the agent, so both must be validated independently. + """ + from fastapi import HTTPException + + from litellm.proxy._types import UserAPIKeyAuth + + agent = _make_agent_mock() + mock_request = _make_request_mock( + "tasks/pushNotificationConfig/set", + { + "taskId": "task-1", + "url": "https://1.1.1.1/hook", + "pushNotificationConfig": {"url": "https://192.168.1.100/hook"}, + }, + ) + user_api_key_dict = UserAPIKeyAuth(api_key="sk-test", user_id="u1", team_id="t1") + + with ExitStack() as stack: + for p in _base_patches(agent): + stack.enter_context(p) + + from litellm.proxy.agent_endpoints.a2a_endpoints import invoke_agent_a2a + + with pytest.raises(HTTPException) as exc_info: + await invoke_agent_a2a( + agent_id="test-agent", + request=mock_request, + fastapi_response=MagicMock(), + user_api_key_dict=user_api_key_dict, + ) + + assert exc_info.value.status_code == 400 + assert "blocked address" in exc_info.value.detail.lower() + + +def test_push_notification_config_set_rejects_private_dns_resolution(): + from fastapi import HTTPException + + from litellm.proxy.agent_endpoints.a2a_endpoints import ( + _validate_push_notification_url, + ) + + with patch( + "litellm.litellm_core_utils.url_utils.socket.getaddrinfo", + return_value=[ + ( + socket.AF_INET, + socket.SOCK_STREAM, + socket.IPPROTO_TCP, + "", + ("10.0.0.5", 443), + ) + ], + ): + with pytest.raises(HTTPException) as exc_info: + _validate_push_notification_url("https://webhook.example.com/hook") + + assert exc_info.value.status_code == 400 + assert "blocked address" in exc_info.value.detail.lower() + + +@pytest.mark.asyncio +async def test_push_notification_config_set_rejects_null_push_config(): + from fastapi import HTTPException + + from litellm.proxy._types import UserAPIKeyAuth + + agent = _make_agent_mock() + mock_request = _make_request_mock( + "tasks/pushNotificationConfig/set", + {"taskId": "task-1", "pushNotificationConfig": None}, + ) + user_api_key_dict = UserAPIKeyAuth(api_key="sk-test", user_id="u1", team_id="t1") + + with ExitStack() as stack: + for p in _base_patches(agent): + stack.enter_context(p) + + from litellm.proxy.agent_endpoints.a2a_endpoints import invoke_agent_a2a + + with pytest.raises(HTTPException) as exc_info: + await invoke_agent_a2a( + agent_id="test-agent", + request=mock_request, + fastapi_response=MagicMock(), + user_api_key_dict=user_api_key_dict, + ) + + assert exc_info.value.status_code == 400 + assert "pushNotificationConfig must be an object" in exc_info.value.detail + + +@pytest.mark.asyncio +async def test_caller_identity_headers_cannot_be_spoofed_via_forwarded_headers(): + """A client must not be able to override X-LiteLLM-User-Id / X-LiteLLM-Team-Id + by including x-a2a--x-litellm-user-id in their request headers. + The authenticated identity must always win.""" + from litellm.proxy._types import UserAPIKeyAuth + + upstream_response = { + "jsonrpc": "2.0", + "id": "req-1", + "result": {"id": "task-1", "status": {"state": "completed"}}, + } + agent = _make_agent_mock() + mock_request = _make_request_mock("tasks/get", {"id": "task-1"}) + mock_request.headers = { + "x-a2a-test-agent-x-litellm-user-id": "attacker-user", + "x-a2a-test-agent-x-litellm-team-id": "attacker-team", + } + user_api_key_dict = UserAPIKeyAuth( + api_key="sk-test", user_id="real-user", team_id="real-team" + ) + + mock_http_response = MagicMock() + mock_http_response.json.return_value = upstream_response + mock_http_response.is_success = True + + mock_handler = MagicMock() + mock_handler.post = AsyncMock(return_value=mock_http_response) + + with ExitStack() as stack: + for p in _base_patches(agent): + stack.enter_context(p) + stack.enter_context( + patch( + "litellm.llms.custom_httpx.http_handler.get_async_httpx_client", + return_value=mock_handler, + ) + ) + + from litellm.proxy.agent_endpoints.a2a_endpoints import invoke_agent_a2a + + await invoke_agent_a2a( + agent_id="test-agent", + request=mock_request, + fastapi_response=MagicMock(), + user_api_key_dict=user_api_key_dict, + ) + + posted_headers = mock_handler.post.call_args.kwargs.get("headers") or {} + assert ( + posted_headers.get("X-LiteLLM-User-Id") == "real-user" + ), "authenticated user id must not be overridden by forwarded client headers" + assert ( + posted_headers.get("X-LiteLLM-Team-Id") == "real-team" + ), "authenticated team id must not be overridden by forwarded client headers" From 53a206a179e0976785bdb731ab97ad820cf83bd8 Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 3 Jun 2026 11:26:13 -0700 Subject: [PATCH 5/7] fix(anthropic/adapter): emit thinking block for reasoning_content-only streaming chunks (#29600) * fix(anthropic/adapter): open thinking block for reasoning_content-only streaming chunks The /v1/messages streaming content-block classifier (_translate_streaming_openai_chunk_to_anthropic_content_block) only recognized thinking_blocks. OpenAI-compatible reasoning backends (vLLM/SGLang reasoning parsers: DeepSeek-R1, Qwen3, gpt-oss, ...) populate reasoning_content with thinking_blocks=None, so the classifier fell through to a text block. The delta translator already emits thinking_delta for reasoning_content, so those deltas landed inside a text block and Anthropic streaming clients (Claude Code, SDK .stream()) silently dropped the chain-of-thought. Mirror the reasoning_content fallback already present in the non-stream translator and the streaming delta translator so the classifier opens a thinking block. Adds a focused regression test. * fix(anthropic/adapter): reach reasoning_content branch when thinking_blocks attr is absent Delta deletes the thinking_blocks attribute when unset, so the prior nested check was unreachable for reasoning-only chunks (vLLM/SGLang). Make it a sibling elif so the content block is classified as thinking. * test(proxy): stop component-allowlist test leaking DATABASE_URL into xdist peers The component-allowlist test pins throwaway DATABASE_URL/LITELLM_MASTER_KEY values at import time via os.environ so importing proxy_server doesn't need a live database. Those values persisted for the whole pytest-xdist worker, so a sibling test sharing the worker (test_key_rotation_e2e's DB-backed E2E case) saw the leaked sqlite DATABASE_URL, treated it as an available database instead of skipping, and the Prisma engine rejected the non-postgres URL (P1012 -> httpx.ConnectError). Restore the prior environment after the import so the throwaway values never escape the module. --------- Co-authored-by: Tai An --- .../adapters/transformation.py | 11 ++++++ ...al_pass_through_adapters_transformation.py | 38 +++++++++++++++++++ .../proxy/test_component_allowlists.py | 20 ++++++++-- 3 files changed, 66 insertions(+), 3 deletions(-) diff --git a/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py b/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py index 02e0c562654..150f056dc81 100644 --- a/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py +++ b/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py @@ -1510,6 +1510,17 @@ class LiteLLMAnthropicMessagesAdapter: return "thinking", ChatCompletionThinkingBlock( type="thinking", thinking=thinking, signature=signature ) + # OpenAI-compatible reasoning backends (e.g. vLLM/SGLang reasoning + # parsers) populate ``reasoning_content`` without ``thinking_blocks``. + # ``Delta`` deletes the ``thinking_blocks`` attribute when unset, so the + # branch above is skipped entirely; open a ``thinking`` block here so the + # matching ``thinking_delta`` stream is not emitted into a text block. + elif isinstance(choice, StreamingChoices) and getattr( + choice.delta, "reasoning_content", None + ): + return "thinking", ChatCompletionThinkingBlock( + type="thinking", thinking="", signature="" + ) return "text", TextBlock(type="text", text="") diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py index 74e1e17e6d7..a81261d5ffd 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py @@ -170,6 +170,44 @@ def test_translate_streaming_openai_chunk_to_anthropic_thinking_content_block(): } +def test_translate_streaming_openai_chunk_to_anthropic_reasoning_content_only_content_block(): + """OpenAI-compatible reasoning backends (vLLM/SGLang) emit ``reasoning_content`` + without ``thinking_blocks``. The content-block classifier must still open a + ``thinking`` block so the matching ``thinking_delta`` stream is not emitted + inside a text block (which silently drops chain-of-thought for /v1/messages + streaming clients).""" + choices = [ + StreamingChoices( + finish_reason=None, + index=0, + delta=Delta( + reasoning_content="Let me think", + thinking_blocks=None, + content=None, + role="assistant", + function_call=None, + tool_calls=None, + audio=None, + ), + logprobs=None, + ) + ] + + ( + block_type, + content_block_start, + ) = LiteLLMAnthropicMessagesAdapter()._translate_streaming_openai_chunk_to_anthropic_content_block( + choices=choices + ) + + assert block_type == "thinking" + assert content_block_start == { + "type": "thinking", + "thinking": "", + "signature": "", + } + + def test_translate_streaming_openai_chunk_to_anthropic_thinking_signature_block(): choices = [ StreamingChoices( diff --git a/tests/test_litellm/proxy/test_component_allowlists.py b/tests/test_litellm/proxy/test_component_allowlists.py index d20e1781169..ad25856b972 100644 --- a/tests/test_litellm/proxy/test_component_allowlists.py +++ b/tests/test_litellm/proxy/test_component_allowlists.py @@ -23,9 +23,17 @@ import sys # Importing ``litellm.proxy.proxy_server`` runs its module-level setup, which # reads ``DATABASE_URL`` (Prisma) and ``LITELLM_MASTER_KEY``. Tier-zero CI # runners don't set these. We pin throwaway values before the import so the -# test never depends on a live database or master key. -os.environ.setdefault("DATABASE_URL", "sqlite:///:memory:") -os.environ.setdefault("LITELLM_MASTER_KEY", "sk-test-component-allowlist") +# test never depends on a live database or master key, then restore the prior +# environment so the throwaway values don't leak into sibling tests sharing the +# xdist worker (a leaked non-postgres ``DATABASE_URL`` makes DB-backed tests +# treat a phantom database as available instead of skipping). +_THROWAWAY_ENV = { + "DATABASE_URL": "sqlite:///:memory:", + "LITELLM_MASTER_KEY": "sk-test-component-allowlist", +} +_PRE_EXISTING_ENV = {key: os.environ.get(key) for key in _THROWAWAY_ENV} +for _key, _value in _THROWAWAY_ENV.items(): + os.environ.setdefault(_key, _value) from fastapi.routing import Mount @@ -38,6 +46,12 @@ from backend.routes.allowlist import BACKEND_EXACT_PATHS, BACKEND_PATH_PREFIXES from gateway.routes.allowlist import GATEWAY_EXACT_PATHS, GATEWAY_PATH_PREFIXES from litellm.proxy.proxy_server import app +for _key, _previous in _PRE_EXISTING_ENV.items(): + if _previous is None: + os.environ.pop(_key, None) + else: + os.environ[_key] = _previous + def _component_paths(routes, exact_paths, path_prefixes) -> set[str]: """Reproduce ``gateway.main._is_gateway_route`` / ``backend.main._is_backend_route``.""" From 34293fa80af6ba22da55d6be66ef05a35187a2d1 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Wed, 3 Jun 2026 11:28:08 -0700 Subject: [PATCH 6/7] ci: reproduce default-Windows wheel install to guard MAX_PATH (#29597) * ci: reproduce default-Windows wheel install to guard MAX_PATH The existing using_litellm_on_windows job installs the project with `uv sync`, an editable source install that never copies package files into a deep site-packages path, so it cannot see the 260-char MAX_PATH overflow that breaks `pip install litellm` on default Windows. The content-filter benchmark fixtures have hit that limit three times (#21941, #22039, #29536), each caught only after release. This adds a guard to the same job that builds the wheel and installs it the way an end user would: into a venv whose site-packages prefix is padded to a realistic worst-case Windows length (~100 chars), then asserts the install completes and litellm imports. Any packaged path long enough to bust MAX_PATH at that prefix is reported up front, so the check is deterministic regardless of the runner's long-path setting, while the real install also covers failure modes a length heuristic cannot (half-unpacked packages, reserved names, case collisions). This commit is the guard only; on the current tree it correctly fails because nine fixtures still exceed the limit. The rename that brings them back under it follows on this branch. * fix(packaging): shorten content-filter benchmark fixtures under MAX_PATH The 10 content-filter benchmark result fixtures used the legacy block_{topic}_-_contentfilter_({yaml}).json naming, up to 176 chars inside the wheel, which busts the Windows 260-char MAX_PATH limit once extracted under a realistic site-packages prefix and aborts `pip install litellm` on default Windows. Rename them to the short {topic}_cf.json scheme that _save_confusion_results already emits today (it splits the label on the em-dash and writes f"{topic}_cf"), matching the insults_cf.json and investment_cf.json files fixed earlier. Re-running the eval suite now regenerates these same short names rather than recreating the long ones. This drops the longest packaged path from 176 to 128, so the guard added in the previous commit goes from red to green with a 32-char margin. * test(windows): tidy MAX_PATH guard per review Close the wheel zip via a context manager rather than leaning on refcount collection, and select the wheel under dist/ by newest mtime so a stale artifact from an earlier build cannot be tested instead of the one just produced. Also pin down the venv-depth formula with a short note: the +2 is the separator joining the venv root to "Lib" plus the trailing separator before the entry, which lands the simulated site-packages prefix at exactly 100 chars. --- .circleci/config.yml | 9 ++- ....yaml).json => age_discrimination_cf.json} | 0 ...ml).json => claims_fraud_coaching_cf.json} | 0 ...ml).json => claims_medical_advice_cf.json} | 0 ...ml).json => claims_phi_disclosure_cf.json} | 0 ....json => claims_prior_auth_gaming_cf.json} | 0 ...l).json => claims_system_override_cf.json} | 0 ...json => disability_discrimination_cf.json} | 0 ...ml).json => gender_discrimination_cf.json} | 0 ...).json => military_discrimination_cf.json} | 0 ...).json => religion_discrimination_cf.json} | 0 .../check_windows_wheel_install.py | 76 +++++++++++++++++++ .../test_check_windows_wheel_install.py | 36 +++++++++ 13 files changed, 120 insertions(+), 1 deletion(-) rename litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/guardrail_benchmarks/results/{block_age_discrimination_-_contentfilter_(age_discrimination.yaml).json => age_discrimination_cf.json} (100%) rename litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/guardrail_benchmarks/results/{block_claims_fraud_coaching_-_contentfilter_(claims_fraud_coaching.yaml).json => claims_fraud_coaching_cf.json} (100%) rename litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/guardrail_benchmarks/results/{block_claims_medical_advice_-_contentfilter_(claims_medical_advice.yaml).json => claims_medical_advice_cf.json} (100%) rename litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/guardrail_benchmarks/results/{block_claims_phi_disclosure_-_contentfilter_(claims_phi_disclosure.yaml).json => claims_phi_disclosure_cf.json} (100%) rename litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/guardrail_benchmarks/results/{block_claims_prior_auth_gaming_-_contentfilter_(claims_prior_auth_gaming.yaml).json => claims_prior_auth_gaming_cf.json} (100%) rename litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/guardrail_benchmarks/results/{block_claims_system_override_-_contentfilter_(claims_system_override.yaml).json => claims_system_override_cf.json} (100%) rename litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/guardrail_benchmarks/results/{block_disability_discrimination_-_contentfilter_(disability.yaml).json => disability_discrimination_cf.json} (100%) rename litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/guardrail_benchmarks/results/{block_gender_discrimination_-_contentfilter_(gender_sexual_orientation.yaml).json => gender_discrimination_cf.json} (100%) rename litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/guardrail_benchmarks/results/{block_military_discrimination_-_contentfilter_(military_status.yaml).json => military_discrimination_cf.json} (100%) rename litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/guardrail_benchmarks/results/{block_religion_discrimination_-_contentfilter_(religion.yaml).json => religion_discrimination_cf.json} (100%) create mode 100644 tests/windows_tests/check_windows_wheel_install.py create mode 100644 tests/windows_tests/test_check_windows_wheel_install.py diff --git a/.circleci/config.yml b/.circleci/config.yml index 23ef423039f..f5a8fe77a25 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -182,7 +182,14 @@ jobs: - run: name: Run Windows-specific test command: | - uv run --no-sync python -m pytest tests/windows_tests/test_litellm_on_windows.py -v + uv run --no-sync python -m pytest tests/windows_tests/ -v + - run: + name: Guard against MAX_PATH-busting packaged wheel paths + environment: + UV_HTTP_TIMEOUT: "300" + command: | + uv build --wheel --out-dir dist + uv run --no-sync python tests/windows_tests/check_windows_wheel_install.py local_testing_part1: docker: diff --git a/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/guardrail_benchmarks/results/block_age_discrimination_-_contentfilter_(age_discrimination.yaml).json b/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/guardrail_benchmarks/results/age_discrimination_cf.json similarity index 100% rename from litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/guardrail_benchmarks/results/block_age_discrimination_-_contentfilter_(age_discrimination.yaml).json rename to litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/guardrail_benchmarks/results/age_discrimination_cf.json diff --git a/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/guardrail_benchmarks/results/block_claims_fraud_coaching_-_contentfilter_(claims_fraud_coaching.yaml).json b/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/guardrail_benchmarks/results/claims_fraud_coaching_cf.json similarity index 100% rename from litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/guardrail_benchmarks/results/block_claims_fraud_coaching_-_contentfilter_(claims_fraud_coaching.yaml).json rename to litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/guardrail_benchmarks/results/claims_fraud_coaching_cf.json diff --git a/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/guardrail_benchmarks/results/block_claims_medical_advice_-_contentfilter_(claims_medical_advice.yaml).json b/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/guardrail_benchmarks/results/claims_medical_advice_cf.json similarity index 100% rename from litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/guardrail_benchmarks/results/block_claims_medical_advice_-_contentfilter_(claims_medical_advice.yaml).json rename to litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/guardrail_benchmarks/results/claims_medical_advice_cf.json diff --git a/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/guardrail_benchmarks/results/block_claims_phi_disclosure_-_contentfilter_(claims_phi_disclosure.yaml).json b/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/guardrail_benchmarks/results/claims_phi_disclosure_cf.json similarity index 100% rename from litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/guardrail_benchmarks/results/block_claims_phi_disclosure_-_contentfilter_(claims_phi_disclosure.yaml).json rename to litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/guardrail_benchmarks/results/claims_phi_disclosure_cf.json diff --git a/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/guardrail_benchmarks/results/block_claims_prior_auth_gaming_-_contentfilter_(claims_prior_auth_gaming.yaml).json b/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/guardrail_benchmarks/results/claims_prior_auth_gaming_cf.json similarity index 100% rename from litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/guardrail_benchmarks/results/block_claims_prior_auth_gaming_-_contentfilter_(claims_prior_auth_gaming.yaml).json rename to litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/guardrail_benchmarks/results/claims_prior_auth_gaming_cf.json diff --git a/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/guardrail_benchmarks/results/block_claims_system_override_-_contentfilter_(claims_system_override.yaml).json b/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/guardrail_benchmarks/results/claims_system_override_cf.json similarity index 100% rename from litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/guardrail_benchmarks/results/block_claims_system_override_-_contentfilter_(claims_system_override.yaml).json rename to litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/guardrail_benchmarks/results/claims_system_override_cf.json diff --git a/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/guardrail_benchmarks/results/block_disability_discrimination_-_contentfilter_(disability.yaml).json b/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/guardrail_benchmarks/results/disability_discrimination_cf.json similarity index 100% rename from litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/guardrail_benchmarks/results/block_disability_discrimination_-_contentfilter_(disability.yaml).json rename to litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/guardrail_benchmarks/results/disability_discrimination_cf.json diff --git a/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/guardrail_benchmarks/results/block_gender_discrimination_-_contentfilter_(gender_sexual_orientation.yaml).json b/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/guardrail_benchmarks/results/gender_discrimination_cf.json similarity index 100% rename from litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/guardrail_benchmarks/results/block_gender_discrimination_-_contentfilter_(gender_sexual_orientation.yaml).json rename to litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/guardrail_benchmarks/results/gender_discrimination_cf.json diff --git a/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/guardrail_benchmarks/results/block_military_discrimination_-_contentfilter_(military_status.yaml).json b/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/guardrail_benchmarks/results/military_discrimination_cf.json similarity index 100% rename from litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/guardrail_benchmarks/results/block_military_discrimination_-_contentfilter_(military_status.yaml).json rename to litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/guardrail_benchmarks/results/military_discrimination_cf.json diff --git a/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/guardrail_benchmarks/results/block_religion_discrimination_-_contentfilter_(religion.yaml).json b/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/guardrail_benchmarks/results/religion_discrimination_cf.json similarity index 100% rename from litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/guardrail_benchmarks/results/block_religion_discrimination_-_contentfilter_(religion.yaml).json rename to litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/guardrail_benchmarks/results/religion_discrimination_cf.json diff --git a/tests/windows_tests/check_windows_wheel_install.py b/tests/windows_tests/check_windows_wheel_install.py new file mode 100644 index 00000000000..6dbb9da6288 --- /dev/null +++ b/tests/windows_tests/check_windows_wheel_install.py @@ -0,0 +1,76 @@ +"""Reproduce a default-Windows ``pip install litellm`` to catch the 260-char +MAX_PATH regression that content-filter benchmark fixtures keep reintroducing +(#21941, #22039, #29536). Run after ``uv build --wheel --out-dir dist``. +""" + +import glob +import os +import subprocess +import sys +import zipfile + +MAX_PATH = 260 +# Worst-case Windows site-packages prefix: long profile name + roaming AppData venv. +WORST_CASE_PREFIX = 100 + + +def overlong_install_paths(wheel, prefix_len=WORST_CASE_PREFIX, max_path=MAX_PATH): + with zipfile.ZipFile(wheel) as zf: + names = zf.namelist() + return sorted( + (n for n in names if prefix_len + len(n) > max_path), key=len, reverse=True + ) + + +def _deep_venv_dir(target_prefix=WORST_CASE_PREFIX): + drive = os.path.splitdrive(os.getcwd())[0] or "C:" + root = drive + os.sep + "lmwin" + os.sep + # +2: the sep joining the venv root to "Lib", plus the trailing sep before the entry + suffix = len(os.path.join("Lib", "site-packages")) + 2 + return root + "x" * (target_prefix - suffix - len(root)) + + +def _run(cmd): + print("+ " + subprocess.list2cmdline(cmd), flush=True) + return subprocess.call(cmd) + + +def main(): + wheels = glob.glob(os.path.join("dist", "*.whl")) + if not wheels: + print("::error::no wheel in dist/; run `uv build --wheel --out-dir dist` first") + return 1 + wheel = max(wheels, key=os.path.getmtime) + + offenders = overlong_install_paths(wheel) + if offenders: + print( + f"::error::{len(offenders)} packaged path(s) bust the Windows MAX_PATH limit " + f"at a {WORST_CASE_PREFIX}-char install prefix:" + ) + for n in offenders[:15]: + print(f" on-disk {WORST_CASE_PREFIX + len(n):4} {n}") + return 1 + + venv = _deep_venv_dir() + os.makedirs(os.path.dirname(venv), exist_ok=True) + if _run(["uv", "venv", venv]) != 0: + return 1 + python = os.path.join(venv, "Scripts", "python.exe") + if _run(["uv", "pip", "install", "--python", python, wheel]) != 0: + print( + f"::error::installing {os.path.basename(wheel)} into a deep prefix failed" + ) + return 1 + if _run([python, "-c", "import litellm; import litellm.types.utils"]) != 0: + print("::error::litellm did not import after install (half-unpacked package)") + return 1 + + print( + f"ok: {os.path.basename(wheel)} installs into a worst-case prefix and imports" + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/windows_tests/test_check_windows_wheel_install.py b/tests/windows_tests/test_check_windows_wheel_install.py new file mode 100644 index 00000000000..22a197604ed --- /dev/null +++ b/tests/windows_tests/test_check_windows_wheel_install.py @@ -0,0 +1,36 @@ +import zipfile + +from check_windows_wheel_install import ( + MAX_PATH, + WORST_CASE_PREFIX, + overlong_install_paths, +) + + +def _wheel(tmp_path, *entry_names): + path = tmp_path / "pkg.whl" + with zipfile.ZipFile(path, "w") as zf: + for name in entry_names: + zf.writestr(name, "{}") + return str(path) + + +def test_flags_entry_one_char_over_budget(tmp_path): + busts = "a" * (MAX_PATH - WORST_CASE_PREFIX + 1) + assert overlong_install_paths(_wheel(tmp_path, busts)) == [busts] + + +def test_allows_entry_exactly_at_budget(tmp_path): + at_limit = "a" * (MAX_PATH - WORST_CASE_PREFIX) + assert ( + overlong_install_paths(_wheel(tmp_path, at_limit, "litellm/__init__.py")) == [] + ) + + +def test_orders_offenders_longest_first(tmp_path): + longer = "a" * (MAX_PATH - WORST_CASE_PREFIX + 5) + shorter = "b" * (MAX_PATH - WORST_CASE_PREFIX + 1) + assert overlong_install_paths(_wheel(tmp_path, shorter, longer)) == [ + longer, + shorter, + ] From cc55662e5fdc6af4f118a1f3ff885068b75450d9 Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 3 Jun 2026 11:34:04 -0700 Subject: [PATCH 7/7] fix(vertex): strip output_config.effort for Vertex Claude models that reject it (Haiku 4.5) (#29585) * fix(vertex): strip output_config.effort for models that reject it Haiku 4.5 on Vertex AI does not support output_config.effort and 400s with "output_config.effort: Extra inputs are not permitted". PR #27074 emptied VERTEX_UNSUPPORTED_OUTPUT_CONFIG_KEYS so effort would forward for Opus/Sonnet 4.6+, but that made the strip unconditional across every Vertex Anthropic model, including ones that don't support it. Claude Code injects effort into its default Messages payload, so `claude --model claude-haiku-4.5` started failing. Make the sanitizer model-aware: drop output_config.effort for models that don't advertise output_config support (or any reasoning effort level) while forwarding it for those that do. The fix covers both the chat-completion and Messages pass-through transformation paths since they share the helper. * chore(vertex): log at debug when dropping unsupported output_config.effort Operators pointing an unregistered Vertex Claude alias that does support effort would otherwise see it stripped with no signal. Debug level keeps it out of normal logs since Claude Code sends effort on every request. --- .../transformation.py | 2 +- .../anthropic/output_params_utils.py | 51 ++++++++++++++----- .../anthropic/transformation.py | 2 +- ...artner_models_anthropic_messages_config.py | 34 +++++++++++++ ...partner_models_anthropic_transformation.py | 46 ++++++++++++++--- 5 files changed, 112 insertions(+), 23 deletions(-) diff --git a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/transformation.py b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/transformation.py index 4be4c2d5e78..1e92754857b 100644 --- a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/transformation.py +++ b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/transformation.py @@ -159,6 +159,6 @@ class VertexAIPartnerModelsAnthropicMessagesConfig(AnthropicMessagesConfig, Vert "model", None ) # do not pass model in request body to vertex ai - sanitize_vertex_anthropic_output_params(anthropic_messages_request) + sanitize_vertex_anthropic_output_params(anthropic_messages_request, model) return anthropic_messages_request diff --git a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/output_params_utils.py b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/output_params_utils.py index a33ad677789..280cc1c888a 100644 --- a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/output_params_utils.py +++ b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/output_params_utils.py @@ -10,23 +10,38 @@ import; extracting the helper into a leaf module resolves the warning and keeps the parent module's import surface narrow. """ -# Keys inside ``output_config`` that Vertex AI Claude does not accept. -# Add an entry only when a 400 "Extra inputs are not permitted" is -# reproducible against the live Vertex endpoint. +# Keys inside ``output_config`` that Vertex AI Claude rejects regardless of +# the target model. Add an entry only when a 400 "Extra inputs are not +# permitted" is reproducible against the live Vertex endpoint for every model. VERTEX_UNSUPPORTED_OUTPUT_CONFIG_KEYS: frozenset = frozenset() -def sanitize_vertex_anthropic_output_params(data: dict) -> None: +def _model_accepts_output_config_effort(model: str) -> bool: + """Whether ``model`` accepts ``output_config.effort`` on Vertex. + + Opus/Sonnet 4.6+ advertise ``supports_output_config`` (or a reasoning + effort level) and accept it; Haiku 4.5 advertises neither and 400s on + ``output_config.effort: Extra inputs are not permitted``. Imported lazily + so this stays a leaf module (see module docstring). + """ + from litellm.llms.anthropic.chat.transformation import AnthropicConfig + + return AnthropicConfig._model_supports_effort_param(model) + + +def sanitize_vertex_anthropic_output_params(data: dict, model: str) -> None: """ Strip Vertex-unsupported keys from ``output_config`` / ``output_format`` in-place; forward whatever remains. Behavior: - * ``output_config`` containing only unsupported keys (e.g. ``effort`` - alone) is removed entirely so the request body has no empty dict. - * ``output_config`` containing a mix of supported + unsupported keys - has the unsupported subset filtered out and the rest forwarded. - * ``output_config`` that is supported in full passes through unchanged. + * ``output_config.effort`` is dropped for models that don't accept it + (e.g. Haiku 4.5) and forwarded for those that do (Opus/Sonnet 4.6+). + Clients like Claude Code inject it into every Messages payload, so the + gate has to live here rather than rely on the caller. + * Keys in ``VERTEX_UNSUPPORTED_OUTPUT_CONFIG_KEYS`` are always filtered. + * ``output_config`` left empty after filtering is removed so the request + body has no empty dict. * ``output_format`` is forwarded as-is (Vertex AI Claude accepts it). * Non-dict values for ``output_config`` are dropped to avoid sending malformed payloads downstream. @@ -37,11 +52,19 @@ def sanitize_vertex_anthropic_output_params(data: dict) -> None: if not isinstance(output_config, dict): data.pop("output_config", None) return - sanitized = { - k: v - for k, v in output_config.items() - if k not in VERTEX_UNSUPPORTED_OUTPUT_CONFIG_KEYS - } + + drop_keys = set(VERTEX_UNSUPPORTED_OUTPUT_CONFIG_KEYS) + if "effort" in output_config and not _model_accepts_output_config_effort(model): + from litellm._logging import verbose_logger + + verbose_logger.debug( + "Dropping unsupported output_config.effort for vertex_ai model=%s " + "(no supports_output_config in the model map)", + model, + ) + drop_keys.add("effort") + + sanitized = {k: v for k, v in output_config.items() if k not in drop_keys} if sanitized: data["output_config"] = sanitized else: diff --git a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py index 4627d9f6df3..c852909d475 100644 --- a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py +++ b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py @@ -106,7 +106,7 @@ class VertexAIAnthropicConfig(AnthropicConfig): data.pop("model", None) # vertex anthropic doesn't accept 'model' parameter - sanitize_vertex_anthropic_output_params(data) + sanitize_vertex_anthropic_output_params(data, model) tools = optional_params.get("tools") tool_search_used = self.is_tool_search_used(tools) diff --git a/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_messages_config.py b/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_messages_config.py index b8cd65d3c99..6f4bb4e59c2 100644 --- a/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_messages_config.py +++ b/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_messages_config.py @@ -313,6 +313,40 @@ def test_transform_anthropic_messages_request_removes_scope_from_cache_control() assert result["messages"][0]["content"][0]["cache_control"]["type"] == "ephemeral" +def test_messages_request_strips_effort_for_haiku_45(): + """Regression: Claude Code (``claude --model claude-haiku-4.5``) sends + ``output_config.effort`` in its default Messages payload. Haiku 4.5 on + Vertex rejects it with 400 ``output_config.effort: Extra inputs are not + permitted``, so the pass-through must strip it for Haiku while keeping it + for Opus/Sonnet 4.6+.""" + config = VertexAIPartnerModelsAnthropicMessagesConfig() + messages = [{"role": "user", "content": "Hello"}] + + haiku_result = config.transform_anthropic_messages_request( + model="claude-haiku-4-5@20251001", + messages=messages, + anthropic_messages_optional_request_params={ + "max_tokens": 1024, + "output_config": {"effort": "high"}, + }, + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + assert "output_config" not in haiku_result + + opus_result = config.transform_anthropic_messages_request( + model="claude-opus-4-6", + messages=messages, + anthropic_messages_optional_request_params={ + "max_tokens": 1024, + "output_config": {"effort": "high"}, + }, + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + assert opus_result["output_config"] == {"effort": "high"} + + def test_provider_config_manager_reuses_vertex_anthropic_messages_config_instance(): """ Regression test: repeated provider config lookups for the same Vertex Claude model diff --git a/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_transformation.py b/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_transformation.py index d89d09a4e63..ac2368130d8 100644 --- a/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_transformation.py +++ b/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_transformation.py @@ -675,28 +675,60 @@ def test_sanitize_vertex_anthropic_output_params_unit(): sanitize_vertex_anthropic_output_params, ) + supported = "claude-opus-4-6" + # No-op when output_config absent. data: dict = {"max_tokens": 8} - sanitize_vertex_anthropic_output_params(data) + sanitize_vertex_anthropic_output_params(data, supported) assert data == {"max_tokens": 8} - # Effort-only → preserved (Vertex 4.6/4.7 accept it on rawPredict). + # Effort-only on a supporting model → preserved (Vertex 4.6/4.7 accept it). data = {"output_config": {"effort": "high"}} - sanitize_vertex_anthropic_output_params(data) + sanitize_vertex_anthropic_output_params(data, supported) assert data["output_config"] == {"effort": "high"} # Format-only → preserved unchanged. fmt = {"format": {"type": "json_schema", "schema": {"type": "object"}}} data = {"output_config": dict(fmt)} - sanitize_vertex_anthropic_output_params(data) + sanitize_vertex_anthropic_output_params(data, supported) assert data["output_config"] == fmt - # Mixed → both effort and format kept (no current Vertex-unsupported keys). + # Mixed on a supporting model → both effort and format kept. data = {"output_config": {"format": fmt["format"], "effort": "high"}} - sanitize_vertex_anthropic_output_params(data) + sanitize_vertex_anthropic_output_params(data, supported) assert data["output_config"] == {"format": fmt["format"], "effort": "high"} # Non-dict → dropped defensively. data = {"output_config": "garbage"} - sanitize_vertex_anthropic_output_params(data) + sanitize_vertex_anthropic_output_params(data, supported) assert "output_config" not in data + + +def test_sanitize_strips_effort_for_haiku_45(): + """Regression: Haiku 4.5 on Vertex does not support ``output_config.effort`` + and 400s with ``Extra inputs are not permitted``. Claude Code injects + ``effort`` into every Messages payload, so the helper must strip it for + models that don't advertise output_config support while leaving it intact + for Opus/Sonnet 4.6+.""" + from litellm.llms.vertex_ai.vertex_ai_partner_models.anthropic.output_params_utils import ( + sanitize_vertex_anthropic_output_params, + ) + + haiku = "claude-haiku-4-5@20251001" + + # Effort-only → output_config removed entirely (no empty dict on the wire). + data: dict = {"output_config": {"effort": "high"}, "max_tokens": 8} + sanitize_vertex_anthropic_output_params(data, haiku) + assert "output_config" not in data + assert data["max_tokens"] == 8 + + # Mixed → effort stripped, format preserved. + fmt = {"type": "json_schema", "schema": {"type": "object"}} + data = {"output_config": {"effort": "high", "format": fmt}} + sanitize_vertex_anthropic_output_params(data, haiku) + assert data["output_config"] == {"format": fmt} + + # Same payload on a supporting model keeps effort untouched. + data = {"output_config": {"effort": "high"}} + sanitize_vertex_anthropic_output_params(data, "vertex_ai/claude-opus-4-6") + assert data["output_config"] == {"effort": "high"}