From e3da917e679ff68aaa86f5ed67dcb6528117f072 Mon Sep 17 00:00:00 2001 From: Souravrajvi0 <144546710+Souravrajvi0@users.noreply.github.com> Date: Tue, 11 Aug 2026 06:29:07 +0000 Subject: [PATCH 01/14] fix(proxy): parse form-encoded video edit/extension bodies after auth Fixes #36487 video_edit, video_extension, and video_remix called request.body() after user_api_key_auth had already parsed multipart/form bodies via _read_request_body(), causing RuntimeError Stream consumed and 500s for OpenAI SDK clients. Use _read_request_body consistently and normalize bare-string or JSON-string video references from form fields into video_id. --- litellm/proxy/video_endpoints/endpoints.py | 28 ++++------- litellm/proxy/video_endpoints/utils.py | 21 +++++++++ .../proxy/video_endpoints/test_endpoints.py | 24 ++++++++++ tests/test_litellm/test_video_generation.py | 47 +++++++++++++++++++ 4 files changed, 100 insertions(+), 20 deletions(-) diff --git a/litellm/proxy/video_endpoints/endpoints.py b/litellm/proxy/video_endpoints/endpoints.py index 6c6b004fd17..cb014bcceee 100644 --- a/litellm/proxy/video_endpoints/endpoints.py +++ b/litellm/proxy/video_endpoints/endpoints.py @@ -2,7 +2,6 @@ from typing import Any, Final -import orjson from fastapi import APIRouter, Depends, File, Form, Request, Response, UploadFile from fastapi.responses import ORJSONResponse @@ -20,6 +19,7 @@ from litellm.proxy.video_endpoints.utils import ( encode_character_id_in_response, extract_model_from_target_model_names, get_custom_provider_from_data, + pop_video_reference_to_video_id, ) from litellm.types.videos.utils import ( decode_character_id_with_provider, @@ -451,9 +451,7 @@ async def video_remix( version, ) - # Read request body - body: Final = await request.body() - data: Final = orjson.loads(body) + data: Final = await _read_request_body(request=request) data["video_id"] = video_id decoded: Final = decode_video_id_with_provider(video_id) @@ -760,15 +758,10 @@ async def video_edit( version, ) - body: Final = await request.body() - data: Final = orjson.loads(body) + data: Final = await _read_request_body(request=request) + pop_video_reference_to_video_id(data) - # Extract video_id from nested video object - video_ref: Final = data.pop("video", {}) - video_id: Final = video_ref.get("id", "") if isinstance(video_ref, dict) else "" - data["video_id"] = video_id - - decoded: Final = decode_video_id_with_provider(video_id) + decoded: Final = decode_video_id_with_provider(data["video_id"]) provider_from_id: Final = decoded.get("custom_llm_provider") model_id_from_decoded: Final = decoded.get("model_id") @@ -860,15 +853,10 @@ async def video_extension( version, ) - body: Final = await request.body() - data: Final = orjson.loads(body) + data: Final = await _read_request_body(request=request) + pop_video_reference_to_video_id(data) - # Extract video_id from nested video object - video_ref: Final = data.pop("video", {}) - video_id: Final = video_ref.get("id", "") if isinstance(video_ref, dict) else "" - data["video_id"] = video_id - - decoded: Final = decode_video_id_with_provider(video_id) + decoded: Final = decode_video_id_with_provider(data["video_id"]) provider_from_id: Final = decoded.get("custom_llm_provider") model_id_from_decoded: Final = decoded.get("model_id") diff --git a/litellm/proxy/video_endpoints/utils.py b/litellm/proxy/video_endpoints/utils.py index d6b398e3476..5f508cd02ca 100644 --- a/litellm/proxy/video_endpoints/utils.py +++ b/litellm/proxy/video_endpoints/utils.py @@ -13,6 +13,27 @@ def extract_model_from_target_model_names(target_model_names: Any) -> str | None return target_model_names[0] if target_model_names else None +def pop_video_reference_to_video_id(data: dict[str, Any]) -> None: + """ + Normalize OpenAI video edit/extension payloads into ``video_id``. + + JSON bodies use ``video: {"id": ...}``. Multipart and form-urlencoded bodies + may send a bare id string or a JSON-encoded reference object as a string field. + """ + video_ref: Final = data.pop("video", {}) + if isinstance(video_ref, dict): + video_id: Final = video_ref.get("id", "") + elif isinstance(video_ref, str): + try: + parsed_ref: Final = orjson.loads(video_ref) + except orjson.JSONDecodeError: + parsed_ref = None + video_id = parsed_ref.get("id", "") if isinstance(parsed_ref, dict) else video_ref + else: + video_id = "" + data["video_id"] = video_id + + def get_custom_provider_from_data(data: dict[str, Any]) -> str | None: custom_llm_provider: Final = data.get("custom_llm_provider") if custom_llm_provider: diff --git a/tests/test_litellm/proxy/video_endpoints/test_endpoints.py b/tests/test_litellm/proxy/video_endpoints/test_endpoints.py index 40a26fad3c3..78f32600f73 100644 --- a/tests/test_litellm/proxy/video_endpoints/test_endpoints.py +++ b/tests/test_litellm/proxy/video_endpoints/test_endpoints.py @@ -375,6 +375,7 @@ async def test_content__model_encoded_id(harness): async def call_edit( harness: Harness, *, body: Dict[str, Any], headers=None, query=None ): + harness.read_body.return_value = dict(body) return await endpoints.video_edit( request=FakeRequest(headers=headers, query=query, raw_body=orjson.dumps(body)), fastapi_response=Response(), @@ -431,6 +432,27 @@ async def test_edit__missing_video_object_defaults_to_openai(harness): assert "video" not in data +@pytest.mark.asyncio +async def test_edit__bare_string_video_id_from_form_field(harness): + await call_edit(harness, body={"prompt": "brighter", "video": "video_plain"}) + + assert harness.processor_data() == { + "prompt": "brighter", + "video_id": "video_plain", + "custom_llm_provider": "openai", + } + + +@pytest.mark.asyncio +async def test_edit__json_string_video_reference_from_form_field(harness): + await call_edit( + harness, + body={"prompt": "brighter", "video": orjson.dumps({"id": "video_plain"}).decode()}, + ) + + assert harness.processor_data()["video_id"] == "video_plain" + + # =========================================================================== # # GET /v1/videos - video_list # # =========================================================================== # @@ -474,6 +496,7 @@ async def test_list__provider_from_header(harness): async def call_remix( harness: Harness, video_id: str, *, body, headers=None, query=None ): + harness.read_body.return_value = dict(body) return await endpoints.video_remix( video_id=video_id, request=FakeRequest(headers=headers, query=query, raw_body=orjson.dumps(body)), @@ -632,6 +655,7 @@ async def test_get_character__plain_id_defaults_openai_no_encode(harness): async def call_extension(harness: Harness, *, body, headers=None, query=None): + harness.read_body.return_value = dict(body) return await endpoints.video_extension( request=FakeRequest(headers=headers, query=query, raw_body=orjson.dumps(body)), fastapi_response=Response(), diff --git a/tests/test_litellm/test_video_generation.py b/tests/test_litellm/test_video_generation.py index 3d0472ef96e..fc7ba773c38 100644 --- a/tests/test_litellm/test_video_generation.py +++ b/tests/test_litellm/test_video_generation.py @@ -2321,6 +2321,53 @@ def test_edit_and_extension_support_custom_provider_from_extra_body( assert captured_data["custom_llm_provider"] == "vertex_ai" +@pytest.mark.parametrize("endpoint", ["/v1/videos/edits", "/v1/videos/extensions"]) +def test_edit_and_extension_accept_form_encoded_after_auth_reads_body( + video_proxy_test_client, endpoint +): + from fastapi import Request + from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing + from litellm.proxy.common_utils.http_parsing_utils import _read_request_body + from litellm.proxy.auth.user_api_key_auth import user_api_key_auth + + captured_data = {} + + async def _mock_base_process(self, **kwargs): + captured_data.update(self.data) + return { + "id": "video_resp_123", + "object": "video", + "status": "queued", + "created_at": 1712697600, + } + + async def auth_that_reads_body_first(request: Request): + await _read_request_body(request=request) + return MagicMock() + + app = video_proxy_test_client.app + app.dependency_overrides[user_api_key_auth] = auth_that_reads_body_first + + with patch.object( + ProxyBaseLLMRequestProcessing, + "base_process_llm_request", + new=_mock_base_process, + ): + response = video_proxy_test_client.post( + endpoint, + headers={"Authorization": "Bearer sk-1234"}, + data={ + "model": "my-video-model", + "prompt": "brighter", + "video": "video_123", + }, + ) + + assert response.status_code == 200, response.text + assert captured_data["video_id"] == "video_123" + assert captured_data["prompt"] == "brighter" + + @pytest.mark.parametrize("endpoint", ["/v1/videos/edits", "/v1/videos/extensions"]) def test_edit_and_extension_route_with_encoded_video_ids( video_proxy_test_client, endpoint From b573679384282e8a9ffa265a5af0b47e61a8bf89 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Fri, 21 Aug 2026 18:44:46 -0700 Subject: [PATCH 02/14] fix(proxy): keep every value of a repeated form key in get_form_data get_form_data collapsed the FormData multidict with dict(form) before the loop that rebuilds `foo[]` arrays ever ran, so a request sending timestamp_granularities[]=word and timestamp_granularities[]=segment reached the provider as ["segment"] with the first value silently dropped. Read the multidict with multi_items() instead. The test could not catch it because its mock was a plain dict carrying the same key twice, which Python collapses exactly the way the bug did. Every request.form mock that fed get_form_data now returns real FormData. --- .../proxy/common_utils/http_parsing_utils.py | 6 +- tests/test_litellm/ocr/test_ocr_file_input.py | 3 +- .../common_utils/test_http_parsing_utils.py | 74 ++++++++----------- .../test_llm_pass_through_endpoints.py | 3 +- 4 files changed, 38 insertions(+), 48 deletions(-) diff --git a/litellm/proxy/common_utils/http_parsing_utils.py b/litellm/proxy/common_utils/http_parsing_utils.py index 1e4344a71f4..4cb55f6966e 100644 --- a/litellm/proxy/common_utils/http_parsing_utils.py +++ b/litellm/proxy/common_utils/http_parsing_utils.py @@ -274,10 +274,10 @@ async def get_form_data(request: Request) -> dict[str, Any]: Handles when OpenAI SDKs pass form keys as `timestamp_granularities[]="word"` instead of `timestamp_granularities=["word", "sentence"]` """ form: Final = await request.form() - form_data: Final = dict(form) parsed_form_data: Final[dict[str, Any]] = {} - for key, value in form_data.items(): - # OpenAI SDKs pass form keys as `timestamp_granularities[]="word"` instead of `timestamp_granularities=["word", "sentence"]` + # multi_items(), not dict(form): a dict drops every value but the last of a repeated key, + # which is the whole array this function exists to rebuild + for key, value in form.multi_items(): if key.endswith("[]"): clean_key = key[:-2] parsed_form_data.setdefault(clean_key, []).append(value) diff --git a/tests/test_litellm/ocr/test_ocr_file_input.py b/tests/test_litellm/ocr/test_ocr_file_input.py index e6216d7c580..feb98d14c03 100644 --- a/tests/test_litellm/ocr/test_ocr_file_input.py +++ b/tests/test_litellm/ocr/test_ocr_file_input.py @@ -18,6 +18,7 @@ from unittest.mock import AsyncMock, MagicMock import orjson import pytest +from starlette.datastructures import FormData from litellm.ocr.main import convert_file_document_to_url_document, get_mime_type @@ -470,7 +471,7 @@ class TestProxySecurityGuard: mock_request = MagicMock() mock_request.headers = {"content-type": "multipart/form-data; boundary=---"} - mock_request.form = AsyncMock(return_value=mock_form) + mock_request.form = AsyncMock(return_value=FormData(mock_form)) result = await self._parse_multipart(mock_request) diff --git a/tests/test_litellm/proxy/common_utils/test_http_parsing_utils.py b/tests/test_litellm/proxy/common_utils/test_http_parsing_utils.py index 869d228d5a4..98af1fb7ce2 100644 --- a/tests/test_litellm/proxy/common_utils/test_http_parsing_utils.py +++ b/tests/test_litellm/proxy/common_utils/test_http_parsing_utils.py @@ -7,6 +7,7 @@ import orjson import pytest from fastapi import Request from fastapi.testclient import TestClient +from starlette.datastructures import FormData sys.path.insert( 0, os.path.abspath("../../../..") @@ -73,7 +74,7 @@ async def test_form_data_parsing(): test_data = {"name": "test_user", "message": "hello world"} # Mock the form method to return the test data as an awaitable - mock_request.form = AsyncMock(return_value=test_data) + mock_request.form = AsyncMock(return_value=FormData(test_data)) mock_request.headers = {"content-type": "application/x-www-form-urlencoded"} mock_request.scope = {} mock_request.state._cached_headers = None @@ -124,7 +125,7 @@ async def test_form_data_with_json_metadata(): } # Mock the form method to return the test data as an awaitable - mock_request.form = AsyncMock(return_value=test_data) + mock_request.form = AsyncMock(return_value=FormData(test_data)) mock_request.headers = {"content-type": "multipart/form-data"} mock_request.scope = {} mock_request.state._cached_headers = None @@ -165,7 +166,7 @@ async def test_form_data_with_invalid_json_metadata(): } # Mock the form method to return the test data - mock_request.form = AsyncMock(return_value=test_data) + mock_request.form = AsyncMock(return_value=FormData(test_data)) mock_request.headers = {"content-type": "multipart/form-data"} mock_request.scope = {} mock_request.state._cached_headers = None @@ -188,7 +189,7 @@ async def test_form_data_without_metadata(): test_data = {"model": "whisper-1", "file": "audio.mp3", "language": "en"} # Mock the form method to return the test data - mock_request.form = AsyncMock(return_value=test_data) + mock_request.form = AsyncMock(return_value=FormData(test_data)) mock_request.headers = {"content-type": "application/x-www-form-urlencoded"} mock_request.scope = {} mock_request.state._cached_headers = None @@ -219,7 +220,7 @@ async def test_form_data_with_empty_metadata(): } # Mock the form method to return the test data - mock_request.form = AsyncMock(return_value=test_data) + mock_request.form = AsyncMock(return_value=FormData(test_data)) mock_request.headers = {"content-type": "multipart/form-data"} mock_request.scope = {} mock_request.state._cached_headers = None @@ -254,7 +255,7 @@ async def test_form_data_with_dict_metadata(): } # Mock the form method to return the test data - mock_request.form = AsyncMock(return_value=test_data) + mock_request.form = AsyncMock(return_value=FormData(test_data)) mock_request.headers = {"content-type": "multipart/form-data"} mock_request.scope = {} mock_request.state._cached_headers = None @@ -285,7 +286,7 @@ async def test_form_data_with_none_metadata(): } # Mock the form method to return the test data - mock_request.form = AsyncMock(return_value=test_data) + mock_request.form = AsyncMock(return_value=FormData(test_data)) mock_request.headers = {"content-type": "multipart/form-data"} mock_request.scope = {} mock_request.state._cached_headers = None @@ -500,33 +501,29 @@ async def test_surrogate_repair_skipped_above_size_limit(monkeypatch): @pytest.mark.asyncio async def test_get_form_data(): """ - Test that get_form_data correctly handles form data with array notation. - Tests audio transcription parameters as a specific example. + A repeated `foo[]` key is how the OpenAI SDKs send a list, so every value has to + survive. `FormData`, not a dict: a dict cannot even hold the duplicate key. """ - # Create a mock request with transcription form data mock_request = MagicMock() + mock_request.form = AsyncMock( + return_value=FormData( + [ + ("file", "file_object"), + ("model", "gpt-4o-transcribe"), + ("include[]", "logprobs"), + ("language", "en"), + ("prompt", "Transcribe this audio file"), + ("response_format", "json"), + ("stream", "false"), + ("temperature", "0.2"), + ("timestamp_granularities[]", "word"), + ("timestamp_granularities[]", "segment"), + ] + ) + ) - # Create mock form data with array notation for timestamp_granularities - mock_form_data = { - "file": "file_object", # In a real request this would be an UploadFile - "model": "gpt-4o-transcribe", - "include[]": "logprobs", # Array notation - "language": "en", - "prompt": "Transcribe this audio file", - "response_format": "json", - "stream": "false", - "temperature": "0.2", - "timestamp_granularities[]": "word", # First array item - "timestamp_granularities[]": "segment", # Second array item (would overwrite in dict, but handled by the function) - } - - # Mock the form method to return the test data - mock_request.form = AsyncMock(return_value=mock_form_data) - - # Call the function being tested result = await get_form_data(mock_request) - # Verify regular form fields are preserved assert result["file"] == "file_object" assert result["model"] == "gpt-4o-transcribe" assert result["language"] == "en" @@ -534,17 +531,8 @@ async def test_get_form_data(): assert result["response_format"] == "json" assert result["stream"] == "false" assert result["temperature"] == "0.2" - - # Verify array fields are correctly parsed - assert "include" in result - assert isinstance(result["include"], list) - assert "logprobs" in result["include"] - - assert "timestamp_granularities" in result - assert isinstance(result["timestamp_granularities"], list) - # Note: In a real MultiDict, both values would be present - # But in our mock dictionary the second value overwrites the first - assert "segment" in result["timestamp_granularities"] + assert result["include"] == ["logprobs"] + assert result["timestamp_granularities"] == ["word", "segment"] def test_get_tags_from_request_body_with_metadata_tags(): @@ -958,7 +946,7 @@ class TestReadRequestBodyNonCanonicalContentType: mock_request = MagicMock() mock_request.body = AsyncMock(return_value=orjson.dumps(payload)) - mock_request.form = AsyncMock(return_value={}) + mock_request.form = AsyncMock(return_value=FormData({})) mock_request.headers = {"content-type": content_type} mock_request.scope = {} @@ -969,7 +957,7 @@ class TestReadRequestBodyNonCanonicalContentType: @pytest.mark.asyncio async def test_real_form_post_still_parsed_as_form(self): mock_request = MagicMock() - mock_request.form = AsyncMock(return_value={"k": "v"}) + mock_request.form = AsyncMock(return_value=FormData({"k": "v"})) mock_request.body = AsyncMock(return_value=b"") mock_request.headers = {"content-type": "application/x-www-form-urlencoded"} mock_request.scope = {} @@ -1025,7 +1013,7 @@ class TestGetRequestBody: mock_request = MagicMock() mock_request.method = "POST" mock_request.headers = {"content-type": "multipart/form-data; boundary=x"} - mock_request.form = AsyncMock(return_value={"k": "v"}) + mock_request.form = AsyncMock(return_value=FormData({"k": "v"})) mock_request.scope = {} result = await get_request_body(mock_request) diff --git a/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py b/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py index 6568f6aeacf..88141fd1c90 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py @@ -13,6 +13,7 @@ import httpx import pytest from fastapi import HTTPException, Request, Response from fastapi.testclient import TestClient +from starlette.datastructures import FormData sys.path.insert( 0, os.path.abspath("../../../..") @@ -1384,7 +1385,7 @@ async def test_is_streaming_request_fn(): mock_request = Mock() mock_request.method = "POST" mock_request.headers = {"content-type": "multipart/form-data"} - mock_request.form = AsyncMock(return_value={"stream": "true"}) + mock_request.form = AsyncMock(return_value=FormData({"stream": "true"})) assert await is_streaming_request_fn(mock_request) is True From b7f8016002c080f64ca76a1136fc3fe103a5ee75 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Fri, 21 Aug 2026 18:44:57 -0700 Subject: [PATCH 03/14] test: gate the test suite on F601, B023, B025 and F632 Four more ruff rules for code the test suite runs but never checks. F601 is the one that paid: the duplicate key it flagged in a get_form_data fixture was the mock reproducing the production bug fixed in the previous commit. B025 removed two unreachable handlers, one of them a pytest.skip shadowed by an earlier `pass`, so an upstream Vertex flake reported green having asserted nothing. F632 turned an `is ""` identity check, which passes only on CPython interning, into the `== ""` it meant. B023 fixed three closures over loop variables, all latent today but one iteration-order change away from checking the last case N times. --- ruff-tests.toml | 15 ++++++++++++++ tests/code_coverage_tests/bedrock_pricing.py | 2 +- tests/load_tests/test_langsmith_load_test.py | 5 ----- .../test_amazing_vertex_completion.py | 10 ++-------- .../logging_callback_tests/test_spend_logs.py | 1 - .../test_key_generate_prisma.py | 20 ++++++++----------- .../test_ollama_completion_transformation.py | 2 +- .../llms/watsonx/test_watsonx_common_utils.py | 6 +----- tests/test_litellm/test_utils.py | 2 -- 9 files changed, 28 insertions(+), 35 deletions(-) diff --git a/ruff-tests.toml b/ruff-tests.toml index de0931f5e69..8f21ca31a6f 100644 --- a/ruff-tests.toml +++ b/ruff-tests.toml @@ -36,6 +36,17 @@ # `re.search`, so a `.` copied out of an error message is a wildcard and the block # accepts messages the author never meant to accept. Mark a real regex raw, wrap a # literal message in `re.escape`, and the pattern says which one it is +# F601 the same key literal twice in one dict. Python keeps the last value, so the +# first is dropped before the test ever runs, and a fixture that looks like it +# covers two cases covers one +# B023 a closure over a loop variable. Every closure sees the last iteration's value, +# so a per-case callback built in a loop checks the last case N times. Bind the +# value as a parameter instead +# B025 an `except` for a type an earlier `except` already catches. The second handler +# is unreachable, so the recovery or skip written there never happens +# F632 `is` against a literal. It compares identity, so it passes only where CPython +# happens to intern the value and stops meaning what it says the moment the +# value is built at runtime # # No target-version here on purpose: it resolves from requires-python (>=3.10), so # 3.11-only builtins like BaseExceptionGroup are correctly flagged in a tree that @@ -58,4 +69,8 @@ lint.select = [ "PLR0133", "PLW0127", "RUF043", + "F601", + "B023", + "B025", + "F632", ] diff --git a/tests/code_coverage_tests/bedrock_pricing.py b/tests/code_coverage_tests/bedrock_pricing.py index b2c9e78b06c..e6219109f6d 100644 --- a/tests/code_coverage_tests/bedrock_pricing.py +++ b/tests/code_coverage_tests/bedrock_pricing.py @@ -95,7 +95,7 @@ def get_bedrock_pricing(url, providers): else: # General logic for other providers section = soup.find( - "h2", text=lambda t: t and provider.lower() in t.lower() + "h2", text=lambda t, needle=provider.lower(): t and needle in t.lower() ) if not section: pricing_data[provider] = "Provider section not found" diff --git a/tests/load_tests/test_langsmith_load_test.py b/tests/load_tests/test_langsmith_load_test.py index cf9fe526b74..40b976541a5 100644 --- a/tests/load_tests/test_langsmith_load_test.py +++ b/tests/load_tests/test_langsmith_load_test.py @@ -66,11 +66,6 @@ def test_langsmith_logging_async(): except Exception as e: pytest.fail(f"An exception occurred - {e}") - except litellm.Timeout as e: - pass - except Exception as e: - pytest.fail(f"An exception occurred - {e}") - async def make_async_calls(metadata=None, **completion_kwargs): total_tasks = 300 diff --git a/tests/local_testing/test_amazing_vertex_completion.py b/tests/local_testing/test_amazing_vertex_completion.py index a52b5975f6e..53b3b2d6071 100644 --- a/tests/local_testing/test_amazing_vertex_completion.py +++ b/tests/local_testing/test_amazing_vertex_completion.py @@ -4207,13 +4207,7 @@ def test_gemini_google_maps_tool_simple(): ) print(f"Response: {response.model_dump_json(indent=4)}") assert response.choices[0].message.content is not None - except (litellm.RateLimitError, litellm.InternalServerError): - # Transient Vertex-side failures (rate limiting, 500 INTERNAL from the - # Google Maps grounding backend) are not LiteLLM bugs — don't fail CI. - pass - except litellm.InternalServerError: - pytest.skip( - "Google Maps Platform returned a transient 500 (upstream flake); skipping." - ) + except (litellm.RateLimitError, litellm.InternalServerError) as e: + pytest.skip(f"Transient Vertex-side failure, not a LiteLLM bug: {e}") except Exception as e: pytest.fail(f"Error occurred: {e}") diff --git a/tests/logging_callback_tests/test_spend_logs.py b/tests/logging_callback_tests/test_spend_logs.py index 709aa81f421..3aa0b3ebd90 100644 --- a/tests/logging_callback_tests/test_spend_logs.py +++ b/tests/logging_callback_tests/test_spend_logs.py @@ -148,7 +148,6 @@ def test_spend_logs_payload(model_id: Optional[str]): "completion_start_time": datetime.datetime(2024, 6, 7, 12, 43, 30, 954146), "max_tokens": 10, "extra_body": {}, - "custom_llm_provider": "azure", "input": [ {"role": "system", "content": "you are a helpful assistant.\n"}, {"role": "user", "content": "bom dia"}, diff --git a/tests/proxy_unit_tests/test_key_generate_prisma.py b/tests/proxy_unit_tests/test_key_generate_prisma.py index efedc156429..4037107c474 100644 --- a/tests/proxy_unit_tests/test_key_generate_prisma.py +++ b/tests/proxy_unit_tests/test_key_generate_prisma.py @@ -3327,16 +3327,17 @@ async def test_team_access_groups(prisma_client): request._url = URL(url="/chat/completions") + def body_reader(requested_model: str): + async def return_body() -> bytes: + return f'{{"model": "{requested_model}"}}'.encode() + + return return_body + for model in ["gpt-4o", "gemini-pro-vision"]: # Expect these to pass - async def return_body(): - return_string = f'{{"model": "{model}"}}' - # return string as bytes - return return_string.encode() - request = Request(scope={"type": "http"}) request._url = URL(url="/chat/completions") - request.body = return_body + request.body = body_reader(model) # use generated key to auth in print( @@ -3346,14 +3347,9 @@ async def test_team_access_groups(prisma_client): for model in ["gpt-4", "gpt-4o-mini", "gemini-experimental"]: # Expect these to fail - async def return_body_2(): - return_string = f'{{"model": "{model}"}}' - # return string as bytes - return return_string.encode() - request = Request(scope={"type": "http"}) request._url = URL(url="/chat/completions") - request.body = return_body_2 + request.body = body_reader(model) # use generated key to auth in print( diff --git a/tests/test_litellm/llms/ollama/test_ollama_completion_transformation.py b/tests/test_litellm/llms/ollama/test_ollama_completion_transformation.py index dd59cdcac1c..e746c1bfd6b 100644 --- a/tests/test_litellm/llms/ollama/test_ollama_completion_transformation.py +++ b/tests/test_litellm/llms/ollama/test_ollama_completion_transformation.py @@ -480,7 +480,7 @@ class TestOllamaTextCompletionResponseIterator: assert isinstance(result, ModelResponseStream) assert result.choices and result.choices[0].delta is not None assert result.choices[0].delta.content == None - assert getattr(result.choices[0].delta, "reasoning_content", None) is "" + assert getattr(result.choices[0].delta, "reasoning_content", None) == "" def test_chunk_parser_done_chunk(self): """Test that done chunks work correctly.""" diff --git a/tests/test_litellm/llms/watsonx/test_watsonx_common_utils.py b/tests/test_litellm/llms/watsonx/test_watsonx_common_utils.py index be74dc40eda..0f3a6bae1ff 100644 --- a/tests/test_litellm/llms/watsonx/test_watsonx_common_utils.py +++ b/tests/test_litellm/llms/watsonx/test_watsonx_common_utils.py @@ -129,11 +129,7 @@ class TestGenerateIAMToken: mock_client.reset_mock() mock_cache.reset_mock() - # Configure mock to return values based on env_keys - def get_secret_side_effect(key): - return env_keys.get(key) - - mock_get_secret_str.side_effect = get_secret_side_effect + mock_get_secret_str.side_effect = env_keys.get mock_response = MagicMock() mock_response.json.return_value = { diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index bd23ca11fbe..a01540c51e8 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -864,7 +864,6 @@ def test_aaamodel_prices_and_context_window_json_is_valid(): "input_cost_per_character_above_128k_tokens": {"type": "number"}, "input_cost_per_image": {"type": "number"}, "input_cost_per_image_above_128k_tokens": {"type": "number"}, - "input_cost_per_image_token": {"type": "number"}, "input_cost_per_video_token": {"type": "number"}, "input_cost_per_token_above_200k_tokens": {"type": "number"}, "input_cost_per_token_above_256k_tokens": {"type": "number"}, @@ -1008,7 +1007,6 @@ def test_aaamodel_prices_and_context_window_json_is_valid(): }, "bedrock_converse_supports_strict_tools": {"type": "boolean"}, "tpm": {"type": "number"}, - "provider_specific_entry": {"type": "object"}, "supported_endpoints": { "type": "array", "items": { From ee935cec230207a549f2939a6a9ca69023c609d6 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Mon, 24 Aug 2026 09:46:35 -0700 Subject: [PATCH 04/14] refactor(proxy): trim the multi_items comment to the non-obvious clause --- litellm/proxy/common_utils/http_parsing_utils.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/litellm/proxy/common_utils/http_parsing_utils.py b/litellm/proxy/common_utils/http_parsing_utils.py index 4cb55f6966e..96621b08ba1 100644 --- a/litellm/proxy/common_utils/http_parsing_utils.py +++ b/litellm/proxy/common_utils/http_parsing_utils.py @@ -275,9 +275,7 @@ async def get_form_data(request: Request) -> dict[str, Any]: """ form: Final = await request.form() parsed_form_data: Final[dict[str, Any]] = {} - # multi_items(), not dict(form): a dict drops every value but the last of a repeated key, - # which is the whole array this function exists to rebuild - for key, value in form.multi_items(): + for key, value in form.multi_items(): # not dict(form), which keeps only the last repeat if key.endswith("[]"): clean_key = key[:-2] parsed_form_data.setdefault(clean_key, []).append(value) From 91b2a9c360dab5bedec2076a2735b66d042ea595 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 24 Aug 2026 10:40:28 -0700 Subject: [PATCH 05/14] fix(proxy): keep video reference normalization within lint budgets --- litellm/proxy/video_endpoints/utils.py | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/litellm/proxy/video_endpoints/utils.py b/litellm/proxy/video_endpoints/utils.py index 5f508cd02ca..a3406a21573 100644 --- a/litellm/proxy/video_endpoints/utils.py +++ b/litellm/proxy/video_endpoints/utils.py @@ -13,6 +13,18 @@ def extract_model_from_target_model_names(target_model_names: Any) -> str | None return target_model_names[0] if target_model_names else None +def _video_reference_to_id(video_ref: object) -> str: + if isinstance(video_ref, dict): + return video_ref.get("id", "") + if not isinstance(video_ref, str): + return "" + try: + parsed_ref: Final = orjson.loads(video_ref) + except orjson.JSONDecodeError: + return video_ref + return parsed_ref.get("id", "") if isinstance(parsed_ref, dict) else video_ref + + def pop_video_reference_to_video_id(data: dict[str, Any]) -> None: """ Normalize OpenAI video edit/extension payloads into ``video_id``. @@ -20,18 +32,7 @@ def pop_video_reference_to_video_id(data: dict[str, Any]) -> None: JSON bodies use ``video: {"id": ...}``. Multipart and form-urlencoded bodies may send a bare id string or a JSON-encoded reference object as a string field. """ - video_ref: Final = data.pop("video", {}) - if isinstance(video_ref, dict): - video_id: Final = video_ref.get("id", "") - elif isinstance(video_ref, str): - try: - parsed_ref: Final = orjson.loads(video_ref) - except orjson.JSONDecodeError: - parsed_ref = None - video_id = parsed_ref.get("id", "") if isinstance(parsed_ref, dict) else video_ref - else: - video_id = "" - data["video_id"] = video_id + data["video_id"] = _video_reference_to_id(data.pop("video", {})) def get_custom_provider_from_data(data: dict[str, Any]) -> str | None: From d0dd24ed6dafeb88032f6f852ef38f349f71986c Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 24 Aug 2026 11:01:34 -0700 Subject: [PATCH 06/14] fix(health): apply model_info.health_check_params to health check probes --- litellm/proxy/health_check.py | 13 ++ .../health_endpoints/_health_endpoints.py | 9 +- .../proxy/test_health_check_max_tokens.py | 142 ++++++++++++++++++ 3 files changed, 160 insertions(+), 4 deletions(-) diff --git a/litellm/proxy/health_check.py b/litellm/proxy/health_check.py index f9d408fb7de..ae95f35333a 100644 --- a/litellm/proxy/health_check.py +++ b/litellm/proxy/health_check.py @@ -445,6 +445,9 @@ def _update_litellm_params_for_health_check(model_info: dict, litellm_params: di """ Update the litellm params for health check. + - merges `model_info.health_check_params` into the probe request, so a deployment whose provider + requires a payload field litellm does not synthesize (e.g. `mediaSource` for Bedrock TwelveLabs + Pegasus) can supply it. The dedicated knobs below are applied afterwards and win on conflict. - gets a short `messages` param for health check - adds a bounded `max_tokens` when the deployment is a chat-style mode (`chat`, `completion`, `responses`) or the operator explicitly opts in @@ -459,6 +462,16 @@ def _update_litellm_params_for_health_check(model_info: dict, litellm_params: di model_info, litellm_params, # any-ok: untyped router config dict ) + _health_check_params: Final = model_info.get("health_check_params", None) + if isinstance(_health_check_params, dict): + litellm_params.update(_health_check_params) + elif _health_check_params is not None: + logger.warning( + "health_check_params for model %s is a %s, expected a dict. Ignoring it.", + litellm_params.get("model"), + type(_health_check_params).__name__, + ) + litellm_params["messages"] = _get_random_llm_message() if _should_inject_health_check_max_tokens( model_info, diff --git a/litellm/proxy/health_endpoints/_health_endpoints.py b/litellm/proxy/health_endpoints/_health_endpoints.py index 33894777bc3..1feddca9328 100644 --- a/litellm/proxy/health_endpoints/_health_endpoints.py +++ b/litellm/proxy/health_endpoints/_health_endpoints.py @@ -1888,6 +1888,8 @@ async def test_model_connection( # already resolved before reaching this endpoint; any remaining # reference must have come from the request body. _reject_os_environ_references(request_litellm_params) + if model_info: + _reject_os_environ_references(model_info) model_name: Final = request_litellm_params.get("model") # Look up model configuration from router if model name is provided @@ -1951,20 +1953,19 @@ async def test_model_connection( } ## Auth check - auth_model_info: Final = loaded_model_info if loaded_model_info is not None else model_info + resolved_model_info: Final = loaded_model_info if loaded_model_info is not None else model_info await ModelManagementAuthChecks.can_user_make_model_call( model_params=Deployment( model_name="test_model", litellm_params=LiteLLM_Params(**litellm_params), - model_info=auth_model_info, + model_info=resolved_model_info, ), user_api_key_dict=user_api_key_dict, prisma_client=prisma_client, premium_user=premium_user, ) - # Include health_check_params if provided litellm_params = _update_litellm_params_for_health_check( - model_info={}, + model_info=resolved_model_info or {}, litellm_params=litellm_params, ) mode = mode or litellm_params.pop("mode", None) diff --git a/tests/test_litellm/proxy/test_health_check_max_tokens.py b/tests/test_litellm/proxy/test_health_check_max_tokens.py index 5a606d5f74e..e20b18c8813 100644 --- a/tests/test_litellm/proxy/test_health_check_max_tokens.py +++ b/tests/test_litellm/proxy/test_health_check_max_tokens.py @@ -1,7 +1,11 @@ +import json +import logging from unittest.mock import AsyncMock, MagicMock, patch import pytest +import respx +import litellm from litellm.litellm_core_utils.health_check_helpers import HealthCheckHelpers from litellm.proxy import health_check as hc_module from litellm.proxy.health_check import ( @@ -543,3 +547,141 @@ async def test_run_model_health_check_skips_auto_router_deployment(): fake_ahealth_check.assert_not_called() assert result == {} + + +# --------------------------------------------------------------------------- +# model_info.health_check_params +# +# Some providers require a payload field litellm does not synthesize for a +# probe. Bedrock TwelveLabs Pegasus rejects any Invoke body without a top-level +# `mediaSource`, so every health check on such a deployment failed with +# "Invalid JSON: $: required property 'mediaSource' not found". The config key +# was accepted and then never read, so operators had no way to supply it. +# --------------------------------------------------------------------------- + + +def test_health_check_params_merge_into_probe_params(): + """health_check_params reach the probe request for the deployment that declares them.""" + media_source = {"s3Location": {"uri": "s3://my-bucket/clip.mp4"}} + + updated = _update_litellm_params_for_health_check( + {"mode": "chat", "health_check_params": {"mediaSource": media_source}}, + {"model": "bedrock/us.twelvelabs.pegasus-1-2-v1:0"}, + ) + + assert updated["mediaSource"] == media_source + assert updated["model"] == "us.twelvelabs.pegasus-1-2-v1:0" + assert updated["custom_llm_provider"] == "bedrock" + + +def test_health_check_params_lose_to_dedicated_health_check_knobs(): + """The dedicated knobs are applied after the merge, so they win on conflict.""" + model_info = { + "mode": "chat", + "health_check_params": { + "max_tokens": 4096, + "model": "openai/expensive-model", + "messages": [{"role": "user", "content": "from health_check_params"}], + "reasoning_effort": "high", + }, + "health_check_max_tokens": 5, + "health_check_model": "openai/cheap-model", + "health_check_reasoning_effort": "none", + } + + updated = _update_litellm_params_for_health_check(model_info, {"model": "openai/dummy"}) + + assert updated["max_tokens"] == 5 + assert updated["model"] == "openai/cheap-model" + assert updated["reasoning_effort"] == "none" + assert updated["messages"] != model_info["health_check_params"]["messages"] + + +def test_health_check_params_lose_to_the_audio_speech_voice_knob(): + """health_check_voice still wins for audio_speech deployments.""" + updated = _update_litellm_params_for_health_check( + { + "mode": "audio_speech", + "health_check_params": {"voice": "sage", "response_format": "wav"}, + "health_check_voice": "shimmer", + }, + {"model": "openai/tts-1"}, + ) + + assert updated["voice"] == "shimmer" + assert updated["response_format"] == "wav" + + +@pytest.mark.parametrize( + "bad_value", + ["mediaSource", ["mediaSource"], 5, True], +) +def test_health_check_params_ignored_when_not_a_dict(bad_value, caplog): + """A misconfigured health_check_params is skipped with a warning instead of breaking the probe.""" + with caplog.at_level(logging.WARNING, logger="litellm.proxy.health_check"): + updated = _update_litellm_params_for_health_check( + {"mode": "chat", "health_check_params": bad_value}, + {"model": "openai/dummy"}, + ) + + assert updated["model"] == "openai/dummy" + assert updated["max_tokens"] == 16 + assert "health_check_params" in caplog.text + + +def test_health_check_params_apply_to_non_chat_modes(): + """Non-chat probes get health_check_params too, and still no max_tokens.""" + updated = _update_litellm_params_for_health_check( + {"mode": "embedding", "health_check_params": {"dimensions": 8}}, + {"model": "bedrock/amazon.titan-embed-text-v2:0"}, + ) + + assert updated["dimensions"] == 8 + assert "max_tokens" not in updated + + +async def _pegasus_health_check_request_body(model_info: dict, monkeypatch) -> dict: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + + litellm_params = _update_litellm_params_for_health_check( + model_info, + { + "model": "bedrock/us.twelvelabs.pegasus-1-2-v1:0", + "aws_access_key_id": "fake-access-key", + "aws_secret_access_key": "fake-secret-key", + "aws_region_name": "us-east-1", + }, + ) + + with respx.mock(assert_all_called=True) as respx_mock: + invoke_route = respx_mock.post( + host="bedrock-runtime.us-east-1.amazonaws.com", + path__regex=r"/model/.+/invoke", + ).respond(json={"message": "a person walks a dog", "finishReason": "stop"}) + result = await litellm.ahealth_check(litellm_params, mode="chat") + + assert "error" not in result, result + return json.loads(invoke_route.calls.last.request.content) + + +@pytest.mark.asyncio +async def test_health_check_params_reach_the_bedrock_invoke_body(monkeypatch): + """The probe Bedrock actually receives carries mediaSource, which is what unblocks Pegasus.""" + media_source = {"s3Location": {"uri": "s3://my-bucket/clip.mp4"}} + + body = await _pegasus_health_check_request_body( + {"mode": "chat", "health_check_params": {"mediaSource": media_source}}, monkeypatch + ) + + assert body["mediaSource"] == media_source + assert body["maxOutputTokens"] == 16 + assert body["inputPrompt"] + + +@pytest.mark.asyncio +async def test_bedrock_invoke_body_has_no_media_source_without_health_check_params(monkeypatch): + """Negative control: the field only appears because the deployment asked for it.""" + body = await _pegasus_health_check_request_body({"mode": "chat"}, monkeypatch) + + assert "mediaSource" not in body From fef5f41985c12b5b5278b99bdbadbe5ee0381ff0 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 24 Aug 2026 11:12:54 -0700 Subject: [PATCH 07/14] fix(anthropic): keep legacy thinking budget_tokens on Claude 4.6 models --- litellm/llms/anthropic/common_utils.py | 10 ++ .../messages/transformation.py | 10 +- ...odel_prices_and_context_window_backup.json | 36 +++++++ litellm/types/utils.py | 1 + litellm/utils.py | 1 + model_prices_and_context_window.json | 36 +++++++ .../test_reasoning_effort_translation.py | 93 +++++++++++++++---- 7 files changed, 168 insertions(+), 19 deletions(-) diff --git a/litellm/llms/anthropic/common_utils.py b/litellm/llms/anthropic/common_utils.py index 3297aa95715..9461e40cf2e 100644 --- a/litellm/llms/anthropic/common_utils.py +++ b/litellm/llms/anthropic/common_utils.py @@ -440,6 +440,16 @@ class AnthropicModelInfo(BaseLLMModelInfo): """ return AnthropicModelInfo._supports_model_capability(model, "thinking_always_on", custom_llm_provider) + @staticmethod + def _supports_legacy_thinking(model: str, custom_llm_provider: str) -> bool: + """Whether ``model`` is an adaptive-thinking model that still accepts legacy + ``thinking.type=enabled`` with ``budget_tokens`` (the Claude 4.6 family). + The model cost map is authoritative: an explicit ``supports_legacy_thinking`` + entry resolved under ``custom_llm_provider``, or a ``fallback_generalizations`` + rule for unmapped 4.6 ids. Absent flag means the model rejects the legacy shape. + """ + return AnthropicModelInfo._supports_model_capability(model, "supports_legacy_thinking", custom_llm_provider) + @staticmethod def maybe_drop_disabled_thinking( model: str, diff --git a/litellm/llms/anthropic/experimental_pass_through/messages/transformation.py b/litellm/llms/anthropic/experimental_pass_through/messages/transformation.py index adabfa2d62d..032bf0130ce 100644 --- a/litellm/llms/anthropic/experimental_pass_through/messages/transformation.py +++ b/litellm/llms/anthropic/experimental_pass_through/messages/transformation.py @@ -379,13 +379,19 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): def _translate_legacy_thinking_for_adaptive_model( model: str, optional_params: dict, custom_llm_provider: str ) -> None: - """Translate legacy ``thinking.type=enabled`` to adaptive for 4.6/4.7. - Caller-provided ``output_config.effort`` is never overridden. + """Translate legacy ``thinking.type=enabled`` to adaptive for the + adaptive-thinking models that reject it (4.7+ and the 5 families). + Models flagged ``supports_legacy_thinking`` (the 4.6 family) accept the + legacy shape natively, so it is forwarded verbatim and the caller's + ``budget_tokens`` cap keeps applying. Caller-provided + ``output_config.effort`` is never overridden. """ from litellm.llms.anthropic.chat.transformation import AnthropicConfig if not AnthropicModelInfo._is_adaptive_thinking_model(model, custom_llm_provider): return + if AnthropicModelInfo._supports_legacy_thinking(model, custom_llm_provider): + return thinking: Final = optional_params.get("thinking") if not isinstance(thinking, dict) or thinking.get("type") != "enabled": return diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 3af7d9e5019..dda327bf9f6 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -1019,6 +1019,7 @@ }, "anthropic.claude-opus-4-6-v1": { "supports_adaptive_thinking": true, + "supports_legacy_thinking": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, @@ -1053,6 +1054,7 @@ }, "global.anthropic.claude-opus-4-6-v1": { "supports_adaptive_thinking": true, + "supports_legacy_thinking": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, @@ -1087,6 +1089,7 @@ }, "us.anthropic.claude-opus-4-6-v1": { "supports_adaptive_thinking": true, + "supports_legacy_thinking": true, "cache_creation_input_token_cost": 6.875e-06, "cache_creation_input_token_cost_above_1hr": 1.1e-05, "cache_read_input_token_cost": 5.5e-07, @@ -1121,6 +1124,7 @@ }, "eu.anthropic.claude-opus-4-6-v1": { "supports_adaptive_thinking": true, + "supports_legacy_thinking": true, "cache_creation_input_token_cost": 6.875e-06, "cache_creation_input_token_cost_above_1hr": 1.1e-05, "cache_read_input_token_cost": 5.5e-07, @@ -1155,6 +1159,7 @@ }, "au.anthropic.claude-opus-4-6-v1": { "supports_adaptive_thinking": true, + "supports_legacy_thinking": true, "cache_creation_input_token_cost": 6.875e-06, "cache_creation_input_token_cost_above_1hr": 1.1e-05, "cache_read_input_token_cost": 5.5e-07, @@ -2233,6 +2238,7 @@ }, "anthropic.claude-sonnet-4-6": { "supports_adaptive_thinking": true, + "supports_legacy_thinking": true, "cache_creation_input_token_cost": 3.75e-06, "cache_creation_input_token_cost_above_1hr": 6e-06, "cache_read_input_token_cost": 3e-07, @@ -2266,6 +2272,7 @@ }, "global.anthropic.claude-sonnet-4-6": { "supports_adaptive_thinking": true, + "supports_legacy_thinking": true, "cache_creation_input_token_cost": 3.75e-06, "cache_creation_input_token_cost_above_1hr": 6e-06, "cache_read_input_token_cost": 3e-07, @@ -2299,6 +2306,7 @@ }, "us.anthropic.claude-sonnet-4-6": { "supports_adaptive_thinking": true, + "supports_legacy_thinking": true, "cache_creation_input_token_cost": 4.125e-06, "cache_creation_input_token_cost_above_1hr": 6.6e-06, "cache_read_input_token_cost": 3.3e-07, @@ -2332,6 +2340,7 @@ }, "eu.anthropic.claude-sonnet-4-6": { "supports_adaptive_thinking": true, + "supports_legacy_thinking": true, "cache_creation_input_token_cost": 4.125e-06, "cache_creation_input_token_cost_above_1hr": 6.6e-06, "cache_read_input_token_cost": 3.3e-07, @@ -2365,6 +2374,7 @@ }, "au.anthropic.claude-sonnet-4-6": { "supports_adaptive_thinking": true, + "supports_legacy_thinking": true, "cache_creation_input_token_cost": 4.125e-06, "cache_creation_input_token_cost_above_1hr": 6.6e-06, "cache_read_input_token_cost": 3.3e-07, @@ -2398,6 +2408,7 @@ }, "jp.anthropic.claude-sonnet-4-6": { "supports_adaptive_thinking": true, + "supports_legacy_thinking": true, "cache_creation_input_token_cost": 4.125e-06, "cache_creation_input_token_cost_above_1hr": 6.6e-06, "cache_read_input_token_cost": 3.3e-07, @@ -2950,6 +2961,7 @@ "azure_ai/claude-opus-4-6": { "deprecation_date": "2027-02-02", "supports_adaptive_thinking": true, + "supports_legacy_thinking": true, "input_cost_per_token": 5e-06, "output_cost_per_token": 2.5e-05, "litellm_provider": "azure_ai", @@ -3181,6 +3193,7 @@ "azure_ai/claude-sonnet-4-6": { "deprecation_date": "2027-02-10", "supports_adaptive_thinking": true, + "supports_legacy_thinking": true, "cache_creation_input_token_cost": 3.75e-06, "cache_creation_input_token_cost_above_1hr": 6e-06, "cache_read_input_token_cost": 3e-07, @@ -12489,6 +12502,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "supports_legacy_thinking": true, "supports_assistant_prefill": true, "supports_computer_use": true, "supports_function_calling": true, @@ -12698,6 +12712,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "supports_legacy_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -12735,6 +12750,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "supports_legacy_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -14677,6 +14693,7 @@ "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", "supports_assistant_prefill": true, "supports_function_calling": true, + "supports_legacy_thinking": true, "supports_reasoning": true, "supports_tool_choice": true }, @@ -14753,6 +14770,7 @@ "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", "supports_assistant_prefill": true, "supports_function_calling": true, + "supports_legacy_thinking": true, "supports_reasoning": true, "supports_tool_choice": true }, @@ -23180,6 +23198,7 @@ }, "github_copilot/claude-opus-4.6-fast": { "supports_adaptive_thinking": true, + "supports_legacy_thinking": true, "litellm_provider": "github_copilot", "max_input_tokens": 128000, "max_output_tokens": 16000, @@ -33563,6 +33582,7 @@ }, "openrouter/anthropic/claude-sonnet-4.6": { "supports_adaptive_thinking": true, + "supports_legacy_thinking": true, "cache_creation_input_token_cost": 3.75e-06, "cache_creation_input_token_cost_above_200k_tokens": 7.5e-06, "cache_read_input_token_cost": 3e-07, @@ -33607,6 +33627,7 @@ }, "openrouter/anthropic/claude-opus-4.6": { "supports_adaptive_thinking": true, + "supports_legacy_thinking": true, "cache_creation_input_token_cost": 6.25e-06, "cache_read_input_token_cost": 5e-07, "input_cost_per_token": 5e-06, @@ -35681,6 +35702,7 @@ }, "perplexity/anthropic/claude-opus-4-6": { "supports_adaptive_thinking": true, + "supports_legacy_thinking": true, "litellm_provider": "perplexity", "mode": "responses", "supports_web_search": true, @@ -39056,6 +39078,7 @@ }, "vercel_ai_gateway/anthropic/claude-opus-4.6": { "supports_adaptive_thinking": true, + "supports_legacy_thinking": true, "cache_creation_input_token_cost": 6.25e-06, "cache_read_input_token_cost": 5e-07, "input_cost_per_token": 5e-06, @@ -40315,6 +40338,7 @@ "deprecation_date": "2027-02-05", "regional_endpoint_uplift_multiplier": 1.1, "supports_adaptive_thinking": true, + "supports_legacy_thinking": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, @@ -40347,6 +40371,7 @@ "deprecation_date": "2027-02-05", "regional_endpoint_uplift_multiplier": 1.1, "supports_adaptive_thinking": true, + "supports_legacy_thinking": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, @@ -40712,6 +40737,7 @@ "vertex_ai/claude-sonnet-4-6": { "regional_endpoint_uplift_multiplier": 1.1, "supports_adaptive_thinking": true, + "supports_legacy_thinking": true, "cache_creation_input_token_cost": 3.75e-06, "cache_creation_input_token_cost_above_1hr": 6e-06, "cache_read_input_token_cost": 3e-07, @@ -48481,6 +48507,7 @@ "vertex_ai/claude-sonnet-4-6@default": { "regional_endpoint_uplift_multiplier": 1.1, "supports_adaptive_thinking": true, + "supports_legacy_thinking": true, "cache_creation_input_token_cost": 3.75e-06, "cache_creation_input_token_cost_above_1hr": 6e-06, "cache_read_input_token_cost": 3e-07, @@ -49265,6 +49292,7 @@ }, "snowflake/claude-sonnet-4-6": { "supports_adaptive_thinking": true, + "supports_legacy_thinking": true, "max_tokens": 16384, "max_input_tokens": 200000, "max_output_tokens": 16384, @@ -50255,6 +50283,14 @@ "supports_adaptive_thinking": true } }, + { + "name": "claude-legacy-thinking", + "pattern": "claude-[a-z]+-4[-._]6(?!\\d)", + "description": "Claude at version 4.6 exactly, in any id shape that contains claude--4-6 (dotted and underscored minors included, dated releases such as claude-sonnet-4-6-20260219 too). The 4.6 family is adaptive-thinking yet still accepts legacy thinking.type=enabled with budget_tokens, so the caller's hard budget cap is forwarded verbatim instead of being rewritten to an uncapped output_config.effort. The lookahead keeps two-digit minors such as 4-60 from matching. 4.7+ and 5+ majors reject the legacy shape and stay on the adaptive translation.", + "model_info": { + "supports_legacy_thinking": true + } + }, { "name": "claude-always-on-thinking", "pattern": "claude-(?:fable|mythos)-", diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 94526de0757..e8e8edb85b3 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -154,6 +154,7 @@ class ProviderSpecificModelInfo(TypedDict, total=False): supports_web_search: bool | None supports_reasoning: bool | None supports_adaptive_thinking: bool | None + supports_legacy_thinking: ReadOnly[bool | None] thinking_always_on: ReadOnly[bool | None] supports_tool_search: bool | None supports_mid_conversation_system: bool | None diff --git a/litellm/utils.py b/litellm/utils.py index e5ce7157e77..c0e26ef19d4 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -5753,6 +5753,7 @@ def _get_model_info_helper( supports_url_context=_model_info.get("supports_url_context", None), supports_reasoning=_model_info.get("supports_reasoning", None), supports_adaptive_thinking=_model_info.get("supports_adaptive_thinking", None), + supports_legacy_thinking=_model_info.get("supports_legacy_thinking", None), thinking_always_on=_model_info.get("thinking_always_on", None), supports_tool_search=_model_info.get("supports_tool_search", None), supports_mid_conversation_system=_model_info.get("supports_mid_conversation_system", None), diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 3af7d9e5019..dda327bf9f6 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -1019,6 +1019,7 @@ }, "anthropic.claude-opus-4-6-v1": { "supports_adaptive_thinking": true, + "supports_legacy_thinking": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, @@ -1053,6 +1054,7 @@ }, "global.anthropic.claude-opus-4-6-v1": { "supports_adaptive_thinking": true, + "supports_legacy_thinking": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, @@ -1087,6 +1089,7 @@ }, "us.anthropic.claude-opus-4-6-v1": { "supports_adaptive_thinking": true, + "supports_legacy_thinking": true, "cache_creation_input_token_cost": 6.875e-06, "cache_creation_input_token_cost_above_1hr": 1.1e-05, "cache_read_input_token_cost": 5.5e-07, @@ -1121,6 +1124,7 @@ }, "eu.anthropic.claude-opus-4-6-v1": { "supports_adaptive_thinking": true, + "supports_legacy_thinking": true, "cache_creation_input_token_cost": 6.875e-06, "cache_creation_input_token_cost_above_1hr": 1.1e-05, "cache_read_input_token_cost": 5.5e-07, @@ -1155,6 +1159,7 @@ }, "au.anthropic.claude-opus-4-6-v1": { "supports_adaptive_thinking": true, + "supports_legacy_thinking": true, "cache_creation_input_token_cost": 6.875e-06, "cache_creation_input_token_cost_above_1hr": 1.1e-05, "cache_read_input_token_cost": 5.5e-07, @@ -2233,6 +2238,7 @@ }, "anthropic.claude-sonnet-4-6": { "supports_adaptive_thinking": true, + "supports_legacy_thinking": true, "cache_creation_input_token_cost": 3.75e-06, "cache_creation_input_token_cost_above_1hr": 6e-06, "cache_read_input_token_cost": 3e-07, @@ -2266,6 +2272,7 @@ }, "global.anthropic.claude-sonnet-4-6": { "supports_adaptive_thinking": true, + "supports_legacy_thinking": true, "cache_creation_input_token_cost": 3.75e-06, "cache_creation_input_token_cost_above_1hr": 6e-06, "cache_read_input_token_cost": 3e-07, @@ -2299,6 +2306,7 @@ }, "us.anthropic.claude-sonnet-4-6": { "supports_adaptive_thinking": true, + "supports_legacy_thinking": true, "cache_creation_input_token_cost": 4.125e-06, "cache_creation_input_token_cost_above_1hr": 6.6e-06, "cache_read_input_token_cost": 3.3e-07, @@ -2332,6 +2340,7 @@ }, "eu.anthropic.claude-sonnet-4-6": { "supports_adaptive_thinking": true, + "supports_legacy_thinking": true, "cache_creation_input_token_cost": 4.125e-06, "cache_creation_input_token_cost_above_1hr": 6.6e-06, "cache_read_input_token_cost": 3.3e-07, @@ -2365,6 +2374,7 @@ }, "au.anthropic.claude-sonnet-4-6": { "supports_adaptive_thinking": true, + "supports_legacy_thinking": true, "cache_creation_input_token_cost": 4.125e-06, "cache_creation_input_token_cost_above_1hr": 6.6e-06, "cache_read_input_token_cost": 3.3e-07, @@ -2398,6 +2408,7 @@ }, "jp.anthropic.claude-sonnet-4-6": { "supports_adaptive_thinking": true, + "supports_legacy_thinking": true, "cache_creation_input_token_cost": 4.125e-06, "cache_creation_input_token_cost_above_1hr": 6.6e-06, "cache_read_input_token_cost": 3.3e-07, @@ -2950,6 +2961,7 @@ "azure_ai/claude-opus-4-6": { "deprecation_date": "2027-02-02", "supports_adaptive_thinking": true, + "supports_legacy_thinking": true, "input_cost_per_token": 5e-06, "output_cost_per_token": 2.5e-05, "litellm_provider": "azure_ai", @@ -3181,6 +3193,7 @@ "azure_ai/claude-sonnet-4-6": { "deprecation_date": "2027-02-10", "supports_adaptive_thinking": true, + "supports_legacy_thinking": true, "cache_creation_input_token_cost": 3.75e-06, "cache_creation_input_token_cost_above_1hr": 6e-06, "cache_read_input_token_cost": 3e-07, @@ -12489,6 +12502,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "supports_legacy_thinking": true, "supports_assistant_prefill": true, "supports_computer_use": true, "supports_function_calling": true, @@ -12698,6 +12712,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "supports_legacy_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -12735,6 +12750,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "supports_legacy_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -14677,6 +14693,7 @@ "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", "supports_assistant_prefill": true, "supports_function_calling": true, + "supports_legacy_thinking": true, "supports_reasoning": true, "supports_tool_choice": true }, @@ -14753,6 +14770,7 @@ "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", "supports_assistant_prefill": true, "supports_function_calling": true, + "supports_legacy_thinking": true, "supports_reasoning": true, "supports_tool_choice": true }, @@ -23180,6 +23198,7 @@ }, "github_copilot/claude-opus-4.6-fast": { "supports_adaptive_thinking": true, + "supports_legacy_thinking": true, "litellm_provider": "github_copilot", "max_input_tokens": 128000, "max_output_tokens": 16000, @@ -33563,6 +33582,7 @@ }, "openrouter/anthropic/claude-sonnet-4.6": { "supports_adaptive_thinking": true, + "supports_legacy_thinking": true, "cache_creation_input_token_cost": 3.75e-06, "cache_creation_input_token_cost_above_200k_tokens": 7.5e-06, "cache_read_input_token_cost": 3e-07, @@ -33607,6 +33627,7 @@ }, "openrouter/anthropic/claude-opus-4.6": { "supports_adaptive_thinking": true, + "supports_legacy_thinking": true, "cache_creation_input_token_cost": 6.25e-06, "cache_read_input_token_cost": 5e-07, "input_cost_per_token": 5e-06, @@ -35681,6 +35702,7 @@ }, "perplexity/anthropic/claude-opus-4-6": { "supports_adaptive_thinking": true, + "supports_legacy_thinking": true, "litellm_provider": "perplexity", "mode": "responses", "supports_web_search": true, @@ -39056,6 +39078,7 @@ }, "vercel_ai_gateway/anthropic/claude-opus-4.6": { "supports_adaptive_thinking": true, + "supports_legacy_thinking": true, "cache_creation_input_token_cost": 6.25e-06, "cache_read_input_token_cost": 5e-07, "input_cost_per_token": 5e-06, @@ -40315,6 +40338,7 @@ "deprecation_date": "2027-02-05", "regional_endpoint_uplift_multiplier": 1.1, "supports_adaptive_thinking": true, + "supports_legacy_thinking": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, @@ -40347,6 +40371,7 @@ "deprecation_date": "2027-02-05", "regional_endpoint_uplift_multiplier": 1.1, "supports_adaptive_thinking": true, + "supports_legacy_thinking": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, @@ -40712,6 +40737,7 @@ "vertex_ai/claude-sonnet-4-6": { "regional_endpoint_uplift_multiplier": 1.1, "supports_adaptive_thinking": true, + "supports_legacy_thinking": true, "cache_creation_input_token_cost": 3.75e-06, "cache_creation_input_token_cost_above_1hr": 6e-06, "cache_read_input_token_cost": 3e-07, @@ -48481,6 +48507,7 @@ "vertex_ai/claude-sonnet-4-6@default": { "regional_endpoint_uplift_multiplier": 1.1, "supports_adaptive_thinking": true, + "supports_legacy_thinking": true, "cache_creation_input_token_cost": 3.75e-06, "cache_creation_input_token_cost_above_1hr": 6e-06, "cache_read_input_token_cost": 3e-07, @@ -49265,6 +49292,7 @@ }, "snowflake/claude-sonnet-4-6": { "supports_adaptive_thinking": true, + "supports_legacy_thinking": true, "max_tokens": 16384, "max_input_tokens": 200000, "max_output_tokens": 16384, @@ -50255,6 +50283,14 @@ "supports_adaptive_thinking": true } }, + { + "name": "claude-legacy-thinking", + "pattern": "claude-[a-z]+-4[-._]6(?!\\d)", + "description": "Claude at version 4.6 exactly, in any id shape that contains claude--4-6 (dotted and underscored minors included, dated releases such as claude-sonnet-4-6-20260219 too). The 4.6 family is adaptive-thinking yet still accepts legacy thinking.type=enabled with budget_tokens, so the caller's hard budget cap is forwarded verbatim instead of being rewritten to an uncapped output_config.effort. The lookahead keeps two-digit minors such as 4-60 from matching. 4.7+ and 5+ majors reject the legacy shape and stay on the adaptive translation.", + "model_info": { + "supports_legacy_thinking": true + } + }, { "name": "claude-always-on-thinking", "pattern": "claude-(?:fable|mythos)-", diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_reasoning_effort_translation.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_reasoning_effort_translation.py index f393a7b50b1..12ab536ed45 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_reasoning_effort_translation.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_reasoning_effort_translation.py @@ -2,7 +2,6 @@ import pytest -import litellm from litellm.constants import ( DEFAULT_REASONING_EFFORT_HIGH_THINKING_BUDGET, DEFAULT_REASONING_EFFORT_MEDIUM_THINKING_BUDGET, @@ -17,7 +16,6 @@ from litellm.llms.bedrock.messages.invoke_transformations.anthropic_claude3_tran ) - @pytest.mark.parametrize( "reasoning_effort,expected_effort", [ @@ -258,19 +256,22 @@ def test_reasoning_effort_in_supported_params(): "model", [ "claude-sonnet-4-6", - "bedrock/invoke/us.anthropic.claude-sonnet-4-6", - "vertex_ai/claude-sonnet-4-6", "claude-opus-4-6", + "claude-sonnet-4-6-20260219", + "bedrock/invoke/us.anthropic.claude-sonnet-4-6", "bedrock/invoke/us.anthropic.claude-opus-4-6-v1:0", + "vertex_ai/claude-sonnet-4-6", "vertex_ai/claude-opus-4-6", + "azure_ai/claude-sonnet-4-6", ], ) -def test_legacy_thinking_high_budget_clamps_to_high_when_xhigh_unsupported( - local_model_cost_map, model -): - """Claude Code sends ``thinking.budget_tokens=31999``; Sonnet 4.6 and Opus 4.6 - have no ``xhigh`` tier, so the translator must emit ``high`` rather than the - provider-invalid ``xhigh`` (regression for issue #29282).""" +def test_legacy_thinking_budget_preserved_verbatim_on_46(local_model_cost_map, model): + """Regression for the passthrough silently dropping a caller's hard thinking + budget: the 4.6 family accepts ``thinking.type=enabled`` with ``budget_tokens`` + natively, so rewriting it to ``thinking.type=adaptive`` + ``output_config.effort`` + (which carries no ceiling) let reasoning run past the requested cap. The legacy + shape must be forwarded verbatim, in every 4.6 id shape including unmapped dated + releases resolved by the ``claude-legacy-thinking`` fallback rule.""" config = AnthropicMessagesConfig() optional_params = { "max_tokens": 1024, @@ -285,8 +286,8 @@ def test_legacy_thinking_high_budget_clamps_to_high_when_xhigh_unsupported( headers={}, ) - assert result.get("thinking") == {"type": "adaptive"} - assert result.get("output_config") == {"effort": "high"} + assert result.get("thinking") == {"type": "enabled", "budget_tokens": 31999} + assert "output_config" not in result def test_legacy_thinking_high_budget_keeps_xhigh_when_supported(): @@ -343,11 +344,44 @@ def test_legacy_thinking_translates_to_adaptive_for_opus_48( assert result.get("output_config") == {"effort": "xhigh"} +@pytest.mark.parametrize( + "model,expected_effort", + [ + ("claude-sonnet-5", "xhigh"), + ("claude-opus-5", "xhigh"), + ("claude-newfamily-6", "high"), + ], +) +def test_legacy_thinking_translates_to_adaptive_for_5_and_future_models( + local_model_cost_map, model, expected_effort +): + """The 5 families reject ``thinking.type=enabled``, so the adaptive translation + stays the safe default for every adaptive model not flagged + ``supports_legacy_thinking``, unmapped future ids included. An unmapped id + cannot prove ``xhigh`` support, so its high-budget bucket clamps to ``high``.""" + config = AnthropicMessagesConfig() + optional_params = { + "max_tokens": 1024, + "thinking": {"type": "enabled", "budget_tokens": 31999}, + } + + result = config.transform_anthropic_messages_request( + model=model, + messages=[{"role": "user", "content": "Hello"}], + anthropic_messages_optional_request_params=optional_params, + litellm_params={}, + headers={}, + ) + + assert result.get("thinking") == {"type": "adaptive"} + assert result.get("output_config") == {"effort": expected_effort} + + @pytest.mark.parametrize( "budget_tokens,expected_effort", [ - (DEFAULT_REASONING_EFFORT_XHIGH_THINKING_BUDGET * 2, "high"), - (DEFAULT_REASONING_EFFORT_XHIGH_THINKING_BUDGET, "high"), + (DEFAULT_REASONING_EFFORT_XHIGH_THINKING_BUDGET * 2, "xhigh"), + (DEFAULT_REASONING_EFFORT_XHIGH_THINKING_BUDGET, "xhigh"), (DEFAULT_REASONING_EFFORT_HIGH_THINKING_BUDGET, "high"), (DEFAULT_REASONING_EFFORT_HIGH_THINKING_BUDGET - 1, "medium"), (DEFAULT_REASONING_EFFORT_MEDIUM_THINKING_BUDGET, "medium"), @@ -355,7 +389,9 @@ def test_legacy_thinking_translates_to_adaptive_for_opus_48( (1, "low"), ], ) -def test_legacy_thinking_budget_buckets_on_sonnet_46(budget_tokens, expected_effort): +def test_legacy_thinking_budget_buckets_on_opus_48( + local_model_cost_map, budget_tokens, expected_effort +): config = AnthropicMessagesConfig() optional_params = { "max_tokens": 1024, @@ -363,7 +399,7 @@ def test_legacy_thinking_budget_buckets_on_sonnet_46(budget_tokens, expected_eff } result = config.transform_anthropic_messages_request( - model="claude-sonnet-4-6", + model="claude-opus-4-8", messages=[{"role": "user", "content": "Hello"}], anthropic_messages_optional_request_params=optional_params, litellm_params={}, @@ -373,7 +409,29 @@ def test_legacy_thinking_budget_buckets_on_sonnet_46(budget_tokens, expected_eff assert result.get("output_config") == {"effort": expected_effort} -def test_legacy_thinking_does_not_override_explicit_output_config(): +def test_legacy_thinking_does_not_override_explicit_output_config(local_model_cost_map): + config = AnthropicMessagesConfig() + optional_params = { + "max_tokens": 1024, + "thinking": {"type": "enabled", "budget_tokens": 31999}, + "output_config": {"effort": "low"}, + } + + result = config.transform_anthropic_messages_request( + model="claude-opus-4-8", + messages=[{"role": "user", "content": "Hello"}], + anthropic_messages_optional_request_params=optional_params, + litellm_params={}, + headers={}, + ) + + assert result.get("thinking") == {"type": "adaptive"} + assert result.get("output_config") == {"effort": "low"} + + +def test_legacy_thinking_with_explicit_output_config_untouched_on_46( + local_model_cost_map, +): config = AnthropicMessagesConfig() optional_params = { "max_tokens": 1024, @@ -389,6 +447,7 @@ def test_legacy_thinking_does_not_override_explicit_output_config(): headers={}, ) + assert result.get("thinking") == {"type": "enabled", "budget_tokens": 31999} assert result.get("output_config") == {"effort": "low"} From 6407a6637510126177806fe503376d5d457f1ae4 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 24 Aug 2026 11:23:03 -0700 Subject: [PATCH 08/14] fix(runwayml): route every generation endpoint and fix video cost tracking Six defects in the RunwayML video provider: - transform_video_create_request hardcoded /image_to_video, so text-to-video 400'd and video-to-video was unreachable; the endpoint is now selected from the inputs present (promptVideo/videoUri, promptImage, or text only) - get_error_class raised instead of returning, turning a provider 4xx into a proxy 500 APIConnectionError; it now returns a RunwayMLError - VideoObject.progress was typed int while Runway sends a 0..1 float, 500'ing status polls while RUNNING; it is now scaled to a 0..100 percent - custom per-deployment pricing stored under litellm_metadata was ignored for video; the deployment model_info lookup now checks both metadata keys - stale cost-map entries (gen3a_turbo, gen4_aleph) were removed and current models added, with output_cost_per_second_480p/_4k tier keys plumbed through the model-info and router types - video cost now falls back to Runway's estimatedCost from the create response when no custom pricing is configured, and custom pricing always wins over it Fixes #36483 --- litellm/cost_calculator.py | 18 ++- .../exception_mapping_utils.py | 1 + litellm/llms/openai/cost_calculation.py | 9 +- .../llms/runwayml/videos/transformation.py | 115 +++++++++---- ...odel_prices_and_context_window_backup.json | 151 +++++++++++++++++- litellm/types/router.py | 4 +- litellm/types/utils.py | 4 + litellm/utils.py | 2 + model_prices_and_context_window.json | 151 +++++++++++++++++- .../test_exception_mapping_utils.py | 3 + .../test_runway_video_transformation.py | 140 +++++++++++++++- tests/test_litellm/test_utils.py | 5 + tests/test_litellm/test_video_generation.py | 111 +++++++++++++ ui/litellm-dashboard/src/lib/http/schema.d.ts | 8 + 14 files changed, 667 insertions(+), 55 deletions(-) diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 8f7cd09d364..58d29d99e3e 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -150,6 +150,7 @@ _VIDEO_CALL_TYPES: Final = frozenset( } ) + _SPEECH_CALL_TYPES: Final = frozenset( { CallTypes.speech.value, @@ -1372,23 +1373,36 @@ def completion_cost( if custom_pricing and litellm_logging_obj is not None: _litellm_params = getattr(litellm_logging_obj, "litellm_params", None) if _litellm_params is not None: - _metadata = _litellm_params.get("metadata", {}) or {} - _video_model_info = _metadata.get("model_info", None) + _video_model_info = next( + ( + model_info + for _metadata_key in ("metadata", "litellm_metadata") + if (model_info := (_litellm_params.get(_metadata_key) or {}).get("model_info")) + is not None + ), + None, + ) usage_obj = getattr(completion_response, "usage", None) duration_seconds: float | None = None video_resolution: str | None = None + provider_reported_cost: float | None = None if completion_response is not None and usage_obj: # Handle both dict and Pydantic Usage object if isinstance(usage_obj, dict): duration_seconds = usage_obj.get("duration_seconds", None) _vr = usage_obj.get("video_resolution", None) + provider_reported_cost = usage_obj.get("provider_reported_cost_usd", None) else: duration_seconds = getattr(usage_obj, "duration_seconds", None) _vr = getattr(usage_obj, "video_resolution", None) + provider_reported_cost = getattr(usage_obj, "provider_reported_cost_usd", None) if _vr is not None: video_resolution = str(_vr).strip().lower() + if _video_model_info is None and provider_reported_cost is not None: + return float(provider_reported_cost) + if duration_seconds is not None: # Calculate cost based on video duration using video-specific cost calculation from litellm.llms.openai.cost_calculation import ( diff --git a/litellm/litellm_core_utils/exception_mapping_utils.py b/litellm/litellm_core_utils/exception_mapping_utils.py index 4a25eb218c0..4ee726b67de 100644 --- a/litellm/litellm_core_utils/exception_mapping_utils.py +++ b/litellm/litellm_core_utils/exception_mapping_utils.py @@ -2301,6 +2301,7 @@ def exception_type( or custom_llm_provider == "custom_openai" or custom_llm_provider in litellm.openai_compatible_providers or custom_llm_provider == "mistral" + or custom_llm_provider == "runwayml" ): _map_openai_exception( model=model, diff --git a/litellm/llms/openai/cost_calculation.py b/litellm/llms/openai/cost_calculation.py index 0352d246c09..115b2e27983 100644 --- a/litellm/llms/openai/cost_calculation.py +++ b/litellm/llms/openai/cost_calculation.py @@ -134,14 +134,7 @@ def cost_per_second(model: str, custom_llm_provider: str | None, duration: float def _video_resolution_to_cost_field_suffix(resolution: str) -> str | None: - """ - Map usage resolution to a safe suffix for ``output_cost_per_second_`` keys. - - Note: Currently only ``output_cost_per_second_1080p`` is explicitly declared in - ModelInfo (types/utils.py). Other resolution tiers (e.g., 720p, 4k) can be added - to model_prices_and_context_window.json but are not exposed via get_model_info() - until added to the ModelInfo TypedDict. - """ + """Map usage resolution to a safe suffix for ``output_cost_per_second_`` keys.""" r: Final = resolution.strip().lower() if not r: return None diff --git a/litellm/llms/runwayml/videos/transformation.py b/litellm/llms/runwayml/videos/transformation.py index b8e57fa7cc0..065661a8731 100644 --- a/litellm/llms/runwayml/videos/transformation.py +++ b/litellm/llms/runwayml/videos/transformation.py @@ -1,5 +1,6 @@ from collections.abc import Mapping, Sequence from datetime import datetime +from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Literal import httpx @@ -33,6 +34,10 @@ else: LiteLLMLoggingObj = Any +class RunwayMLError(BaseLLMException): + pass + + class _RunwayTaskResponse(TypedDict, total=False): id: ReadOnly[str] status: ReadOnly[str] @@ -41,7 +46,8 @@ class _RunwayTaskResponse(TypedDict, total=False): output: ReadOnly[Sequence[str] | str] failureCode: ReadOnly[str] failure: ReadOnly[str] - progress: ReadOnly[int] + progress: ReadOnly[float] + estimatedCost: ReadOnly[Mapping[str, float]] class _VideoObjectData(TypedDict, extra_items=object): @@ -56,12 +62,54 @@ def _parse_runway_task_response(raw_response: httpx.Response) -> _RunwayTaskResp return response_data +_USD_PER_CREDIT: Final = 0.01 + +_RESOLUTION_AREA_TIERS: Final[tuple[tuple[int, str], ...]] = ( + (600_000, "480p"), + (1_500_000, "720p"), + (4_000_000, "1080p"), +) + + +def _ratio_to_resolution(ratio: object) -> str | None: + if not isinstance(ratio, str) or ":" not in ratio: + return None + width_str, _, height_str = ratio.partition(":") + if not (width_str.isdigit() and height_str.isdigit()): + return None + area: Final = int(width_str) * int(height_str) + return next((label for threshold, label in _RESOLUTION_AREA_TIERS if area < threshold), "4k") + + +def _duration_seconds(seconds: str | None) -> float | None: + if not seconds: + return None + try: + return float(seconds) + except ValueError: + return None + + +def _estimated_cost_usd(response_data: _RunwayTaskResponse) -> float | None: + estimated_cost: Final = response_data.get("estimatedCost") + if not isinstance(estimated_cost, Mapping): + return None + credits: Final = estimated_cost.get("credits") + if not isinstance(credits, (int, float)): + return None + return float(credits) * _USD_PER_CREDIT + + +def _progress_percent(progress: float) -> int: + return min(100, max(0, round(float(progress) * 100))) + + class RunwayMLVideoConfig(BaseVideoConfig): """ Configuration class for RunwayML video generation. RunwayML uses a task-based API where: - 1. POST /v1/image_to_video creates a task + 1. POST /v1/text_to_video, /v1/image_to_video, or /v1/video_to_video creates a task 2. The task returns immediately with a task ID 3. Client must poll or wait for task completion """ @@ -195,31 +243,36 @@ class RunwayMLVideoConfig(BaseVideoConfig): """ Transform the video creation request for RunwayML API. - RunwayML expects: - { - "model": "gen4_turbo", - "promptImage": "https://... or data:image/...", - "promptText": "description", - "ratio": "1280:720", - "duration": 5 - } + RunwayML has three generation endpoints discriminated by which input is + present, and each request body rejects unknown fields: + - /text_to_video: promptText only (rejects promptImage) + - /image_to_video: promptImage (+ optional promptText) + - /video_to_video: promptVideo or videoUri (rejects promptImage) """ - # Build the request data + merged_params: Final = MappingProxyType( + { + "model": model, + "promptText": prompt, + **video_create_optional_request_params, + } + ) + + endpoint: Final = self._select_generation_endpoint(merged_params) + request_data: Final[dict[str, object]] = { - "model": model, - "promptText": prompt, + key: value for key, value in merged_params.items() if endpoint == "image_to_video" or key != "promptImage" } - # Add mapped parameters - request_data.update(video_create_optional_request_params) - - # RunwayML uses JSON body, no files multipart files_list: Final[RequestFiles] = [] - # Append the specific endpoint for video generation - full_api_base: Final = f"{api_base}/image_to_video" + return request_data, files_list, f"{api_base}/{endpoint}" - return request_data, files_list, full_api_base + def _select_generation_endpoint(self, request_data: Mapping[str, object]) -> str: + if request_data.get("promptVideo") is not None or request_data.get("videoUri") is not None: + return "video_to_video" + if request_data.get("promptImage") is not None: + return "image_to_video" + return "text_to_video" def transform_video_create_response( self, @@ -285,13 +338,15 @@ class RunwayMLVideoConfig(BaseVideoConfig): video_obj.id = encode_video_id_with_provider(video_obj.id, custom_llm_provider, model) # Add usage data for cost tracking - usage_data: Final = {} - if video_obj and hasattr(video_obj, "seconds") and video_obj.seconds: - try: - usage_data["duration_seconds"] = float(video_obj.seconds) - except (ValueError, TypeError): - pass - video_obj.usage = usage_data + video_obj.usage = { + key: value + for key, value in ( + ("duration_seconds", _duration_seconds(video_obj.seconds)), + ("video_resolution", _ratio_to_resolution(request_data.get("ratio") if request_data else None)), + ("provider_reported_cost_usd", _estimated_cost_usd(response_data)), + ) + if value is not None + } return video_obj @@ -582,7 +637,7 @@ class RunwayMLVideoConfig(BaseVideoConfig): video_data["completed_at"] = self._parse_runway_timestamp(response_data.get("completedAt")) if "progress" in response_data: - video_data["progress"] = response_data["progress"] + video_data["progress"] = _progress_percent(response_data["progress"]) if "failureCode" in response_data or "failure" in response_data: video_data["error"] = { @@ -646,9 +701,7 @@ class RunwayMLVideoConfig(BaseVideoConfig): raise NotImplementedError("video extension is not supported for RunwayML") def get_error_class(self, error_message: str, status_code: int, headers: dict | httpx.Headers) -> BaseLLMException: - from ...base_llm.chat.transformation import BaseLLMException - - raise BaseLLMException( + return RunwayMLError( status_code=status_code, message=error_message, headers=headers, diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 3af7d9e5019..3c80e645d6b 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -43795,10 +43795,10 @@ "comment": "5 credits per second @ $0.01 per credit = $0.05 per second" } }, - "runwayml/gen4_aleph": { + "runwayml/gen4.5": { "litellm_provider": "runwayml", "mode": "video_generation", - "output_cost_per_video_per_second": 0.15, + "output_cost_per_second": 0.12, "source": "https://docs.dev.runwayml.com/guides/pricing/", "supported_modalities": [ "text", @@ -43808,13 +43808,136 @@ "video" ], "metadata": { - "comment": "15 credits per second @ $0.01 per credit = $0.15 per second" + "comment": "12 credits per second @ $0.01 per credit = $0.12 per second" } }, - "runwayml/gen3a_turbo": { + "runwayml/aleph2": { "litellm_provider": "runwayml", "mode": "video_generation", - "output_cost_per_video_per_second": 0.05, + "output_cost_per_second": 0.28, + "source": "https://docs.dev.runwayml.com/guides/pricing/", + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ], + "metadata": { + "comment": "28 credits per second @ $0.01 per credit = $0.28 per second; 56 credit minimum per task not modeled" + } + }, + "runwayml/seedance2": { + "litellm_provider": "runwayml", + "mode": "video_generation", + "output_cost_per_second": 0.36, + "output_cost_per_second_1080p": 0.4, + "output_cost_per_second_4k": 1.5, + "source": "https://docs.dev.runwayml.com/guides/pricing/", + "supported_modalities": [ + "text", + "image", + "video" + ], + "supported_output_modalities": [ + "video" + ], + "metadata": { + "comment": "36 credits per second at 480p/720p, 40 at 1080p, 150 at 4K @ $0.01 per credit" + } + }, + "runwayml/seedance2_fast": { + "litellm_provider": "runwayml", + "mode": "video_generation", + "output_cost_per_second": 0.29, + "source": "https://docs.dev.runwayml.com/guides/pricing/", + "supported_modalities": [ + "text", + "image", + "video" + ], + "supported_output_modalities": [ + "video" + ], + "metadata": { + "comment": "29 credits per second at 480p/720p @ $0.01 per credit = $0.29 per second" + } + }, + "runwayml/seedance2_mini": { + "litellm_provider": "runwayml", + "mode": "video_generation", + "output_cost_per_second": 0.16, + "source": "https://docs.dev.runwayml.com/guides/pricing/", + "supported_modalities": [ + "text", + "image", + "video" + ], + "supported_output_modalities": [ + "video" + ], + "metadata": { + "comment": "16 credits per second @ $0.01 per credit = $0.16 per second; 64 credit minimum per task not modeled" + } + }, + "runwayml/seedance2_5": { + "litellm_provider": "runwayml", + "mode": "video_generation", + "output_cost_per_second": 0.3, + "output_cost_per_second_480p": 0.2, + "output_cost_per_second_1080p": 0.68, + "source": "https://docs.dev.runwayml.com/guides/pricing/", + "supported_modalities": [ + "text", + "image", + "video" + ], + "supported_output_modalities": [ + "video" + ], + "metadata": { + "comment": "Output: 20/30/68 credits per second at 480p/720p/1080p @ $0.01 per credit; input video billed additionally at 10/15/34 credits per input second and the 80 credit minimum per task are not modeled" + } + }, + "runwayml/hailuo3": { + "litellm_provider": "runwayml", + "mode": "video_generation", + "output_cost_per_second": 0.1, + "output_cost_per_second_1080p": 0.15, + "source": "https://docs.dev.runwayml.com/guides/pricing/", + "supported_modalities": [ + "text", + "image", + "video" + ], + "supported_output_modalities": [ + "video" + ], + "metadata": { + "comment": "10 credits per second at 768P, 15 at 2K (mapped to the 1080p tier) @ $0.01 per credit; 2 credits per reference image not modeled" + } + }, + "runwayml/gemini_omni_flash": { + "litellm_provider": "runwayml", + "mode": "video_generation", + "output_cost_per_second": 0.1, + "source": "https://docs.dev.runwayml.com/guides/pricing/", + "supported_modalities": [ + "text", + "image", + "video" + ], + "supported_output_modalities": [ + "video" + ], + "metadata": { + "comment": "10 credits per second @ $0.01 per credit = $0.10 per second" + } + }, + "runwayml/veo3.1": { + "litellm_provider": "runwayml", + "mode": "video_generation", + "output_cost_per_second": 0.4, "source": "https://docs.dev.runwayml.com/guides/pricing/", "supported_modalities": [ "text", @@ -43824,7 +43947,23 @@ "video" ], "metadata": { - "comment": "5 credits per second @ $0.01 per credit = $0.05 per second" + "comment": "40 credits per second with audio, 20 without @ $0.01 per credit; priced at the with-audio rate" + } + }, + "runwayml/veo3.1_fast": { + "litellm_provider": "runwayml", + "mode": "video_generation", + "output_cost_per_second": 0.15, + "source": "https://docs.dev.runwayml.com/guides/pricing/", + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ], + "metadata": { + "comment": "15 credits per second with audio, 10 without @ $0.01 per credit; priced at the with-audio rate" } }, "runwayml/gen4_image": { diff --git a/litellm/types/router.py b/litellm/types/router.py index 99a4603ae49..9fd5cfa96ef 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -10,7 +10,7 @@ from typing import Any, ClassVar, Final, Generic, Literal, TypeVar, get_type_hin import httpx from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator -from typing_extensions import Protocol, Required, TypedDict, runtime_checkable +from typing_extensions import Protocol, ReadOnly, Required, TypedDict, runtime_checkable from litellm._uuid import uuid @@ -480,7 +480,9 @@ class LiteLLMParamsTypedDict(TypedDict, total=False): output_cost_per_token: float | None input_cost_per_second: float | None output_cost_per_second: float | None + output_cost_per_second_480p: ReadOnly[float | None] output_cost_per_second_1080p: float | None + output_cost_per_second_4k: ReadOnly[float | None] num_retries: int | None ## MOCK RESPONSES ## mock_response: str | ModelResponse | Exception | None diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 67eae2b4f21..b9e8518a683 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -277,6 +277,8 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False): output_cost_per_second_1080p: ( float | None ) # video_generation tier: key output_cost_per_second_ (e.g. 1080p, 720p) + output_cost_per_second_480p: ReadOnly[float | None] + output_cost_per_second_4k: ReadOnly[float | None] ocr_cost_per_page: float | None # for OCR models ocr_cost_per_credit: float | None # for OCR models priced by credit annotation_cost_per_page: float | None # for OCR models @@ -3331,6 +3333,8 @@ class CustomPricingLiteLLMParams(MirroredPricingParams): input_cost_per_second: float | None = None output_cost_per_second: float | None = None output_cost_per_second_1080p: float | None = None + output_cost_per_second_480p: float | None = None + output_cost_per_second_4k: float | None = None input_cost_per_pixel: float | None = None output_cost_per_pixel: float | None = None diff --git a/litellm/utils.py b/litellm/utils.py index e5ce7157e77..1d9dec6cfe8 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -5726,6 +5726,8 @@ def _get_model_info_helper( ), output_cost_per_second=_model_info.get("output_cost_per_second", None), output_cost_per_second_1080p=_model_info.get("output_cost_per_second_1080p", None), + output_cost_per_second_480p=_model_info.get("output_cost_per_second_480p", None), + output_cost_per_second_4k=_model_info.get("output_cost_per_second_4k", None), output_cost_per_video_per_second=_model_info.get("output_cost_per_video_per_second", None), output_cost_per_image=_model_info.get("output_cost_per_image", None), output_cost_per_image_token=_model_info.get("output_cost_per_image_token", None), diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 3af7d9e5019..3c80e645d6b 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -43795,10 +43795,10 @@ "comment": "5 credits per second @ $0.01 per credit = $0.05 per second" } }, - "runwayml/gen4_aleph": { + "runwayml/gen4.5": { "litellm_provider": "runwayml", "mode": "video_generation", - "output_cost_per_video_per_second": 0.15, + "output_cost_per_second": 0.12, "source": "https://docs.dev.runwayml.com/guides/pricing/", "supported_modalities": [ "text", @@ -43808,13 +43808,136 @@ "video" ], "metadata": { - "comment": "15 credits per second @ $0.01 per credit = $0.15 per second" + "comment": "12 credits per second @ $0.01 per credit = $0.12 per second" } }, - "runwayml/gen3a_turbo": { + "runwayml/aleph2": { "litellm_provider": "runwayml", "mode": "video_generation", - "output_cost_per_video_per_second": 0.05, + "output_cost_per_second": 0.28, + "source": "https://docs.dev.runwayml.com/guides/pricing/", + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ], + "metadata": { + "comment": "28 credits per second @ $0.01 per credit = $0.28 per second; 56 credit minimum per task not modeled" + } + }, + "runwayml/seedance2": { + "litellm_provider": "runwayml", + "mode": "video_generation", + "output_cost_per_second": 0.36, + "output_cost_per_second_1080p": 0.4, + "output_cost_per_second_4k": 1.5, + "source": "https://docs.dev.runwayml.com/guides/pricing/", + "supported_modalities": [ + "text", + "image", + "video" + ], + "supported_output_modalities": [ + "video" + ], + "metadata": { + "comment": "36 credits per second at 480p/720p, 40 at 1080p, 150 at 4K @ $0.01 per credit" + } + }, + "runwayml/seedance2_fast": { + "litellm_provider": "runwayml", + "mode": "video_generation", + "output_cost_per_second": 0.29, + "source": "https://docs.dev.runwayml.com/guides/pricing/", + "supported_modalities": [ + "text", + "image", + "video" + ], + "supported_output_modalities": [ + "video" + ], + "metadata": { + "comment": "29 credits per second at 480p/720p @ $0.01 per credit = $0.29 per second" + } + }, + "runwayml/seedance2_mini": { + "litellm_provider": "runwayml", + "mode": "video_generation", + "output_cost_per_second": 0.16, + "source": "https://docs.dev.runwayml.com/guides/pricing/", + "supported_modalities": [ + "text", + "image", + "video" + ], + "supported_output_modalities": [ + "video" + ], + "metadata": { + "comment": "16 credits per second @ $0.01 per credit = $0.16 per second; 64 credit minimum per task not modeled" + } + }, + "runwayml/seedance2_5": { + "litellm_provider": "runwayml", + "mode": "video_generation", + "output_cost_per_second": 0.3, + "output_cost_per_second_480p": 0.2, + "output_cost_per_second_1080p": 0.68, + "source": "https://docs.dev.runwayml.com/guides/pricing/", + "supported_modalities": [ + "text", + "image", + "video" + ], + "supported_output_modalities": [ + "video" + ], + "metadata": { + "comment": "Output: 20/30/68 credits per second at 480p/720p/1080p @ $0.01 per credit; input video billed additionally at 10/15/34 credits per input second and the 80 credit minimum per task are not modeled" + } + }, + "runwayml/hailuo3": { + "litellm_provider": "runwayml", + "mode": "video_generation", + "output_cost_per_second": 0.1, + "output_cost_per_second_1080p": 0.15, + "source": "https://docs.dev.runwayml.com/guides/pricing/", + "supported_modalities": [ + "text", + "image", + "video" + ], + "supported_output_modalities": [ + "video" + ], + "metadata": { + "comment": "10 credits per second at 768P, 15 at 2K (mapped to the 1080p tier) @ $0.01 per credit; 2 credits per reference image not modeled" + } + }, + "runwayml/gemini_omni_flash": { + "litellm_provider": "runwayml", + "mode": "video_generation", + "output_cost_per_second": 0.1, + "source": "https://docs.dev.runwayml.com/guides/pricing/", + "supported_modalities": [ + "text", + "image", + "video" + ], + "supported_output_modalities": [ + "video" + ], + "metadata": { + "comment": "10 credits per second @ $0.01 per credit = $0.10 per second" + } + }, + "runwayml/veo3.1": { + "litellm_provider": "runwayml", + "mode": "video_generation", + "output_cost_per_second": 0.4, "source": "https://docs.dev.runwayml.com/guides/pricing/", "supported_modalities": [ "text", @@ -43824,7 +43947,23 @@ "video" ], "metadata": { - "comment": "5 credits per second @ $0.01 per credit = $0.05 per second" + "comment": "40 credits per second with audio, 20 without @ $0.01 per credit; priced at the with-audio rate" + } + }, + "runwayml/veo3.1_fast": { + "litellm_provider": "runwayml", + "mode": "video_generation", + "output_cost_per_second": 0.15, + "source": "https://docs.dev.runwayml.com/guides/pricing/", + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ], + "metadata": { + "comment": "15 credits per second with audio, 10 without @ $0.01 per credit; priced at the with-audio rate" } }, "runwayml/gen4_image": { diff --git a/tests/test_litellm/litellm_core_utils/test_exception_mapping_utils.py b/tests/test_litellm/litellm_core_utils/test_exception_mapping_utils.py index 599ad016827..6f7ea9da640 100644 --- a/tests/test_litellm/litellm_core_utils/test_exception_mapping_utils.py +++ b/tests/test_litellm/litellm_core_utils/test_exception_mapping_utils.py @@ -867,6 +867,7 @@ PROVIDERS_WITH_A_HANDLER = ( "openrouter", "perplexity", "replicate", + "runwayml", "sagemaker", "together_ai", "vertex_ai", @@ -956,6 +957,7 @@ PROVIDERS_THAT_RECOGNISE_A_FULL_CONTEXT_WINDOW = ( "mistral", "openai", "perplexity", + "runwayml", "together_ai", "vertex_ai", "xai", @@ -971,6 +973,7 @@ PROVIDERS_THAT_RECOGNISE_A_CONTENT_POLICY_BLOCK = ( "mistral", "openai", "perplexity", + "runwayml", "together_ai", "xai", ) diff --git a/tests/test_litellm/llms/runwayml/videos/test_runway_video_transformation.py b/tests/test_litellm/llms/runwayml/videos/test_runway_video_transformation.py index 24879ce83f9..e52cf9211ec 100644 --- a/tests/test_litellm/llms/runwayml/videos/test_runway_video_transformation.py +++ b/tests/test_litellm/llms/runwayml/videos/test_runway_video_transformation.py @@ -7,7 +7,12 @@ from unittest.mock import Mock import httpx import pytest -from litellm.llms.runwayml.videos.transformation import RunwayMLVideoConfig +from litellm.llms.base_llm.chat.transformation import BaseLLMException +from litellm.llms.runwayml.videos.transformation import ( + RunwayMLError, + RunwayMLVideoConfig, + _ratio_to_resolution, +) from litellm.types.router import GenericLiteLLMParams from litellm.types.videos.main import VideoObject @@ -49,6 +54,139 @@ class TestRunwayMLVideoTransformation: # Validate URL has correct endpoint assert url == "https://api.dev.runwayml.com/v1/image_to_video" + def test_transform_video_create_request_text_to_video(self): + """A prompt-only request must hit /text_to_video, not /image_to_video.""" + data, files, url = self.config.transform_video_create_request( + model="veo3.1", + prompt="A serene mountain lake at sunrise", + api_base="https://api.dev.runwayml.com/v1", + video_create_optional_request_params={"duration": 8, "ratio": "1280:720"}, + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + + assert url == "https://api.dev.runwayml.com/v1/text_to_video" + assert "promptImage" not in data + assert data["promptText"] == "A serene mountain lake at sunrise" + + def test_transform_video_create_request_video_to_video(self): + """A promptVideo request must hit /video_to_video with promptImage stripped.""" + data, files, url = self.config.transform_video_create_request( + model="aleph2", + prompt="Make it snow", + api_base="https://api.dev.runwayml.com/v1", + video_create_optional_request_params={ + "promptVideo": "https://example.com/source.mp4", + "promptImage": "https://example.com/reference.png", + "ratio": "1280:720", + }, + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + + assert url == "https://api.dev.runwayml.com/v1/video_to_video" + assert data["promptVideo"] == "https://example.com/source.mp4" + assert "promptImage" not in data + + def test_transform_video_create_request_video_uri_routes_to_video_to_video(self): + _, _, url = self.config.transform_video_create_request( + model="aleph2", + prompt="Make it snow", + api_base="https://api.dev.runwayml.com/v1", + video_create_optional_request_params={"videoUri": "https://example.com/source.mp4"}, + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + + assert url == "https://api.dev.runwayml.com/v1/video_to_video" + + def test_status_progress_fraction_scales_to_percent(self): + """Runway reports progress as a 0..1 float; VideoObject.progress is an int percent.""" + mock_response = Mock(spec=httpx.Response) + mock_response.json.return_value = { + "id": "63fd0f13-f29d-4e58-99d3-1cb9efa14a5b", + "createdAt": "2025-11-11T21:48:50.448Z", + "status": "RUNNING", + "progress": 0.027, + } + + result = self.config.transform_video_status_retrieve_response( + raw_response=mock_response, + logging_obj=self.mock_logging_obj, + custom_llm_provider="runwayml", + ) + + assert result.status == "in_progress" + assert result.progress == 3 + + def test_get_error_class_returns_exception_instead_of_raising(self): + error = self.config.get_error_class( + error_message="Invalid API key", + status_code=401, + headers={}, + ) + + assert isinstance(error, RunwayMLError) + assert isinstance(error, BaseLLMException) + assert error.status_code == 401 + assert error.message == "Invalid API key" + + def test_create_response_usage_includes_resolution_and_provider_cost(self): + mock_response = Mock(spec=httpx.Response) + mock_response.json.return_value = { + "id": "test-video-id-123", + "createdAt": "2025-11-11T21:48:50.448Z", + "status": "PENDING", + "estimatedCost": {"credits": 25.0}, + } + + video_obj = self.config.transform_video_create_response( + model="gen4_turbo", + raw_response=mock_response, + logging_obj=self.mock_logging_obj, + custom_llm_provider="runwayml", + request_data={"model": "gen4_turbo", "ratio": "1280:720", "duration": 5}, + ) + + assert video_obj.usage == { + "duration_seconds": 5.0, + "video_resolution": "720p", + "provider_reported_cost_usd": 0.25, + } + + def test_create_response_usage_omits_unknown_fields(self): + mock_response = Mock(spec=httpx.Response) + mock_response.json.return_value = { + "id": "test-video-id-123", + "createdAt": "2025-11-11T21:48:50.448Z", + "status": "PENDING", + } + + video_obj = self.config.transform_video_create_response( + model="gen4_turbo", + raw_response=mock_response, + logging_obj=self.mock_logging_obj, + custom_llm_provider="runwayml", + request_data={"model": "gen4_turbo"}, + ) + + assert video_obj.usage == {} + + @pytest.mark.parametrize( + "ratio,expected", + [ + ("848:480", "480p"), + ("1280:720", "720p"), + ("1920:1080", "1080p"), + ("2560:1440", "1080p"), + ("3840:2160", "4k"), + (None, None), + ("banana", None), + ], + ) + def test_ratio_to_resolution_tiers(self, ratio, expected): + assert _ratio_to_resolution(ratio) == expected + def test_transform_video_status_with_timestamp_handling(self): """Test status retrieval handles RunwayML's ISO 8601 timestamps correctly.""" from litellm.types.videos.utils import encode_video_id_with_provider diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index d655eb96a02..e817924f5fb 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -730,7 +730,9 @@ def validate_model_cost_values(model_data, exceptions=None): "output_cost_per_pixel", "input_cost_per_second", "output_cost_per_second", + "output_cost_per_second_480p", "output_cost_per_second_1080p", + "output_cost_per_second_4k", "input_cost_per_query", "input_cost_per_request", "input_cost_per_audio_token", @@ -944,7 +946,9 @@ def test_aaamodel_prices_and_context_window_json_is_valid(): "output_cost_per_video_token": {"type": "number"}, "output_cost_per_pixel": {"type": "number"}, "output_cost_per_second": {"type": "number"}, + "output_cost_per_second_480p": {"type": "number"}, "output_cost_per_second_1080p": {"type": "number"}, + "output_cost_per_second_4k": {"type": "number"}, "output_cost_per_token": {"type": "number"}, "output_cost_per_token_above_128k_tokens": {"type": "number"}, "output_cost_per_token_above_200k_tokens": {"type": "number"}, @@ -1124,6 +1128,7 @@ def test_aaamodel_prices_and_context_window_json_is_valid(): exceptions = [ # Add any model IDs that should be exempt from the cost validation # Example: "expensive-model-id", + "runwayml/seedance2", # 4K output is 150 credits/second = $1.50/second ] is_valid, violations = validate_model_cost_values(actual_json, exceptions) diff --git a/tests/test_litellm/test_video_generation.py b/tests/test_litellm/test_video_generation.py index fb167a8624e..9125521d7a4 100644 --- a/tests/test_litellm/test_video_generation.py +++ b/tests/test_litellm/test_video_generation.py @@ -421,6 +421,117 @@ class TestVideoGeneration: ) assert cost == 0.5 + def test_completion_cost_video_custom_pricing_under_litellm_metadata(self): + """Video routes store deployment model_info under litellm_metadata, not metadata. + + Regression for https://github.com/BerriAI/litellm/issues/36483: custom video + pricing was silently ignored because completion_cost only read metadata. + """ + from litellm.cost_calculator import completion_cost + + mock_response = MagicMock() + mock_response.usage = {"duration_seconds": 10.0} + type(mock_response)._hidden_params = {} + + mock_logging_obj = MagicMock() + mock_logging_obj.litellm_params = { + "litellm_metadata": { + "model_info": { + "output_cost_per_video_per_second": 0.18, + } + } + } + + cost = completion_cost( + completion_response=mock_response, + model="runwayml/seedance2", + call_type="create_video", + custom_llm_provider="runwayml", + custom_pricing=True, + litellm_logging_obj=mock_logging_obj, + ) + assert abs(cost - 1.8) < 0.001 + + def test_completion_cost_video_uses_provider_reported_cost_without_custom_pricing(self): + """With no custom pricing, the provider's own reported cost wins over a duration estimate.""" + from litellm.cost_calculator import completion_cost + + mock_response = MagicMock() + mock_response.usage = { + "duration_seconds": 5.0, + "video_resolution": "720p", + "provider_reported_cost_usd": 0.31, + } + type(mock_response)._hidden_params = {} + + cost = completion_cost( + completion_response=mock_response, + model="runwayml/gen4_turbo", + call_type="create_video", + custom_llm_provider="runwayml", + ) + assert cost == 0.31 + + def test_completion_cost_video_custom_pricing_beats_provider_reported_cost(self): + """Deployment-level custom pricing overrides the provider's reported cost.""" + from litellm.cost_calculator import completion_cost + + mock_response = MagicMock() + mock_response.usage = { + "duration_seconds": 10.0, + "provider_reported_cost_usd": 0.31, + } + type(mock_response)._hidden_params = {} + + mock_logging_obj = MagicMock() + mock_logging_obj.litellm_params = { + "metadata": { + "model_info": { + "output_cost_per_video_per_second": 0.18, + } + } + } + + cost = completion_cost( + completion_response=mock_response, + model="runwayml/seedance2", + call_type="create_video", + custom_llm_provider="runwayml", + custom_pricing=True, + litellm_logging_obj=mock_logging_obj, + ) + assert abs(cost - 1.8) < 0.001 + + def test_completion_cost_video_resolution_tiers_from_cost_map(self, monkeypatch): + """The 480p/1080p/4k tier keys resolve from the shipped runwayml cost map entries.""" + from litellm.cost_calculator import completion_cost + + local_map_path = os.path.join( + os.path.dirname(__file__), "..", "..", "model_prices_and_context_window.json" + ) + with open(local_map_path, "r") as f: + monkeypatch.setattr(litellm, "model_cost", json.load(f)) + + def cost_for(model: str, resolution: str | None, duration: float) -> float: + mock_response = MagicMock() + mock_response.usage = { + "duration_seconds": duration, + **({"video_resolution": resolution} if resolution else {}), + } + type(mock_response)._hidden_params = {} + return completion_cost( + completion_response=mock_response, + model=model, + call_type="create_video", + custom_llm_provider="runwayml", + ) + + assert abs(cost_for("runwayml/seedance2", "4k", 8.0) - 12.0) < 0.001 + assert abs(cost_for("runwayml/seedance2", "1080p", 8.0) - 3.2) < 0.001 + assert abs(cost_for("runwayml/seedance2", "720p", 8.0) - 2.88) < 0.001 + assert abs(cost_for("runwayml/seedance2_5", "480p", 8.0) - 1.6) < 0.001 + assert abs(cost_for("runwayml/gen4.5", None, 8.0) - 0.96) < 0.001 + def test_video_generation_with_files(self): """Test video generation with file uploads.""" config = OpenAIVideoConfig() diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index d311bfa3cbc..4b285a6efee 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -27624,6 +27624,10 @@ export interface components { output_cost_per_second?: number | null; /** Output Cost Per Second 1080P */ output_cost_per_second_1080p?: number | null; + /** Output Cost Per Second 480P */ + output_cost_per_second_480p?: number | null; + /** Output Cost Per Second 4K */ + output_cost_per_second_4k?: number | null; /** Output Cost Per Token */ output_cost_per_token?: number | null; /** Output Cost Per Token Above 128K Tokens */ @@ -36833,6 +36837,10 @@ export interface components { output_cost_per_second?: number | null; /** Output Cost Per Second 1080P */ output_cost_per_second_1080p?: number | null; + /** Output Cost Per Second 480P */ + output_cost_per_second_480p?: number | null; + /** Output Cost Per Second 4K */ + output_cost_per_second_4k?: number | null; /** Output Cost Per Token */ output_cost_per_token?: number | null; /** Output Cost Per Token Above 128K Tokens */ From bf4069a80c46a3da804e1d078f9052e9b8ce449c Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 24 Aug 2026 11:24:21 -0700 Subject: [PATCH 09/14] fix(proxy): authorize health test-connection on final probe params Move _update_litellm_params_for_health_check before can_user_make_model_call so health_check_params cannot retarget the probe after the auth check. Type the Pegasus test helper signature and drop the redundant test narrative. --- .../proxy/health_endpoints/_health_endpoints.py | 11 ++++++----- .../proxy/test_health_check_max_tokens.py | 15 +++------------ 2 files changed, 9 insertions(+), 17 deletions(-) diff --git a/litellm/proxy/health_endpoints/_health_endpoints.py b/litellm/proxy/health_endpoints/_health_endpoints.py index 1feddca9328..c23cde6052e 100644 --- a/litellm/proxy/health_endpoints/_health_endpoints.py +++ b/litellm/proxy/health_endpoints/_health_endpoints.py @@ -1952,8 +1952,13 @@ async def test_model_connection( **request_litellm_params, } - ## Auth check resolved_model_info: Final = loaded_model_info if loaded_model_info is not None else model_info + litellm_params = _update_litellm_params_for_health_check( + model_info=resolved_model_info or {}, + litellm_params=litellm_params, + ) + + ## Auth check, on the final probe params so health_check_params cannot retarget it afterwards await ModelManagementAuthChecks.can_user_make_model_call( model_params=Deployment( model_name="test_model", @@ -1964,10 +1969,6 @@ async def test_model_connection( prisma_client=prisma_client, premium_user=premium_user, ) - litellm_params = _update_litellm_params_for_health_check( - model_info=resolved_model_info or {}, - litellm_params=litellm_params, - ) mode = mode or litellm_params.pop("mode", None) result: Final = await run_with_timeout( diff --git a/tests/test_litellm/proxy/test_health_check_max_tokens.py b/tests/test_litellm/proxy/test_health_check_max_tokens.py index e20b18c8813..162360328d5 100644 --- a/tests/test_litellm/proxy/test_health_check_max_tokens.py +++ b/tests/test_litellm/proxy/test_health_check_max_tokens.py @@ -549,17 +549,6 @@ async def test_run_model_health_check_skips_auto_router_deployment(): assert result == {} -# --------------------------------------------------------------------------- -# model_info.health_check_params -# -# Some providers require a payload field litellm does not synthesize for a -# probe. Bedrock TwelveLabs Pegasus rejects any Invoke body without a top-level -# `mediaSource`, so every health check on such a deployment failed with -# "Invalid JSON: $: required property 'mediaSource' not found". The config key -# was accepted and then never read, so operators had no way to supply it. -# --------------------------------------------------------------------------- - - def test_health_check_params_merge_into_probe_params(): """health_check_params reach the probe request for the deployment that declares them.""" media_source = {"s3Location": {"uri": "s3://my-bucket/clip.mp4"}} @@ -640,7 +629,9 @@ def test_health_check_params_apply_to_non_chat_modes(): assert "max_tokens" not in updated -async def _pegasus_health_check_request_body(model_info: dict, monkeypatch) -> dict: +async def _pegasus_health_check_request_body( + model_info: dict[str, object], monkeypatch: pytest.MonkeyPatch +) -> dict[str, object]: monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) litellm.in_memory_llm_clients_cache.flush_cache() From b117190b0ba8ac3d9fe288318fc062f0d316f6e2 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 24 Aug 2026 11:43:40 -0700 Subject: [PATCH 10/14] chore(pricing): regenerate model prices schema for new video cost tier fields --- model_prices_and_context_window.schema.json | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/model_prices_and_context_window.schema.json b/model_prices_and_context_window.schema.json index f5560a20ab2..60a016ecf37 100644 --- a/model_prices_and_context_window.schema.json +++ b/model_prices_and_context_window.schema.json @@ -428,6 +428,14 @@ "type": "number", "minimum": 0 }, + "output_cost_per_second_480p": { + "type": "number", + "minimum": 0 + }, + "output_cost_per_second_4k": { + "type": "number", + "minimum": 0 + }, "output_cost_per_token": { "type": "number", "minimum": 0, From 25b379f3c7ce837ee60a0a5952cb90cfcb7d768a Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 24 Aug 2026 11:47:21 -0700 Subject: [PATCH 11/14] test: register supports_legacy_thinking in model-prices schema Regenerate model_prices_and_context_window.schema.json and add the flag to the inline validator schema in test_utils.py so the new cost-map key passes validate-model-prices-json and the JSON-valid test. --- model_prices_and_context_window.schema.json | 3 +++ tests/test_litellm/test_utils.py | 1 + 2 files changed, 4 insertions(+) diff --git a/model_prices_and_context_window.schema.json b/model_prices_and_context_window.schema.json index f5560a20ab2..aeb88962305 100644 --- a/model_prices_and_context_window.schema.json +++ b/model_prices_and_context_window.schema.json @@ -625,6 +625,9 @@ "supports_image_size": { "type": "boolean" }, + "supports_legacy_thinking": { + "type": "boolean" + }, "supports_low_reasoning_effort": { "type": "boolean" }, diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index d655eb96a02..cb451d02efd 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -993,6 +993,7 @@ def test_aaamodel_prices_and_context_window_json_is_valid(): "supports_xhigh_reasoning_effort": {"type": "boolean"}, "supports_max_reasoning_effort": {"type": "boolean"}, "supports_adaptive_thinking": {"type": "boolean"}, + "supports_legacy_thinking": {"type": "boolean"}, "thinking_always_on": {"type": "boolean"}, "supports_mid_conversation_system": {"type": "boolean"}, "supports_sampling_params": {"type": "boolean"}, From 458a63935f55c63cf50b847c35c890784eac58b2 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 24 Aug 2026 11:58:33 -0700 Subject: [PATCH 12/14] test(health): guard that test_connection authorizes on post-merge probe params Add an endpoint-level regression test asserting can_user_make_model_call receives the litellm_params after health_check_params are merged in, so the merge-before-auth ordering cannot silently regress and let a request smuggle a field past authorization. --- .../health_endpoints/test_health_endpoints.py | 56 +++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/tests/test_litellm/proxy/health_endpoints/test_health_endpoints.py b/tests/test_litellm/proxy/health_endpoints/test_health_endpoints.py index 62919200d47..72c6f77a4f7 100644 --- a/tests/test_litellm/proxy/health_endpoints/test_health_endpoints.py +++ b/tests/test_litellm/proxy/health_endpoints/test_health_endpoints.py @@ -898,6 +898,62 @@ async def test_test_model_connection_uses_loaded_deployment_team_id_via_model_na assert passed_model_params.model_info.team_id == deployment_owner_team_id +@pytest.mark.asyncio +async def test_test_model_connection_authorizes_on_params_after_health_check_params_merge(): + """ + Regression guard for the ordering fix: health_check_params from the request + body are merged into the probe params BEFORE the authorization check, so a + caller cannot smuggle a field past auth via health_check_params. Auth is + stubbed to reject, which halts the endpoint right after it records the + params it was handed, so the outbound probe is never reached. If the merge + is moved back to after can_user_make_model_call, the marker is absent from + those params and this test fails. + """ + from fastapi import HTTPException + + from litellm.proxy.management_endpoints.model_management_endpoints import ( + ModelManagementAuthChecks, + ) + from litellm.types.router import Deployment + + marker = "sentinel-from-health-check-params" + mock_can_user_make_model_call = AsyncMock( + side_effect=HTTPException(status_code=403, detail="denied") + ) + + with ( + patch( # test-quality-ok: proxy module global, no injection seam + "litellm.proxy.proxy_server.prisma_client", MagicMock() + ), + patch( # test-quality-ok: proxy module global, no injection seam + "litellm.proxy.proxy_server.llm_router", None + ), + patch.object( # test-quality-ok: capturing the params handed to auth is the assertion + ModelManagementAuthChecks, + "can_user_make_model_call", + mock_can_user_make_model_call, + ), + pytest.raises(HTTPException), + ): + await health_test_model_connection( + request=MagicMock(), + mode="chat", + litellm_params={"model": "openai/gpt-4o"}, + model_info={"health_check_params": {"probe_marker": marker}}, + user_api_key_dict=UserAPIKeyAuth( + token="requester-token", + user_id="admin-user", + user_role=LitellmUserRoles.PROXY_ADMIN, + ), + ) + + assert mock_can_user_make_model_call.called + passed_model_params = mock_can_user_make_model_call.call_args.kwargs["model_params"] + assert isinstance(passed_model_params, Deployment) + authorized_params = passed_model_params.litellm_params.model_dump() + assert authorized_params.get("probe_marker") == marker + + @pytest.mark.asyncio async def test_test_model_connection_authorized_team_admin_passes_real_auth(): """ From 6975b8ea4b48807844e026186a28f86e352f89ac Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Mon, 24 Aug 2026 12:19:03 -0700 Subject: [PATCH 13/14] fix(utils): make prompt_token_calculator count claude models again The claude branch called the anthropic SDK's `Anthropic().count_tokens`, which the SDK removed, so every claude call raised AttributeError. Counting now goes through litellm's own token_counter, which handles anthropic models offline and drops the SDK dependency entirely. Hiding that was a swallowed error: `except Exception: Exception("Anthropic import failed please run `pip install anthropic`")` built the exception without raising it, so an environment missing the SDK fell through to the unguarded `from anthropic import ...` on the next line and got a bare ModuleNotFoundError instead of the install hint. That was the codebase's last PLW0133, so the rule graduates from the ratcheted budget into ruff.toml where it hard-fails, and editors get the diagnostic inline. --- litellm/utils.py | 15 ++------------- ruff-strict-budget.json | 7 ++----- ruff.toml | 6 +++--- tests/test_litellm/test_utils.py | 18 ++++++++++++++++++ 4 files changed, 25 insertions(+), 21 deletions(-) diff --git a/litellm/utils.py b/litellm/utils.py index e5ce7157e77..467b8602c27 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -6526,21 +6526,10 @@ def acreate(*args, **kwargs): ## Thin client to handle the acreate langchain ca def prompt_token_calculator(model, messages): - # use tiktoken or anthropic's tokenizer depending on the model text: Final = " ".join(message["content"] for message in messages) - num_tokens = 0 if "claude" in model: - try: - import anthropic - except Exception: - Exception("Anthropic import failed please run `pip install anthropic`") - from anthropic import AI_PROMPT, HUMAN_PROMPT, Anthropic - - anthropic_obj: Final = Anthropic() - num_tokens = anthropic_obj.count_tokens(text) - else: - num_tokens = len(_get_default_encoding().encode(text)) - return num_tokens + return token_counter(model=model, text=text) + return len(_get_default_encoding().encode(text)) def valid_model(model): diff --git a/ruff-strict-budget.json b/ruff-strict-budget.json index a990f7c3830..3b3c3e8ae5b 100644 --- a/ruff-strict-budget.json +++ b/ruff-strict-budget.json @@ -57,7 +57,7 @@ "limit": 3 }, "BLE001": { - "limit": 2920 + "limit": 2919 }, "C401": { "limit": 8 @@ -108,7 +108,7 @@ "limit": 3 }, "F401": { - "limit": 17 + "limit": 14 }, "LOG015": { "limit": 5 @@ -152,9 +152,6 @@ "PLW0127": { "limit": 57 }, - "PLW0133": { - "limit": 1 - }, "PLW0602": { "limit": 215 }, diff --git a/ruff.toml b/ruff.toml index 9b90910b355..44bdf9d8125 100644 --- a/ruff.toml +++ b/ruff.toml @@ -5,9 +5,9 @@ lint.ignore = ["F405", "E402", "F403"] lint.extend-select = [ "T20", "PGH004", "RUF008", "RUF009", "RUF100", "B033", "FURB136", "FURB168", "FURB188", "I001", "PERF402", "PIE790", "PIE800", "PLC0208", - "PLR0402", "PLR1711", "PLR1730", "PLR2044", "PYI030", "PYI041", "PYI064", "RET501", "RUF010", - "RUF022", "RUF023", "RUF051", "SIM114", "SIM118", "TC005", "UP006", "UP007", "UP008", "UP012", - "UP018", "UP024", "UP032", "UP034", "UP035", "UP037", "UP045", + "PLR0402", "PLR1711", "PLR1730", "PLR2044", "PLW0133", "PYI030", "PYI041", "PYI064", "RET501", + "RUF010", "RUF022", "RUF023", "RUF051", "SIM114", "SIM118", "TC005", "UP006", "UP007", "UP008", + "UP012", "UP018", "UP024", "UP032", "UP034", "UP035", "UP037", "UP045", ] # RUF100 (unused-noqa) only knows the rules enabled in THIS config, so it would strip # `# noqa` directives that protect rules enforced elsewhere. List those codes as external diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index d655eb96a02..97baf818d7c 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -1,6 +1,7 @@ import json import logging import os +import sys from typing import Final from unittest.mock import AsyncMock, MagicMock, patch @@ -41,6 +42,7 @@ from litellm.utils import ( get_prompt_cache_min_tokens, is_cached_message, is_prompt_caching_valid_prompt, + prompt_token_calculator, ) # Adds the parent directory to the system path @@ -4973,3 +4975,19 @@ def test_completion_does_not_leak_rust_flag_into_provider_request_body(): create_kwargs = mock_client.chat.completions.with_raw_response.create.call_args.kwargs assert "rust" not in create_kwargs assert "rust" not in (create_kwargs.get("extra_body") or {}) + + +def test_prompt_token_calculator_counts_claude_without_the_anthropic_sdk(): + """ + The claude branch used to call the anthropic SDK's `count_tokens`, which the SDK + removed, so every claude call raised AttributeError. Counting must work with + `anthropic` unimportable. + """ + messages: Final = [{"role": "user", "content": "the quick brown fox jumps over the lazy dog"}] + + with patch.dict(sys.modules, {"anthropic": None}): + claude_tokens = prompt_token_calculator("claude-sonnet-4-5", messages) + gpt_tokens = prompt_token_calculator("gpt-4o", messages) + + assert claude_tokens == 9 + assert gpt_tokens == 9 From 3030e974b889e744500bd22e8e70a07326100e2c Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 24 Aug 2026 12:20:00 -0700 Subject: [PATCH 14/14] fix(runwayml): skip progress scaling when Runway returns a null progress --- .../llms/runwayml/videos/transformation.py | 5 +++-- .../test_runway_video_transformation.py | 19 +++++++++++++++++++ 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/litellm/llms/runwayml/videos/transformation.py b/litellm/llms/runwayml/videos/transformation.py index 065661a8731..2264f52e2a3 100644 --- a/litellm/llms/runwayml/videos/transformation.py +++ b/litellm/llms/runwayml/videos/transformation.py @@ -636,8 +636,9 @@ class RunwayMLVideoConfig(BaseVideoConfig): if "completedAt" in response_data: video_data["completed_at"] = self._parse_runway_timestamp(response_data.get("completedAt")) - if "progress" in response_data: - video_data["progress"] = _progress_percent(response_data["progress"]) + progress_value: Final = response_data.get("progress") + if progress_value is not None: + video_data["progress"] = _progress_percent(progress_value) if "failureCode" in response_data or "failure" in response_data: video_data["error"] = { diff --git a/tests/test_litellm/llms/runwayml/videos/test_runway_video_transformation.py b/tests/test_litellm/llms/runwayml/videos/test_runway_video_transformation.py index e52cf9211ec..afc8e7ec4a2 100644 --- a/tests/test_litellm/llms/runwayml/videos/test_runway_video_transformation.py +++ b/tests/test_litellm/llms/runwayml/videos/test_runway_video_transformation.py @@ -119,6 +119,25 @@ class TestRunwayMLVideoTransformation: assert result.status == "in_progress" assert result.progress == 3 + def test_status_progress_null_leaves_progress_unset(self): + """Runway sends an explicit null progress for pending polls; scaling it must not crash.""" + mock_response = Mock(spec=httpx.Response) + mock_response.json.return_value = { + "id": "63fd0f13-f29d-4e58-99d3-1cb9efa14a5b", + "createdAt": "2025-11-11T21:48:50.448Z", + "status": "PENDING", + "progress": None, + } + + result = self.config.transform_video_status_retrieve_response( + raw_response=mock_response, + logging_obj=self.mock_logging_obj, + custom_llm_provider="runwayml", + ) + + assert result.status == "queued" + assert result.progress is None + def test_get_error_class_returns_exception_instead_of_raising(self): error = self.config.get_error_class( error_message="Invalid API key",