From 245b204b99ea158bb65a43c7b0c12a3bd66780aa Mon Sep 17 00:00:00 2001 From: Yash Raj Pandey Date: Thu, 16 Jul 2026 12:05:45 -0400 Subject: [PATCH 01/15] fix(proxy): re-encode video id with routed model_id --- litellm/proxy/video_endpoints/endpoints.py | 13 ++- litellm/proxy/video_endpoints/utils.py | 33 ++++++- .../proxy/video_endpoints/test_endpoints.py | 87 ++++++++++++++++++- 3 files changed, 127 insertions(+), 6 deletions(-) diff --git a/litellm/proxy/video_endpoints/endpoints.py b/litellm/proxy/video_endpoints/endpoints.py index fe966c2e31a..5e0b5c68cb9 100644 --- a/litellm/proxy/video_endpoints/endpoints.py +++ b/litellm/proxy/video_endpoints/endpoints.py @@ -18,6 +18,7 @@ from litellm.proxy.common_utils.openai_endpoint_utils import ( from litellm.proxy.image_endpoints.endpoints import batch_to_bytesio from litellm.proxy.video_endpoints.utils import ( encode_character_id_in_response, + encode_video_id_in_response, extract_model_from_target_model_names, get_custom_provider_from_data, video_reference_to_id, @@ -107,6 +108,7 @@ async def video_generation( user_api_base=user_api_base, version=version, ) + encoded_response: Final = encode_video_id_in_response(generated, data.get("model")) except Exception as e: raise await processor._handle_llm_api_exception( e=e, @@ -115,7 +117,7 @@ async def video_generation( version=version, ) else: - return generated + return encoded_response @router.get( @@ -502,6 +504,7 @@ async def video_remix( user_api_base=user_api_base, version=version, ) + encoded_response: Final = encode_video_id_in_response(remixed, data.get("model")) except Exception as e: raise await processor._handle_llm_api_exception( e=e, @@ -510,7 +513,7 @@ async def video_remix( version=version, ) else: - return remixed + return encoded_response @router.post( @@ -815,6 +818,7 @@ async def video_edit( user_api_base=user_api_base, version=version, ) + encoded_response: Final = encode_video_id_in_response(edited, data.get("model")) except Exception as e: raise await processor._handle_llm_api_exception( e=e, @@ -823,7 +827,7 @@ async def video_edit( version=version, ) else: - return edited + return encoded_response @router.post( @@ -912,6 +916,7 @@ async def video_extension( user_api_base=user_api_base, version=version, ) + encoded_response: Final = encode_video_id_in_response(extended, data.get("model")) except Exception as e: raise await processor._handle_llm_api_exception( e=e, @@ -920,4 +925,4 @@ async def video_extension( version=version, ) else: - return extended + return encoded_response diff --git a/litellm/proxy/video_endpoints/utils.py b/litellm/proxy/video_endpoints/utils.py index a38226cc253..6f383909b5a 100644 --- a/litellm/proxy/video_endpoints/utils.py +++ b/litellm/proxy/video_endpoints/utils.py @@ -2,7 +2,11 @@ from typing import Any, Final import orjson -from litellm.types.videos.utils import encode_character_id_with_provider +from litellm.types.videos.utils import ( + decode_video_id_with_provider, + encode_character_id_with_provider, + encode_video_id_with_provider, +) def extract_model_from_target_model_names(target_model_names: Any) -> str | None: @@ -47,6 +51,33 @@ def get_custom_provider_from_data(data: dict[str, Any]) -> str | None: return None +def _hidden_param(response: object, key: str) -> str | None: + params: Final = getattr(response, "_hidden_params", None) + if not isinstance(params, dict): + return None + value: Final = params.get(key) + return value if isinstance(value, str) else None + + +def encode_video_id_in_response(response: object, fallback_model: str | None) -> object: + video_id: Final = response.get("id") if isinstance(response, dict) else getattr(response, "id", None) + if not isinstance(video_id, str) or not video_id: + return response + + decoded: Final = decode_video_id_with_provider(video_id) + encoded: Final = encode_video_id_with_provider( + video_id=decoded.get("video_id", ""), + provider=_hidden_param(response, "custom_llm_provider") or decoded.get("custom_llm_provider") or "openai", + model_id=_hidden_param(response, "model_id") or fallback_model, + ) + if isinstance(response, dict): + response["id"] = encoded # rebind-ok: in-place id rewrite, matching encode_character_id_in_response + return response + + response.id = encoded # rebind-ok: in-place id rewrite, matching encode_character_id_in_response + return response + + def encode_character_id_in_response(response: Any, custom_llm_provider: str, model_id: str | None) -> Any: if isinstance(response, dict) and response.get("id"): response["id"] = encode_character_id_with_provider( diff --git a/tests/test_litellm/proxy/video_endpoints/test_endpoints.py b/tests/test_litellm/proxy/video_endpoints/test_endpoints.py index c5996f95f54..00e1902a942 100644 --- a/tests/test_litellm/proxy/video_endpoints/test_endpoints.py +++ b/tests/test_litellm/proxy/video_endpoints/test_endpoints.py @@ -41,7 +41,9 @@ from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing from litellm.proxy.utils import ProxyLogging from litellm.router import Router +from litellm.types.videos.main import VideoObject from litellm.types.videos.utils import ( + decode_video_id_with_provider, encode_character_id_with_provider, encode_video_id_with_provider, ) @@ -67,7 +69,7 @@ AZURE_CHARACTER_ID = encode_character_id_with_provider( RESOLVED_MODELS: Dict[str, str] = {VIDEO_MODEL_ID: "azure-sora"} # Sentinel propagated by base_process for the passthrough endpoints. -SENTINEL = object() +SENTINEL = VideoObject(id="video_raw", object="video", status="processing") class FakeRequest: @@ -111,6 +113,7 @@ class Harness: @pytest.fixture def harness(): + SENTINEL.id = "video_raw" logging = MagicMock(spec=ProxyLogging) router = MagicMock(spec=Router) @@ -229,6 +232,26 @@ async def test_generation__route_type_data_and_no_provider_default(harness): harness.batch_to_bytesio.assert_not_called() +@pytest.mark.asyncio +async def test_generation__reencodes_id_with_model_id(harness): + response = VideoObject( + id=encode_video_id_with_provider("video_raw", "openai", None), + object="video", + status="processing", + ) + response._hidden_params = { + "custom_llm_provider": "openai", + "model_id": VIDEO_MODEL_ID, + } + harness.base_process.return_value = response + + resp = await call_generation(harness, body={"model": "sora-2"}) + + decoded_video_id = decode_video_id_with_provider(resp.id) + assert decoded_video_id["model_id"] == VIDEO_MODEL_ID + assert decoded_video_id["video_id"] == "video_raw" + + @pytest.mark.asyncio async def test_generation__input_reference_attached(harness): body = {"model": "sora-2", "prompt": "a sunset"} @@ -400,6 +423,28 @@ async def test_edit__extracts_nested_video_id_full_contract(harness): } +@pytest.mark.asyncio +async def test_edit__reencodes_id_with_model_id(harness): + response = VideoObject( + id=encode_video_id_with_provider("video_raw", "openai", None), + object="video", + status="processing", + ) + response._hidden_params = { + "custom_llm_provider": "openai", + "model_id": VIDEO_MODEL_ID, + } + harness.base_process.return_value = response + + resp = await call_edit( + harness, body={"prompt": "brighter", "video": {"id": "video_plain"}} + ) + + decoded_video_id = decode_video_id_with_provider(resp.id) + assert decoded_video_id["model_id"] == VIDEO_MODEL_ID + assert decoded_video_id["video_id"] == "video_raw" + + @pytest.mark.asyncio async def test_edit__provider_from_body_data_for_plain_id(harness): """For a plain id, get_custom_provider_from_data (run for real) pulls the @@ -543,6 +588,26 @@ async def test_remix__model_encoded_id_full_contract(harness): } +@pytest.mark.asyncio +async def test_remix__reencodes_id_with_model_id(harness): + response = VideoObject( + id=encode_video_id_with_provider("video_raw", "openai", None), + object="video", + status="processing", + ) + response._hidden_params = { + "custom_llm_provider": "openai", + "model_id": VIDEO_MODEL_ID, + } + harness.base_process.return_value = response + + resp = await call_remix(harness, "video_plain", body={"prompt": "new colors"}) + + decoded_video_id = decode_video_id_with_provider(resp.id) + assert decoded_video_id["model_id"] == VIDEO_MODEL_ID + assert decoded_video_id["video_id"] == "video_raw" + + @pytest.mark.asyncio async def test_remix__provider_from_body_data_not_request_body_reader(harness): """remix resolves the provider from data.get('custom_llm_provider'), never @@ -701,3 +766,23 @@ async def test_extension__extracts_nested_video_id_full_contract(harness): "custom_llm_provider": "azure", "model": "azure-sora", } + + +@pytest.mark.asyncio +async def test_extension__reencodes_id_with_model_id(harness): + response = VideoObject( + id=encode_video_id_with_provider("video_raw", "openai", None), + object="video", + status="processing", + ) + response._hidden_params = { + "custom_llm_provider": "openai", + "model_id": VIDEO_MODEL_ID, + } + harness.base_process.return_value = response + + resp = await call_extension(harness, body={"prompt": "continue"}) + + decoded_video_id = decode_video_id_with_provider(resp.id) + assert decoded_video_id["model_id"] == VIDEO_MODEL_ID + assert decoded_video_id["video_id"] == "video_raw" From 93f4656ecbd0d8f06f3d8339b27729c59c7e3ded Mon Sep 17 00:00:00 2001 From: Yash Raj Pandey Date: Wed, 2 Sep 2026 14:29:10 -0400 Subject: [PATCH 02/15] fix(proxy): encode the routed deployment id, not the model group The five video id encode sites fell back to data["model"] when the response carried no _hidden_params["model_id"]. On the generic router path that field is absent and the selected deployment sits in litellm_metadata.model_info.id, so the encoded id held the public model group. A later status or content call decodes that group, so the router is free to pick a different deployment. With one key per deployment the second call hits the wrong key and cannot resolve the video. Add deployment_id_for_encoding, which reads the hidden model_id first, then litellm_metadata.model_info.id, then the request model. This is the chain ProxyBaseLLMRequestProcessing._get_model_id_from_response already uses. --- litellm/proxy/video_endpoints/endpoints.py | 11 ++++---- litellm/proxy/video_endpoints/utils.py | 14 +++++++++++ .../proxy/video_endpoints/test_endpoints.py | 25 +++++++++++++++++++ 3 files changed, 45 insertions(+), 5 deletions(-) diff --git a/litellm/proxy/video_endpoints/endpoints.py b/litellm/proxy/video_endpoints/endpoints.py index 5e0b5c68cb9..75e1a0dd8b7 100644 --- a/litellm/proxy/video_endpoints/endpoints.py +++ b/litellm/proxy/video_endpoints/endpoints.py @@ -17,6 +17,7 @@ from litellm.proxy.common_utils.openai_endpoint_utils import ( ) from litellm.proxy.image_endpoints.endpoints import batch_to_bytesio from litellm.proxy.video_endpoints.utils import ( + deployment_id_for_encoding, encode_character_id_in_response, encode_video_id_in_response, extract_model_from_target_model_names, @@ -108,7 +109,7 @@ async def video_generation( user_api_base=user_api_base, version=version, ) - encoded_response: Final = encode_video_id_in_response(generated, data.get("model")) + encoded_response: Final = encode_video_id_in_response(generated, deployment_id_for_encoding(generated, data)) except Exception as e: raise await processor._handle_llm_api_exception( e=e, @@ -504,7 +505,7 @@ async def video_remix( user_api_base=user_api_base, version=version, ) - encoded_response: Final = encode_video_id_in_response(remixed, data.get("model")) + encoded_response: Final = encode_video_id_in_response(remixed, deployment_id_for_encoding(remixed, data)) except Exception as e: raise await processor._handle_llm_api_exception( e=e, @@ -603,7 +604,7 @@ async def video_create_character( if target_model_name: hidden_params: Final = getattr(response, "_hidden_params", {}) or {} provider_for_encoding: Final = hidden_params.get("custom_llm_provider") or custom_llm_provider or "openai" - model_id_for_encoding: Final = hidden_params.get("model_id") or data.get("model") + model_id_for_encoding: Final = deployment_id_for_encoding(response, data) response = encode_character_id_in_response( response=response, custom_llm_provider=provider_for_encoding, @@ -818,7 +819,7 @@ async def video_edit( user_api_base=user_api_base, version=version, ) - encoded_response: Final = encode_video_id_in_response(edited, data.get("model")) + encoded_response: Final = encode_video_id_in_response(edited, deployment_id_for_encoding(edited, data)) except Exception as e: raise await processor._handle_llm_api_exception( e=e, @@ -916,7 +917,7 @@ async def video_extension( user_api_base=user_api_base, version=version, ) - encoded_response: Final = encode_video_id_in_response(extended, data.get("model")) + encoded_response: Final = encode_video_id_in_response(extended, deployment_id_for_encoding(extended, data)) except Exception as e: raise await processor._handle_llm_api_exception( e=e, diff --git a/litellm/proxy/video_endpoints/utils.py b/litellm/proxy/video_endpoints/utils.py index 6f383909b5a..222723267ea 100644 --- a/litellm/proxy/video_endpoints/utils.py +++ b/litellm/proxy/video_endpoints/utils.py @@ -1,3 +1,4 @@ +from collections.abc import Mapping from typing import Any, Final import orjson @@ -59,6 +60,19 @@ def _hidden_param(response: object, key: str) -> str | None: return value if isinstance(value, str) else None +def deployment_id_for_encoding(response: object, data: Mapping[str, Any]) -> str | None: + """The deployment the request actually routed to, not the public model group. + + The router leaves ``model_id`` out of ``_hidden_params`` on the generic path and + records the selected deployment under ``litellm_metadata.model_info.id`` instead. + Encoding ``data["model"]`` there would embed the group, so a later status or + content call could load-balance to a different deployment and fail to resolve. + """ + litellm_metadata: Final = data.get("litellm_metadata") or {} + model_info: Final = litellm_metadata.get("model_info") or {} + return _hidden_param(response, "model_id") or model_info.get("id") or data.get("model") + + def encode_video_id_in_response(response: object, fallback_model: str | None) -> object: video_id: Final = response.get("id") if isinstance(response, dict) else getattr(response, "id", None) if not isinstance(video_id, str) or not video_id: diff --git a/tests/test_litellm/proxy/video_endpoints/test_endpoints.py b/tests/test_litellm/proxy/video_endpoints/test_endpoints.py index 00e1902a942..88c43638fbd 100644 --- a/tests/test_litellm/proxy/video_endpoints/test_endpoints.py +++ b/tests/test_litellm/proxy/video_endpoints/test_endpoints.py @@ -252,6 +252,31 @@ async def test_generation__reencodes_id_with_model_id(harness): assert decoded_video_id["video_id"] == "video_raw" +@pytest.mark.asyncio +async def test_generation__reencodes_id_with_routed_deployment_id(harness): + # data["model"] is the public group here; only litellm_metadata carries the + # deployment the router picked. + response = VideoObject( + id=encode_video_id_with_provider("video_raw", "openai", None), + object="video", + status="processing", + ) + response._hidden_params = {"custom_llm_provider": "openai"} + harness.base_process.return_value = response + + resp = await call_generation( + harness, + body={ + "model": "sora-2", + "litellm_metadata": {"model_info": {"id": VIDEO_MODEL_ID}}, + }, + ) + + decoded_video_id = decode_video_id_with_provider(resp.id) + assert decoded_video_id["model_id"] == VIDEO_MODEL_ID + assert decoded_video_id["video_id"] == "video_raw" + + @pytest.mark.asyncio async def test_generation__input_reference_attached(harness): body = {"model": "sora-2", "prompt": "a sunset"} From f45e6dcf016eb5b689c0b4b3365fff865ee84ca6 Mon Sep 17 00:00:00 2001 From: Yash Raj Pandey Date: Mon, 7 Sep 2026 17:02:03 -0400 Subject: [PATCH 03/15] style(video): explain the id rebind on a non-dict response for basedpyright The lint gate failed with reportAttributeAccessIssue at 481 over the 480 limit. The new branch assigns to response.id after the isinstance(response, dict) test fails, and basedpyright cannot prove object has an id attribute. Suppress it the way this repo already does, with the rule name and the reason on the line. Measured on the two changed files: reportAttributeAccessIssue drops from 2 to 1, and the base b618c7ad86 has 1 in the same file, so the delta is 0. The remaining one sits in encode_character_id_in_response, which this branch does not touch. The 30 tests in tests/test_litellm/proxy/video_endpoints still pass. --- litellm/proxy/video_endpoints/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/proxy/video_endpoints/utils.py b/litellm/proxy/video_endpoints/utils.py index 222723267ea..46bf9c564a8 100644 --- a/litellm/proxy/video_endpoints/utils.py +++ b/litellm/proxy/video_endpoints/utils.py @@ -88,7 +88,7 @@ def encode_video_id_in_response(response: object, fallback_model: str | None) -> response["id"] = encoded # rebind-ok: in-place id rewrite, matching encode_character_id_in_response return response - response.id = encoded # rebind-ok: in-place id rewrite, matching encode_character_id_in_response + response.id = encoded # pyright: ignore[reportAttributeAccessIssue] # rebind-ok: in-place id rewrite on a non-dict response, matching encode_character_id_in_response return response From 9679ba326d68a8e250f3cbac7869eb90f8edef4f Mon Sep 17 00:00:00 2001 From: Yash Raj Pandey Date: Tue, 29 Sep 2026 14:51:38 -0400 Subject: [PATCH 04/15] fix(proxy): pin video follow-up calls to the deployment in the id The create response id carries the id of the deployment that made the video. The status, content, remix, edit, extension and get-character endpoints mapped that id back to its model group, so the router load-balanced every follow-up call again. With two deployments in one group, about half of the follow-up calls went to a deployment that does not have the video. When the model_id in the id is a live deployment id, send that id as the model. The router pins a model that is a deployment id to that deployment. Ids that name a group or a provider model resolve as before, and an id whose deployment was deleted still takes the direct provider call. --- litellm/proxy/video_endpoints/endpoints.py | 13 +-- litellm/proxy/video_endpoints/utils.py | 7 ++ .../proxy/video_endpoints/test_endpoints.py | 93 +++++++++++++++++++ 3 files changed, 107 insertions(+), 6 deletions(-) diff --git a/litellm/proxy/video_endpoints/endpoints.py b/litellm/proxy/video_endpoints/endpoints.py index 75e1a0dd8b7..c46073cfad6 100644 --- a/litellm/proxy/video_endpoints/endpoints.py +++ b/litellm/proxy/video_endpoints/endpoints.py @@ -22,6 +22,7 @@ from litellm.proxy.video_endpoints.utils import ( encode_video_id_in_response, extract_model_from_target_model_names, get_custom_provider_from_data, + routing_model_for_id, video_reference_to_id, ) from litellm.types.videos.utils import ( @@ -272,7 +273,7 @@ async def video_status( # Resolve model_name from model_id if available # This allows the router to automatically inject litellm_params from the model config if model_id_from_decoded and llm_router: - resolved_model: Final = llm_router.resolve_model_name_from_model_id(model_id_from_decoded) + resolved_model: Final = routing_model_for_id(llm_router, model_id_from_decoded) if resolved_model: data["model"] = resolved_model @@ -372,7 +373,7 @@ async def video_content( # Resolve model_name from model_id if available # This allows the router to automatically inject litellm_params from the model config if model_id_from_decoded and llm_router: - resolved_model: Final = llm_router.resolve_model_name_from_model_id(model_id_from_decoded) + resolved_model: Final = routing_model_for_id(llm_router, model_id_from_decoded) if resolved_model: data["model"] = resolved_model # Process request using ProxyBaseLLMRequestProcessing @@ -480,7 +481,7 @@ async def video_remix( # Resolve model_name from model_id if available # This allows the router to automatically inject litellm_params from the model config if model_id_from_decoded and llm_router: - resolved_model: Final = llm_router.resolve_model_name_from_model_id(model_id_from_decoded) + resolved_model: Final = routing_model_for_id(llm_router, model_id_from_decoded) if resolved_model: data["model"] = resolved_model @@ -684,7 +685,7 @@ async def video_get_character( data["custom_llm_provider"] = custom_llm_provider if model_id_from_decoded and llm_router: - resolved_model: Final = llm_router.resolve_model_name_from_model_id(model_id_from_decoded) + resolved_model: Final = routing_model_for_id(llm_router, model_id_from_decoded) if resolved_model: data["model"] = resolved_model @@ -795,7 +796,7 @@ async def video_edit( data["custom_llm_provider"] = custom_llm_provider if model_id_from_decoded and llm_router: - resolved_model: Final = llm_router.resolve_model_name_from_model_id(model_id_from_decoded) + resolved_model: Final = routing_model_for_id(llm_router, model_id_from_decoded) if resolved_model: data["model"] = resolved_model @@ -893,7 +894,7 @@ async def video_extension( data["custom_llm_provider"] = custom_llm_provider if model_id_from_decoded and llm_router: - resolved_model: Final = llm_router.resolve_model_name_from_model_id(model_id_from_decoded) + resolved_model: Final = routing_model_for_id(llm_router, model_id_from_decoded) if resolved_model: data["model"] = resolved_model diff --git a/litellm/proxy/video_endpoints/utils.py b/litellm/proxy/video_endpoints/utils.py index 46bf9c564a8..9efcc61abf4 100644 --- a/litellm/proxy/video_endpoints/utils.py +++ b/litellm/proxy/video_endpoints/utils.py @@ -3,6 +3,7 @@ from typing import Any, Final import orjson +from litellm.router import Router from litellm.types.videos.utils import ( decode_video_id_with_provider, encode_character_id_with_provider, @@ -73,6 +74,12 @@ def deployment_id_for_encoding(response: object, data: Mapping[str, Any]) -> str return _hidden_param(response, "model_id") or model_info.get("id") or data.get("model") +def routing_model_for_id(llm_router: Router, model_id: str) -> str | None: + if llm_router.has_model_id(model_id): + return model_id + return llm_router.resolve_model_name_from_model_id(model_id) + + def encode_video_id_in_response(response: object, fallback_model: str | None) -> object: video_id: Final = response.get("id") if isinstance(response, dict) else getattr(response, "id", None) if not isinstance(video_id, str) or not video_id: diff --git a/tests/test_litellm/proxy/video_endpoints/test_endpoints.py b/tests/test_litellm/proxy/video_endpoints/test_endpoints.py index 88c43638fbd..739aaf4591a 100644 --- a/tests/test_litellm/proxy/video_endpoints/test_endpoints.py +++ b/tests/test_litellm/proxy/video_endpoints/test_endpoints.py @@ -33,8 +33,10 @@ from unittest.mock import AsyncMock, MagicMock, patch import orjson import pytest +import respx +import litellm import litellm.proxy.proxy_server as proxy_server import litellm.proxy.video_endpoints.endpoints as endpoints from litellm.proxy._types import UserAPIKeyAuth @@ -121,6 +123,8 @@ def harness(): side_effect=lambda model_id: RESOLVED_MODELS.get(model_id) ) router.resolve_model_name_from_model_id = resolve_model + router.has_model_id.return_value = False + router.get_deployment.return_value = None read_body = AsyncMock(return_value={}) batch_to_bytesio = AsyncMock(return_value=[b"filebytes"]) @@ -811,3 +815,92 @@ async def test_extension__reencodes_id_with_model_id(harness): decoded_video_id = decode_video_id_with_provider(resp.id) assert decoded_video_id["model_id"] == VIDEO_MODEL_ID assert decoded_video_id["video_id"] == "video_raw" + + +# =========================================================================== # +# Follow-up calls against a real Router # +# =========================================================================== # + + +def _video_router() -> Router: + # weight 0 keeps the group from ever picking deployment-b, so only a pin reaches it + return Router( + model_list=[ + { + "model_name": "sora-2", + "litellm_params": { + "model": "openai/sora-2", + "api_key": "sk-mock-a", + "api_base": "http://a.localhost/v1", + "weight": 1, + }, + "model_info": {"id": "deployment-a"}, + }, + { + "model_name": "sora-2", + "litellm_params": { + "model": "openai/sora-2", + "api_key": "sk-mock-b", + "api_base": "http://b.localhost/v1", + "weight": 0, + }, + "model_info": {"id": "deployment-b"}, + }, + ] + ) + + +def _video_id(model_id: str) -> str: + return encode_video_id_with_provider("video_orig", "openai", model_id) + + +FOLLOW_UP_CALLS = { + "status": lambda h, model_id: call_status(h, _video_id(model_id)), + "content": lambda h, model_id: call_content(h, _video_id(model_id)), + "remix": lambda h, model_id: call_remix(h, _video_id(model_id), body={"prompt": "x"}), + "edit": lambda h, model_id: call_edit(h, body={"prompt": "x", "video": {"id": _video_id(model_id)}}), + "extension": lambda h, model_id: call_extension(h, body={"prompt": "x", "video": {"id": _video_id(model_id)}}), + "get_character": lambda h, model_id: call_get_character( + h, encode_character_id_with_provider("char_orig", "openai", model_id) + ), +} + + +@pytest.mark.asyncio +@pytest.mark.parametrize("endpoint", sorted(FOLLOW_UP_CALLS)) +@pytest.mark.parametrize( + ("model_id", "expected_model"), + [ + ("deployment-b", "deployment-b"), + # ids created before the fix name the group, so they still load-balance + ("sora-2", "sora-2"), + ("", None), + # a deleted deployment takes the direct provider call, as it did before the fix + ("deployment-gone", None), + ], +) +async def test_follow_up__model_comes_from_the_id(harness, endpoint, model_id, expected_model): + harness.base_process.return_value = b"video-bytes" + + with patch.object(proxy_server, "llm_router", _video_router()): + await FOLLOW_UP_CALLS[endpoint](harness, model_id) + + assert harness.processor_data().get("model") == expected_model + + +@pytest.mark.asyncio +async def test_status__real_router_reaches_the_creating_deployment(harness, monkeypatch): + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + llm_router = _video_router() + + with patch.object(proxy_server, "llm_router", llm_router): + await call_status(harness, _video_id("deployment-b")) + with respx.mock(assert_all_called=True) as respx_mock: + deployment_b = respx_mock.get("http://b.localhost/v1/videos/video_orig").respond( + json={"id": "video_orig", "object": "video", "status": "completed", "created_at": 0} + ) + status = await llm_router.avideo_status(**harness.processor_data()) + + assert deployment_b.call_count == 1 + assert status.status == "completed" From d11c9a443de4b2e07781b2a7726ed63eec5bc433 Mon Sep 17 00:00:00 2001 From: Yash Raj Pandey Date: Tue, 29 Sep 2026 14:52:21 -0400 Subject: [PATCH 05/15] fix(proxy): give the provider the model it encoded in the video id The proxy now writes the deployment id into the model_id of a video id. fal.ai builds its queue URLs from that model_id, so a status or content call sent fal.ai a path that starts with the deployment id, and the call failed even with one deployment. Before a follow-up call goes to the provider, replace the deployment id in the video id with the model that the deployment calls. That is the model_id the provider layer wrote at create time. The other video providers do not read it. --- litellm/proxy/video_endpoints/endpoints.py | 6 +++++ litellm/proxy/video_endpoints/utils.py | 15 +++++++++++ .../proxy/video_endpoints/test_endpoints.py | 25 +++++++++++++++++++ 3 files changed, 46 insertions(+) diff --git a/litellm/proxy/video_endpoints/endpoints.py b/litellm/proxy/video_endpoints/endpoints.py index c46073cfad6..e180b573e31 100644 --- a/litellm/proxy/video_endpoints/endpoints.py +++ b/litellm/proxy/video_endpoints/endpoints.py @@ -23,6 +23,7 @@ from litellm.proxy.video_endpoints.utils import ( extract_model_from_target_model_names, get_custom_provider_from_data, routing_model_for_id, + video_id_for_provider, video_reference_to_id, ) from litellm.types.videos.utils import ( @@ -276,6 +277,7 @@ async def video_status( resolved_model: Final = routing_model_for_id(llm_router, model_id_from_decoded) if resolved_model: data["model"] = resolved_model + data["video_id"] = video_id_for_provider(llm_router, video_id) # Process request using ProxyBaseLLMRequestProcessing processor: Final = ProxyBaseLLMRequestProcessing(data=data) @@ -376,6 +378,7 @@ async def video_content( resolved_model: Final = routing_model_for_id(llm_router, model_id_from_decoded) if resolved_model: data["model"] = resolved_model + data["video_id"] = video_id_for_provider(llm_router, video_id) # Process request using ProxyBaseLLMRequestProcessing processor: Final = ProxyBaseLLMRequestProcessing(data=data) try: @@ -484,6 +487,7 @@ async def video_remix( resolved_model: Final = routing_model_for_id(llm_router, model_id_from_decoded) if resolved_model: data["model"] = resolved_model + data["video_id"] = video_id_for_provider(llm_router, video_id) # Process request using ProxyBaseLLMRequestProcessing processor: Final = ProxyBaseLLMRequestProcessing(data=data) @@ -799,6 +803,7 @@ async def video_edit( resolved_model: Final = routing_model_for_id(llm_router, model_id_from_decoded) if resolved_model: data["model"] = resolved_model + data["video_id"] = video_id_for_provider(llm_router, data["video_id"]) processor: Final = ProxyBaseLLMRequestProcessing(data=data) try: @@ -897,6 +902,7 @@ async def video_extension( resolved_model: Final = routing_model_for_id(llm_router, model_id_from_decoded) if resolved_model: data["model"] = resolved_model + data["video_id"] = video_id_for_provider(llm_router, data["video_id"]) processor: Final = ProxyBaseLLMRequestProcessing(data=data) try: diff --git a/litellm/proxy/video_endpoints/utils.py b/litellm/proxy/video_endpoints/utils.py index 9efcc61abf4..d93dab4a494 100644 --- a/litellm/proxy/video_endpoints/utils.py +++ b/litellm/proxy/video_endpoints/utils.py @@ -3,6 +3,7 @@ from typing import Any, Final import orjson +from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider from litellm.router import Router from litellm.types.videos.utils import ( decode_video_id_with_provider, @@ -80,6 +81,20 @@ def routing_model_for_id(llm_router: Router, model_id: str) -> str | None: return llm_router.resolve_model_name_from_model_id(model_id) +def video_id_for_provider(llm_router: Router, video_id: str) -> str: + # fal.ai reads its model path back from the id, so swap the deployment id for the model the deployment calls + decoded: Final = decode_video_id_with_provider(video_id) + provider: Final = decoded.get("custom_llm_provider") + deployment: Final = llm_router.get_deployment(model_id=decoded.get("model_id") or "") + if provider is None or deployment is None: + return video_id + provider_model: Final = get_llm_provider( + model=deployment.litellm_params.model, + custom_llm_provider=deployment.litellm_params.custom_llm_provider, + )[0] + return encode_video_id_with_provider(video_id=decoded["video_id"], provider=provider, model_id=provider_model) + + def encode_video_id_in_response(response: object, fallback_model: str | None) -> object: video_id: Final = response.get("id") if isinstance(response, dict) else getattr(response, "id", None) if not isinstance(video_id, str) or not video_id: diff --git a/tests/test_litellm/proxy/video_endpoints/test_endpoints.py b/tests/test_litellm/proxy/video_endpoints/test_endpoints.py index 739aaf4591a..4a2ebc79185 100644 --- a/tests/test_litellm/proxy/video_endpoints/test_endpoints.py +++ b/tests/test_litellm/proxy/video_endpoints/test_endpoints.py @@ -39,10 +39,12 @@ import respx import litellm import litellm.proxy.proxy_server as proxy_server import litellm.proxy.video_endpoints.endpoints as endpoints +from litellm.llms.fal_ai.videos.transformation import FalAIVideoConfig from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing from litellm.proxy.utils import ProxyLogging from litellm.router import Router +from litellm.types.router import GenericLiteLLMParams from litellm.types.videos.main import VideoObject from litellm.types.videos.utils import ( decode_video_id_with_provider, @@ -821,6 +823,8 @@ async def test_extension__reencodes_id_with_model_id(harness): # Follow-up calls against a real Router # # =========================================================================== # +FAL_MODEL = "fal-ai/kling-video/v2/master/text-to-video" + def _video_router() -> Router: # weight 0 keeps the group from ever picking deployment-b, so only a pin reaches it @@ -846,6 +850,11 @@ def _video_router() -> Router: }, "model_info": {"id": "deployment-b"}, }, + { + "model_name": "kling", + "litellm_params": {"model": f"fal_ai/{FAL_MODEL}", "api_key": "sk-mock-fal"}, + "model_info": {"id": "deployment-fal"}, + }, ] ) @@ -904,3 +913,19 @@ async def test_status__real_router_reaches_the_creating_deployment(harness, monk assert deployment_b.call_count == 1 assert status.status == "completed" + + +@pytest.mark.asyncio +async def test_status__fal_id_keeps_the_model_path_fal_reads(harness): + with patch.object(proxy_server, "llm_router", _video_router()): + await call_status(harness, encode_video_id_with_provider("req-123", "fal_ai", "deployment-fal")) + data = harness.processor_data() + url, _ = FalAIVideoConfig().transform_video_status_retrieve_request( + video_id=data["video_id"], + api_base="https://queue.fal.run", + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + + assert data["model"] == "deployment-fal" + assert url == "https://queue.fal.run/fal-ai/kling-video/requests/req-123/status" From 88b948ef339adb4b92d084e65def29a322faa09f Mon Sep 17 00:00:00 2001 From: Yash Raj Pandey Date: Tue, 29 Sep 2026 14:52:35 -0400 Subject: [PATCH 06/15] fix(proxy): keep the deployment in the video status id The provider status transforms re-encode the video id with an empty model_id. The OpenAI SDK create_and_poll() returns the last status object, so download_content() sends that id. The proxy found no model in it and sent the content call to the provider with the environment credentials, not with the credentials of the deployment. Re-encode the status response id with the routed deployment, as create, remix, edit and extension already do. When the proxy has no deployment for the call, keep the model_id that the provider wrote and do not clear it. --- litellm/proxy/video_endpoints/endpoints.py | 3 +- litellm/proxy/video_endpoints/utils.py | 2 +- .../proxy/video_endpoints/test_endpoints.py | 28 +++++++++++++++++++ 3 files changed, 31 insertions(+), 2 deletions(-) diff --git a/litellm/proxy/video_endpoints/endpoints.py b/litellm/proxy/video_endpoints/endpoints.py index e180b573e31..78c0a18b52c 100644 --- a/litellm/proxy/video_endpoints/endpoints.py +++ b/litellm/proxy/video_endpoints/endpoints.py @@ -300,6 +300,7 @@ async def video_status( user_api_base=user_api_base, version=version, ) + encoded_status: Final = encode_video_id_in_response(status, deployment_id_for_encoding(status, data)) except Exception as e: raise await processor._handle_llm_api_exception( e=e, @@ -308,7 +309,7 @@ async def video_status( version=version, ) else: - return status + return encoded_status @router.get( diff --git a/litellm/proxy/video_endpoints/utils.py b/litellm/proxy/video_endpoints/utils.py index d93dab4a494..bee22a3df03 100644 --- a/litellm/proxy/video_endpoints/utils.py +++ b/litellm/proxy/video_endpoints/utils.py @@ -104,7 +104,7 @@ def encode_video_id_in_response(response: object, fallback_model: str | None) -> encoded: Final = encode_video_id_with_provider( video_id=decoded.get("video_id", ""), provider=_hidden_param(response, "custom_llm_provider") or decoded.get("custom_llm_provider") or "openai", - model_id=_hidden_param(response, "model_id") or fallback_model, + model_id=_hidden_param(response, "model_id") or fallback_model or decoded.get("model_id"), ) if isinstance(response, dict): response["id"] = encoded # rebind-ok: in-place id rewrite, matching encode_character_id_in_response diff --git a/tests/test_litellm/proxy/video_endpoints/test_endpoints.py b/tests/test_litellm/proxy/video_endpoints/test_endpoints.py index 4a2ebc79185..ba709ca611d 100644 --- a/tests/test_litellm/proxy/video_endpoints/test_endpoints.py +++ b/tests/test_litellm/proxy/video_endpoints/test_endpoints.py @@ -929,3 +929,31 @@ async def test_status__fal_id_keeps_the_model_path_fal_reads(harness): assert data["model"] == "deployment-fal" assert url == "https://queue.fal.run/fal-ai/kling-video/requests/req-123/status" + + +@pytest.mark.asyncio +async def test_status__response_id_keeps_the_deployment_for_content(harness): + # The provider drops the model from status ids, and the OpenAI SDK create_and_poll() downloads with the status id + harness.base_process.return_value = VideoObject( + id=encode_video_id_with_provider("video_orig", "openai", None), object="video", status="completed" + ) + + with patch.object(proxy_server, "llm_router", _video_router()): + status = await call_status(harness, _video_id("deployment-b")) + harness.base_process.reset_mock() + harness.base_process.return_value = b"video-bytes" + await call_content(harness, status.id) + + assert decode_video_id_with_provider(status.id)["model_id"] == "deployment-b" + assert harness.processor_data().get("model") == "deployment-b" + + +@pytest.mark.asyncio +async def test_status__without_router_keeps_the_provider_model_in_the_id(harness): + fal_status_id = encode_video_id_with_provider("req-123", "fal_ai", "fal-ai/kling-video") + harness.base_process.return_value = VideoObject(id=fal_status_id, object="video", status="completed") + + with patch.object(proxy_server, "llm_router", None): + status = await call_status(harness, fal_status_id) + + assert status.id == fal_status_id From be1765609fb6416ec5d08e62d08c5acd915208c7 Mon Sep 17 00:00:00 2001 From: Yash Raj Pandey Date: Tue, 29 Sep 2026 15:03:10 -0400 Subject: [PATCH 07/15] style(video): drop code comments, name the test rows instead The repo keeps code comments for complex business logic only. The fal test covers why the provider id is restored, and pytest.param ids now name each follow-up table row. --- litellm/proxy/video_endpoints/utils.py | 1 - .../proxy/video_endpoints/test_endpoints.py | 12 ++++-------- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/litellm/proxy/video_endpoints/utils.py b/litellm/proxy/video_endpoints/utils.py index bee22a3df03..51d2cbd5ddf 100644 --- a/litellm/proxy/video_endpoints/utils.py +++ b/litellm/proxy/video_endpoints/utils.py @@ -82,7 +82,6 @@ def routing_model_for_id(llm_router: Router, model_id: str) -> str | None: def video_id_for_provider(llm_router: Router, video_id: str) -> str: - # fal.ai reads its model path back from the id, so swap the deployment id for the model the deployment calls decoded: Final = decode_video_id_with_provider(video_id) provider: Final = decoded.get("custom_llm_provider") deployment: Final = llm_router.get_deployment(model_id=decoded.get("model_id") or "") diff --git a/tests/test_litellm/proxy/video_endpoints/test_endpoints.py b/tests/test_litellm/proxy/video_endpoints/test_endpoints.py index ba709ca611d..a029b2016f0 100644 --- a/tests/test_litellm/proxy/video_endpoints/test_endpoints.py +++ b/tests/test_litellm/proxy/video_endpoints/test_endpoints.py @@ -827,7 +827,6 @@ FAL_MODEL = "fal-ai/kling-video/v2/master/text-to-video" def _video_router() -> Router: - # weight 0 keeps the group from ever picking deployment-b, so only a pin reaches it return Router( model_list=[ { @@ -880,12 +879,10 @@ FOLLOW_UP_CALLS = { @pytest.mark.parametrize( ("model_id", "expected_model"), [ - ("deployment-b", "deployment-b"), - # ids created before the fix name the group, so they still load-balance - ("sora-2", "sora-2"), - ("", None), - # a deleted deployment takes the direct provider call, as it did before the fix - ("deployment-gone", None), + pytest.param("deployment-b", "deployment-b", id="deployment_id_pins"), + pytest.param("sora-2", "sora-2", id="group_id_load_balances"), + pytest.param("", None, id="no_model_in_id"), + pytest.param("deployment-gone", None, id="deleted_deployment_calls_provider_directly"), ], ) async def test_follow_up__model_comes_from_the_id(harness, endpoint, model_id, expected_model): @@ -933,7 +930,6 @@ async def test_status__fal_id_keeps_the_model_path_fal_reads(harness): @pytest.mark.asyncio async def test_status__response_id_keeps_the_deployment_for_content(harness): - # The provider drops the model from status ids, and the OpenAI SDK create_and_poll() downloads with the status id harness.base_process.return_value = VideoObject( id=encode_video_id_with_provider("video_orig", "openai", None), object="video", status="completed" ) From d216e379edd96a73219dc00686087b1c91ec9fb6 Mon Sep 17 00:00:00 2001 From: Yash Raj Pandey Date: Tue, 29 Sep 2026 15:14:28 -0400 Subject: [PATCH 08/15] test(video): tell the mock router the id is not a deployment id The follow-up endpoints now ask the router whether the model in the id is a deployment id, and edit and extension look up that deployment. A bare MagicMock answers both with a truthy mock, so the character, edit and extension tests pinned to the provider model. The mock now answers the way a real Router does for a model name. --- tests/unit/test_video_generation.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/unit/test_video_generation.py b/tests/unit/test_video_generation.py index 5c1d0bfa884..c2bddfcb0f9 100644 --- a/tests/unit/test_video_generation.py +++ b/tests/unit/test_video_generation.py @@ -2310,6 +2310,7 @@ def test_video_get_character_accepts_encoded_character_id(video_proxy_test_clien ) mock_router = MagicMock() mock_router.resolve_model_name_from_model_id.return_value = "vertex-ai-sora-2" + mock_router.has_model_id.return_value = False with patch("litellm.proxy.proxy_server.llm_router", mock_router): with patch.object( @@ -2467,6 +2468,8 @@ def test_edit_and_extension_route_with_encoded_video_ids( mock_router = MagicMock() mock_router.resolve_model_name_from_model_id.return_value = "vertex-ai-sora-2" + mock_router.has_model_id.return_value = False + mock_router.get_deployment.return_value = None with patch("litellm.proxy.proxy_server.llm_router", mock_router): with patch.object( From c9d648e1f6d23246a5afeb11ed5fa23e1c72ccb1 Mon Sep 17 00:00:00 2001 From: Yash Raj Pandey Date: Tue, 29 Sep 2026 15:23:05 -0400 Subject: [PATCH 09/15] style(video): read the decoded video id with get, as the encoder does DecodedVideoId marks video_id as not required, so basedpyright flags the subscript. encode_video_id_in_response already reads it with get. --- litellm/proxy/video_endpoints/utils.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/video_endpoints/utils.py b/litellm/proxy/video_endpoints/utils.py index 51d2cbd5ddf..80da7a37270 100644 --- a/litellm/proxy/video_endpoints/utils.py +++ b/litellm/proxy/video_endpoints/utils.py @@ -91,7 +91,9 @@ def video_id_for_provider(llm_router: Router, video_id: str) -> str: model=deployment.litellm_params.model, custom_llm_provider=deployment.litellm_params.custom_llm_provider, )[0] - return encode_video_id_with_provider(video_id=decoded["video_id"], provider=provider, model_id=provider_model) + return encode_video_id_with_provider( + video_id=decoded.get("video_id", ""), provider=provider, model_id=provider_model + ) def encode_video_id_in_response(response: object, fallback_model: str | None) -> object: From d0a65a3c0cd77137151376ad76c81a90e89f1fea Mon Sep 17 00:00:00 2001 From: Yash Raj Pandey Date: Tue, 29 Sep 2026 16:25:38 -0400 Subject: [PATCH 10/15] fix(proxy): check the key's access groups before pinning a video id A follow-up video call now routes to the deployment id in the video id. The router applies no access-group filter to a deployment id, so a key limited to one access group could reach a deployment of another access group with a crafted id. Before the proxy pins the deployment, run the router's access-group filter for the caller. When the filter drops the deployment, return the same 400 that the router returns when its access-group filter leaves no deployment. --- litellm/proxy/video_endpoints/endpoints.py | 12 ++-- litellm/proxy/video_endpoints/utils.py | 24 ++++++-- .../proxy/video_endpoints/test_endpoints.py | 59 ++++++++++++++++++- 3 files changed, 84 insertions(+), 11 deletions(-) diff --git a/litellm/proxy/video_endpoints/endpoints.py b/litellm/proxy/video_endpoints/endpoints.py index 78c0a18b52c..25079debcac 100644 --- a/litellm/proxy/video_endpoints/endpoints.py +++ b/litellm/proxy/video_endpoints/endpoints.py @@ -274,7 +274,7 @@ async def video_status( # Resolve model_name from model_id if available # This allows the router to automatically inject litellm_params from the model config if model_id_from_decoded and llm_router: - resolved_model: Final = routing_model_for_id(llm_router, model_id_from_decoded) + resolved_model: Final = routing_model_for_id(llm_router, model_id_from_decoded, user_api_key_dict) if resolved_model: data["model"] = resolved_model data["video_id"] = video_id_for_provider(llm_router, video_id) @@ -376,7 +376,7 @@ async def video_content( # Resolve model_name from model_id if available # This allows the router to automatically inject litellm_params from the model config if model_id_from_decoded and llm_router: - resolved_model: Final = routing_model_for_id(llm_router, model_id_from_decoded) + resolved_model: Final = routing_model_for_id(llm_router, model_id_from_decoded, user_api_key_dict) if resolved_model: data["model"] = resolved_model data["video_id"] = video_id_for_provider(llm_router, video_id) @@ -485,7 +485,7 @@ async def video_remix( # Resolve model_name from model_id if available # This allows the router to automatically inject litellm_params from the model config if model_id_from_decoded and llm_router: - resolved_model: Final = routing_model_for_id(llm_router, model_id_from_decoded) + resolved_model: Final = routing_model_for_id(llm_router, model_id_from_decoded, user_api_key_dict) if resolved_model: data["model"] = resolved_model data["video_id"] = video_id_for_provider(llm_router, video_id) @@ -690,7 +690,7 @@ async def video_get_character( data["custom_llm_provider"] = custom_llm_provider if model_id_from_decoded and llm_router: - resolved_model: Final = routing_model_for_id(llm_router, model_id_from_decoded) + resolved_model: Final = routing_model_for_id(llm_router, model_id_from_decoded, user_api_key_dict) if resolved_model: data["model"] = resolved_model @@ -801,7 +801,7 @@ async def video_edit( data["custom_llm_provider"] = custom_llm_provider if model_id_from_decoded and llm_router: - resolved_model: Final = routing_model_for_id(llm_router, model_id_from_decoded) + resolved_model: Final = routing_model_for_id(llm_router, model_id_from_decoded, user_api_key_dict) if resolved_model: data["model"] = resolved_model data["video_id"] = video_id_for_provider(llm_router, data["video_id"]) @@ -900,7 +900,7 @@ async def video_extension( data["custom_llm_provider"] = custom_llm_provider if model_id_from_decoded and llm_router: - resolved_model: Final = routing_model_for_id(llm_router, model_id_from_decoded) + resolved_model: Final = routing_model_for_id(llm_router, model_id_from_decoded, user_api_key_dict) if resolved_model: data["model"] = resolved_model data["video_id"] = video_id_for_provider(llm_router, data["video_id"]) diff --git a/litellm/proxy/video_endpoints/utils.py b/litellm/proxy/video_endpoints/utils.py index 80da7a37270..1aceb17728a 100644 --- a/litellm/proxy/video_endpoints/utils.py +++ b/litellm/proxy/video_endpoints/utils.py @@ -2,9 +2,12 @@ from collections.abc import Mapping from typing import Any, Final import orjson +from fastapi import status from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider +from litellm.proxy._types import ProxyException, UserAPIKeyAuth from litellm.router import Router +from litellm.types.router import RouterErrors from litellm.types.videos.utils import ( decode_video_id_with_provider, encode_character_id_with_provider, @@ -75,10 +78,23 @@ def deployment_id_for_encoding(response: object, data: Mapping[str, Any]) -> str return _hidden_param(response, "model_id") or model_info.get("id") or data.get("model") -def routing_model_for_id(llm_router: Router, model_id: str) -> str | None: - if llm_router.has_model_id(model_id): - return model_id - return llm_router.resolve_model_name_from_model_id(model_id) +def routing_model_for_id(llm_router: Router, model_id: str, user_api_key_dict: UserAPIKeyAuth) -> str | None: + if not llm_router.has_model_id(model_id): + return llm_router.resolve_model_name_from_model_id(model_id) + deployment: Final = llm_router.get_deployment(model_id=model_id) + if deployment is not None and not llm_router._filter_deployments_by_model_access_groups( # pyright: ignore[reportPrivateUsage, reportUnknownMemberType] # the router skips its access-group filter for a deployment id, so reuse that untyped filter here + model=deployment.model_name, + healthy_deployments=[deployment.model_dump(exclude_none=True)], + request_kwargs={"metadata": {"user_api_key_auth": user_api_key_dict}}, + request_team_id=user_api_key_dict.team_id, + ): + raise ProxyException( + message=f"litellm.BadRequestError: You passed in model={deployment.model_name}. {RouterErrors.no_healthy_deployments.value}", + type="invalid_request_error", + param=None, + code=status.HTTP_400_BAD_REQUEST, + ) + return model_id def video_id_for_provider(llm_router: Router, video_id: str) -> str: diff --git a/tests/test_litellm/proxy/video_endpoints/test_endpoints.py b/tests/test_litellm/proxy/video_endpoints/test_endpoints.py index a029b2016f0..4bcb76befd8 100644 --- a/tests/test_litellm/proxy/video_endpoints/test_endpoints.py +++ b/tests/test_litellm/proxy/video_endpoints/test_endpoints.py @@ -40,7 +40,7 @@ import litellm import litellm.proxy.proxy_server as proxy_server import litellm.proxy.video_endpoints.endpoints as endpoints from litellm.llms.fal_ai.videos.transformation import FalAIVideoConfig -from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy._types import LitellmUserRoles, ProxyException, UserAPIKeyAuth from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing from litellm.proxy.utils import ProxyLogging from litellm.router import Router @@ -953,3 +953,60 @@ async def test_status__without_router_keeps_the_provider_model_in_the_id(harness status = await call_status(harness, fal_status_id) assert status.id == fal_status_id + + +def _access_group_router() -> Router: + return Router( + model_list=[ + { + "model_name": "sora-2", + "litellm_params": {"model": "openai/sora-2", "api_key": "sk-mock-a"}, + "model_info": {"id": "deployment-a", "access_groups": ["group-a"]}, + }, + { + "model_name": "sora-2", + "litellm_params": {"model": "openai/sora-2", "api_key": "sk-mock-b"}, + "model_info": {"id": "deployment-b", "access_groups": ["group-b"]}, + }, + ] + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("endpoint", sorted(FOLLOW_UP_CALLS)) +async def test_follow_up__id_outside_the_key_access_group_is_rejected(harness, monkeypatch, endpoint): + monkeypatch.setitem(globals(), "_user", lambda: UserAPIKeyAuth(api_key="sk-test", models=["group-a"])) + + with patch.object(proxy_server, "llm_router", _access_group_router()): + with pytest.raises(ProxyException) as rejected: + await FOLLOW_UP_CALLS[endpoint](harness, "deployment-b") + + assert rejected.value.code == "400" + assert rejected.value.message == ( + "litellm.BadRequestError: You passed in model=sora-2. There are no healthy deployments for this model" + ) + assert harness.base_process.call_count == 0 + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("key", "model_id"), + [ + pytest.param(UserAPIKeyAuth(api_key="sk-test", models=["group-a"]), "deployment-a", id="own_access_group"), + pytest.param(UserAPIKeyAuth(api_key="sk-test", models=["group-a", "group-b"]), "deployment-b", id="both_groups"), + pytest.param(UserAPIKeyAuth(api_key="sk-test", models=["sora-2"]), "deployment-b", id="group_name"), + pytest.param(UserAPIKeyAuth(api_key="sk-test", models=["*"]), "deployment-b", id="wildcard"), + pytest.param(UserAPIKeyAuth(api_key="sk-test", models=["all-proxy-models"]), "deployment-b", id="all_proxy"), + pytest.param( + UserAPIKeyAuth(api_key="sk-test", user_role=LitellmUserRoles.PROXY_ADMIN), "deployment-b", id="admin" + ), + ], +) +async def test_status__id_inside_the_key_access_pins_the_deployment(harness, monkeypatch, key, model_id): + monkeypatch.setitem(globals(), "_user", lambda: key) + harness.base_process.return_value = b"video-bytes" + + with patch.object(proxy_server, "llm_router", _access_group_router()): + await call_status(harness, _video_id(model_id)) + + assert harness.processor_data().get("model") == model_id From fc073844d720d7a9a93439cc1fbcbbd64c54283a Mon Sep 17 00:00:00 2001 From: Yash Raj Pandey Date: Wed, 30 Sep 2026 15:42:07 -0400 Subject: [PATCH 11/15] fix(proxy): check the key's team before pinning a video id The router also skips its team filter for a deployment id. Edit and extension carry the video id in the body, so the auth layer checks no model for them. A key of one team could then reach a team-scoped deployment of another team with a crafted id. Run the router's team filter on the pinned deployment as well, with the deployment's model name as the model group, as the router sets it. --- litellm/proxy/video_endpoints/utils.py | 27 ++++++++--- .../proxy/video_endpoints/test_endpoints.py | 47 +++++++++++++++++++ 2 files changed, 68 insertions(+), 6 deletions(-) diff --git a/litellm/proxy/video_endpoints/utils.py b/litellm/proxy/video_endpoints/utils.py index 1aceb17728a..1307f67c5cb 100644 --- a/litellm/proxy/video_endpoints/utils.py +++ b/litellm/proxy/video_endpoints/utils.py @@ -7,6 +7,7 @@ from fastapi import status from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider from litellm.proxy._types import ProxyException, UserAPIKeyAuth from litellm.router import Router +from litellm.router_utils.common_utils import filter_team_based_models from litellm.types.router import RouterErrors from litellm.types.videos.utils import ( decode_video_id_with_provider, @@ -82,12 +83,26 @@ def routing_model_for_id(llm_router: Router, model_id: str, user_api_key_dict: U if not llm_router.has_model_id(model_id): return llm_router.resolve_model_name_from_model_id(model_id) deployment: Final = llm_router.get_deployment(model_id=model_id) - if deployment is not None and not llm_router._filter_deployments_by_model_access_groups( # pyright: ignore[reportPrivateUsage, reportUnknownMemberType] # the router skips its access-group filter for a deployment id, so reuse that untyped filter here - model=deployment.model_name, - healthy_deployments=[deployment.model_dump(exclude_none=True)], - request_kwargs={"metadata": {"user_api_key_auth": user_api_key_dict}}, - request_team_id=user_api_key_dict.team_id, - ): + if deployment is None: + return model_id + # The router skips its access-group and team filters for a deployment id, so run both for the caller here. + request_kwargs: Final = { + "metadata": { + "user_api_key_auth": user_api_key_dict, + "user_api_key_team_id": user_api_key_dict.team_id, + "model_group": deployment.model_name, + } + } + allowed: Final = filter_team_based_models( + llm_router._filter_deployments_by_model_access_groups( # pyright: ignore[reportPrivateUsage, reportUnknownMemberType] # untyped router filter, reused because the deployment-id path skips it + model=deployment.model_name, + healthy_deployments=[deployment.model_dump(exclude_none=True)], + request_kwargs=request_kwargs, + request_team_id=user_api_key_dict.team_id, + ), + request_kwargs, + ) + if not allowed: raise ProxyException( message=f"litellm.BadRequestError: You passed in model={deployment.model_name}. {RouterErrors.no_healthy_deployments.value}", type="invalid_request_error", diff --git a/tests/test_litellm/proxy/video_endpoints/test_endpoints.py b/tests/test_litellm/proxy/video_endpoints/test_endpoints.py index 4bcb76befd8..108a3500d8e 100644 --- a/tests/test_litellm/proxy/video_endpoints/test_endpoints.py +++ b/tests/test_litellm/proxy/video_endpoints/test_endpoints.py @@ -1010,3 +1010,50 @@ async def test_status__id_inside_the_key_access_pins_the_deployment(harness, mon await call_status(harness, _video_id(model_id)) assert harness.processor_data().get("model") == model_id + + +def _team_router() -> Router: + return Router( + model_list=[ + { + "model_name": "model_name_team-b_sora", + "litellm_params": {"model": "openai/sora-2", "api_key": "sk-mock-team-b"}, + "model_info": {"id": "team-b-deployment", "team_id": "team-b", "team_public_model_name": "sora-2"}, + } + ] + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("endpoint", sorted(FOLLOW_UP_CALLS)) +async def test_follow_up__id_of_another_team_deployment_is_rejected(harness, monkeypatch, endpoint): + monkeypatch.setitem(globals(), "_user", lambda: UserAPIKeyAuth(api_key="sk-test", team_id="team-a")) + + with patch.object(proxy_server, "llm_router", _team_router()): + with pytest.raises(ProxyException) as rejected: + await FOLLOW_UP_CALLS[endpoint](harness, "team-b-deployment") + + assert rejected.value.code == "400" + assert rejected.value.message == ( + "litellm.BadRequestError: You passed in model=model_name_team-b_sora. " + "There are no healthy deployments for this model" + ) + assert harness.base_process.call_count == 0 + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "key", + [ + pytest.param(UserAPIKeyAuth(api_key="sk-test", team_id="team-b"), id="same_team"), + pytest.param(UserAPIKeyAuth(api_key="sk-test", user_role=LitellmUserRoles.PROXY_ADMIN), id="admin"), + ], +) +async def test_status__id_of_a_team_deployment_pins_it_for_that_team(harness, monkeypatch, key): + monkeypatch.setitem(globals(), "_user", lambda: key) + harness.base_process.return_value = b"video-bytes" + + with patch.object(proxy_server, "llm_router", _team_router()): + await call_status(harness, _video_id("team-b-deployment")) + + assert harness.processor_data().get("model") == "team-b-deployment" From ab808042a62ce2cdf26a60bc40a51442bd5be1ef Mon Sep 17 00:00:00 2001 From: Yash Raj Pandey Date: Wed, 30 Sep 2026 17:13:00 -0400 Subject: [PATCH 12/15] fix(proxy): keep the model group and pin the video deployment in metadata Follow-up video calls sent the deployment id as the model. Model-level guardrails, limits and budgets look up the model group by that name, find no deployment, and skip. A remix, edit or extension with a blocked prompt then reached the provider. Keep the model group as the model and put the deployment id in metadata. The router runs its access-group and team filters first, then keeps only the pinned deployment, and returns the same 400 when the filters drop it. The sync routing path applies the pin too. --- litellm/constants.py | 1 + litellm/proxy/video_endpoints/endpoints.py | 26 ++--- litellm/proxy/video_endpoints/utils.py | 55 ++++------- litellm/router.py | 4 + litellm/router_utils/common_utils.py | 29 +++++- .../proxy/video_endpoints/test_endpoints.py | 98 ++++++++++++++----- .../test_router_utils_common_utils.py | 70 ++++++++++++- 7 files changed, 199 insertions(+), 84 deletions(-) diff --git a/litellm/constants.py b/litellm/constants.py index 39c10d71709..93c05ff235b 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -1541,6 +1541,7 @@ LITELLM_METADATA_FIELD: Final = "litellm_metadata" OLD_LITELLM_METADATA_FIELD: Final = "metadata" RETURN_RAW_MODEL_NAME_METADATA_KEY: Final = "_complexity_router_return_raw_model_name" SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY: Final = "_session_deployment_affinity_ttl" +PINNED_DEPLOYMENT_ID_METADATA_KEY: Final = "pinned_deployment_id" OUTPUT_TOKEN_CEILING_PARAMS: Final = frozenset({"max_tokens", "max_completion_tokens", "max_output_tokens"}) CLIENT_OUTPUT_CEILING_METADATA_KEY: Final = "_client_output_ceiling" CONSUMED_REQUEST_TAGS_METADATA_KEY: Final = "_consumed_request_tags" diff --git a/litellm/proxy/video_endpoints/endpoints.py b/litellm/proxy/video_endpoints/endpoints.py index 25079debcac..71bb21c4524 100644 --- a/litellm/proxy/video_endpoints/endpoints.py +++ b/litellm/proxy/video_endpoints/endpoints.py @@ -22,7 +22,7 @@ from litellm.proxy.video_endpoints.utils import ( encode_video_id_in_response, extract_model_from_target_model_names, get_custom_provider_from_data, - routing_model_for_id, + route_to_encoded_deployment, video_id_for_provider, video_reference_to_id, ) @@ -274,9 +274,7 @@ async def video_status( # Resolve model_name from model_id if available # This allows the router to automatically inject litellm_params from the model config if model_id_from_decoded and llm_router: - resolved_model: Final = routing_model_for_id(llm_router, model_id_from_decoded, user_api_key_dict) - if resolved_model: - data["model"] = resolved_model + route_to_encoded_deployment(llm_router, model_id_from_decoded, data) data["video_id"] = video_id_for_provider(llm_router, video_id) # Process request using ProxyBaseLLMRequestProcessing @@ -376,9 +374,7 @@ async def video_content( # Resolve model_name from model_id if available # This allows the router to automatically inject litellm_params from the model config if model_id_from_decoded and llm_router: - resolved_model: Final = routing_model_for_id(llm_router, model_id_from_decoded, user_api_key_dict) - if resolved_model: - data["model"] = resolved_model + route_to_encoded_deployment(llm_router, model_id_from_decoded, data) data["video_id"] = video_id_for_provider(llm_router, video_id) # Process request using ProxyBaseLLMRequestProcessing processor: Final = ProxyBaseLLMRequestProcessing(data=data) @@ -485,9 +481,7 @@ async def video_remix( # Resolve model_name from model_id if available # This allows the router to automatically inject litellm_params from the model config if model_id_from_decoded and llm_router: - resolved_model: Final = routing_model_for_id(llm_router, model_id_from_decoded, user_api_key_dict) - if resolved_model: - data["model"] = resolved_model + route_to_encoded_deployment(llm_router, model_id_from_decoded, data) data["video_id"] = video_id_for_provider(llm_router, video_id) # Process request using ProxyBaseLLMRequestProcessing @@ -690,9 +684,7 @@ async def video_get_character( data["custom_llm_provider"] = custom_llm_provider if model_id_from_decoded and llm_router: - resolved_model: Final = routing_model_for_id(llm_router, model_id_from_decoded, user_api_key_dict) - if resolved_model: - data["model"] = resolved_model + route_to_encoded_deployment(llm_router, model_id_from_decoded, data) processor: Final = ProxyBaseLLMRequestProcessing(data=data) try: @@ -801,9 +793,7 @@ async def video_edit( data["custom_llm_provider"] = custom_llm_provider if model_id_from_decoded and llm_router: - resolved_model: Final = routing_model_for_id(llm_router, model_id_from_decoded, user_api_key_dict) - if resolved_model: - data["model"] = resolved_model + route_to_encoded_deployment(llm_router, model_id_from_decoded, data) data["video_id"] = video_id_for_provider(llm_router, data["video_id"]) processor: Final = ProxyBaseLLMRequestProcessing(data=data) @@ -900,9 +890,7 @@ async def video_extension( data["custom_llm_provider"] = custom_llm_provider if model_id_from_decoded and llm_router: - resolved_model: Final = routing_model_for_id(llm_router, model_id_from_decoded, user_api_key_dict) - if resolved_model: - data["model"] = resolved_model + route_to_encoded_deployment(llm_router, model_id_from_decoded, data) data["video_id"] = video_id_for_provider(llm_router, data["video_id"]) processor: Final = ProxyBaseLLMRequestProcessing(data=data) diff --git a/litellm/proxy/video_endpoints/utils.py b/litellm/proxy/video_endpoints/utils.py index 1307f67c5cb..8f5cd8e1c85 100644 --- a/litellm/proxy/video_endpoints/utils.py +++ b/litellm/proxy/video_endpoints/utils.py @@ -2,13 +2,11 @@ from collections.abc import Mapping from typing import Any, Final import orjson -from fastapi import status +from litellm.constants import PINNED_DEPLOYMENT_ID_METADATA_KEY from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider -from litellm.proxy._types import ProxyException, UserAPIKeyAuth +from litellm.litellm_core_utils.safe_json_loads import safe_json_loads from litellm.router import Router -from litellm.router_utils.common_utils import filter_team_based_models -from litellm.types.router import RouterErrors from litellm.types.videos.utils import ( decode_video_id_with_provider, encode_character_id_with_provider, @@ -76,40 +74,25 @@ def deployment_id_for_encoding(response: object, data: Mapping[str, Any]) -> str """ litellm_metadata: Final = data.get("litellm_metadata") or {} model_info: Final = litellm_metadata.get("model_info") or {} - return _hidden_param(response, "model_id") or model_info.get("id") or data.get("model") + pinned_id: Final = (data.get("metadata") or {}).get(PINNED_DEPLOYMENT_ID_METADATA_KEY) + return _hidden_param(response, "model_id") or model_info.get("id") or pinned_id or data.get("model") -def routing_model_for_id(llm_router: Router, model_id: str, user_api_key_dict: UserAPIKeyAuth) -> str | None: - if not llm_router.has_model_id(model_id): - return llm_router.resolve_model_name_from_model_id(model_id) - deployment: Final = llm_router.get_deployment(model_id=model_id) - if deployment is None: - return model_id - # The router skips its access-group and team filters for a deployment id, so run both for the caller here. - request_kwargs: Final = { - "metadata": { - "user_api_key_auth": user_api_key_dict, - "user_api_key_team_id": user_api_key_dict.team_id, - "model_group": deployment.model_name, - } - } - allowed: Final = filter_team_based_models( - llm_router._filter_deployments_by_model_access_groups( # pyright: ignore[reportPrivateUsage, reportUnknownMemberType] # untyped router filter, reused because the deployment-id path skips it - model=deployment.model_name, - healthy_deployments=[deployment.model_dump(exclude_none=True)], - request_kwargs=request_kwargs, - request_team_id=user_api_key_dict.team_id, - ), - request_kwargs, - ) - if not allowed: - raise ProxyException( - message=f"litellm.BadRequestError: You passed in model={deployment.model_name}. {RouterErrors.no_healthy_deployments.value}", - type="invalid_request_error", - param=None, - code=status.HTTP_400_BAD_REQUEST, - ) - return model_id +def route_to_encoded_deployment(llm_router: Router, model_id: str, data: dict[str, Any]) -> None: + """Route by the model group, pinned to the deployment the id encodes. + + ``data["model"]`` stays the group so guardrails, limits and budgets keyed on it apply. + The router's own access-group and team filters run first, then the pin narrows the result. + """ + resolved_model: Final = llm_router.resolve_model_name_from_model_id(model_id) + if resolved_model: + data["model"] = resolved_model + if model_id in llm_router.model_names or not llm_router.has_model_id(model_id): + return + raw_metadata: Final = data.get("metadata") or {} + metadata: Final = safe_json_loads(raw_metadata) if isinstance(raw_metadata, str) else raw_metadata + if isinstance(metadata, dict): + data["metadata"] = {**metadata, PINNED_DEPLOYMENT_ID_METADATA_KEY: model_id} def video_id_for_provider(llm_router: Router, video_id: str) -> str: diff --git a/litellm/router.py b/litellm/router.py index 86ba5112435..dabcdb3243b 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -177,6 +177,7 @@ from litellm.router_utils.clientside_credential_handler import ( ) from litellm.router_utils.common_utils import ( _is_proxy_admin_request, + filter_pinned_deployment, filter_team_based_models, filter_web_search_deployments, format_fallback_outcome_message, @@ -12943,6 +12944,8 @@ class Router: if verbose_router_logger.isEnabledFor(logging.DEBUG): verbose_router_logger.debug("healthy_deployments after web search filter: %s", healthy_deployments) + healthy_deployments = filter_pinned_deployment(model, healthy_deployments, request_kwargs) + if isinstance(healthy_deployments, dict): if (healthy_deployments.get("model_info") or {}).get("blocked") is True: raise litellm.ServiceUnavailableError( @@ -14079,6 +14082,7 @@ class Router: specific_deployment=specific_deployment, request_kwargs=request_kwargs, ) + healthy_deployments = filter_pinned_deployment(model, healthy_deployments, request_kwargs) strategy, strategy_selector = self._get_routing_context(model, request_kwargs) if isinstance(healthy_deployments, dict): diff --git a/litellm/router_utils/common_utils.py b/litellm/router_utils/common_utils.py index 7d111a80264..51dbe693e4a 100644 --- a/litellm/router_utils/common_utils.py +++ b/litellm/router_utils/common_utils.py @@ -9,11 +9,11 @@ if TYPE_CHECKING: import litellm from litellm._logging import verbose_logger, verbose_router_logger -from litellm.constants import ROUTER_FALLBACK_ERROR_DETAIL_MAX_CHARS +from litellm.constants import PINNED_DEPLOYMENT_ID_METADATA_KEY, ROUTER_FALLBACK_ERROR_DETAIL_MAX_CHARS from litellm.exceptions import BadRequestError from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider from litellm.litellm_core_utils.sensitive_data_masker import mask_sensitive_structure -from litellm.types.router import CredentialLiteLLMParams +from litellm.types.router import CredentialLiteLLMParams, RouterErrors from litellm.types.utils import LlmProviders @@ -207,6 +207,31 @@ def filter_team_based_models( ] +def filter_pinned_deployment( + model: str, + healthy_deployments: list[dict] | dict, + request_kwargs: Mapping[str, object] | None, +) -> list[dict] | dict: + """Keep only the deployment a follow-up call must reach, such as the one that created a video.""" + if request_kwargs is None or isinstance(healthy_deployments, dict): + return healthy_deployments + buckets: Final = (request_kwargs.get("metadata"), request_kwargs.get("litellm_metadata")) + pinned_ids: Final = [ + bucket.get(PINNED_DEPLOYMENT_ID_METADATA_KEY) for bucket in buckets if isinstance(bucket, Mapping) + ] + pinned_id: Final = next((pinned_id for pinned_id in pinned_ids if pinned_id), None) + if pinned_id is None: + return healthy_deployments + pinned: Final = [d for d in healthy_deployments if (d.get("model_info") or {}).get("id") == pinned_id] + if not pinned: + raise BadRequestError( + message=f"You passed in model={model}. {RouterErrors.no_healthy_deployments.value}", + model=model, + llm_provider="", + ) + return pinned + + def _deployment_supports_web_search(deployment: dict) -> bool: """ Check if a deployment supports web search. diff --git a/tests/test_litellm/proxy/video_endpoints/test_endpoints.py b/tests/test_litellm/proxy/video_endpoints/test_endpoints.py index 108a3500d8e..ed4b6ee1b8b 100644 --- a/tests/test_litellm/proxy/video_endpoints/test_endpoints.py +++ b/tests/test_litellm/proxy/video_endpoints/test_endpoints.py @@ -40,9 +40,9 @@ import litellm import litellm.proxy.proxy_server as proxy_server import litellm.proxy.video_endpoints.endpoints as endpoints from litellm.llms.fal_ai.videos.transformation import FalAIVideoConfig -from litellm.proxy._types import LitellmUserRoles, ProxyException, UserAPIKeyAuth +from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing -from litellm.proxy.utils import ProxyLogging +from litellm.proxy.utils import ProxyLogging, _check_and_merge_model_level_guardrails from litellm.router import Router from litellm.types.router import GenericLiteLLMParams from litellm.types.videos.main import VideoObject @@ -879,7 +879,7 @@ FOLLOW_UP_CALLS = { @pytest.mark.parametrize( ("model_id", "expected_model"), [ - pytest.param("deployment-b", "deployment-b", id="deployment_id_pins"), + pytest.param("deployment-b", "sora-2", id="deployment_id_pins"), pytest.param("sora-2", "sora-2", id="group_id_load_balances"), pytest.param("", None, id="no_model_in_id"), pytest.param("deployment-gone", None, id="deleted_deployment_calls_provider_directly"), @@ -892,6 +892,9 @@ async def test_follow_up__model_comes_from_the_id(harness, endpoint, model_id, e await FOLLOW_UP_CALLS[endpoint](harness, model_id) assert harness.processor_data().get("model") == expected_model + assert (harness.processor_data().get("metadata") or {}).get("pinned_deployment_id") == ( + "deployment-b" if model_id == "deployment-b" else None + ) @pytest.mark.asyncio @@ -924,7 +927,7 @@ async def test_status__fal_id_keeps_the_model_path_fal_reads(harness): headers={}, ) - assert data["model"] == "deployment-fal" + assert data["model"] == "kling" assert url == "https://queue.fal.run/fal-ai/kling-video/requests/req-123/status" @@ -941,7 +944,8 @@ async def test_status__response_id_keeps_the_deployment_for_content(harness): await call_content(harness, status.id) assert decode_video_id_with_provider(status.id)["model_id"] == "deployment-b" - assert harness.processor_data().get("model") == "deployment-b" + assert harness.processor_data().get("model") == "sora-2" + assert harness.processor_data()["metadata"]["pinned_deployment_id"] == "deployment-b" @pytest.mark.asyncio @@ -955,6 +959,45 @@ async def test_status__without_router_keeps_the_provider_model_in_the_id(harness assert status.id == fal_status_id +@pytest.mark.asyncio +@pytest.mark.parametrize("endpoint", sorted(FOLLOW_UP_CALLS)) +async def test_follow_up__pinned_deployment_keeps_its_model_level_guardrails(harness, endpoint): + harness.base_process.return_value = b"video-bytes" + llm_router = Router( + model_list=[ + { + "model_name": "sora-2", + "litellm_params": {"model": "openai/sora-2", "api_key": "sk-mock-b", "guardrails": ["video-guard"]}, + "model_info": {"id": "deployment-b"}, + } + ] + ) + + with patch.object(proxy_server, "llm_router", llm_router): + await FOLLOW_UP_CALLS[endpoint](harness, "deployment-b") + merged = _check_and_merge_model_level_guardrails( + data=harness.processor_data(), llm_router=llm_router, trust_client_model_info=False + ) + + assert merged["metadata"]["guardrails"] == ["video-guard"] + + +@pytest.mark.asyncio +async def test_remix__pin_keeps_the_metadata_sent_as_a_form_string(harness): + harness.base_process.return_value = b"video-bytes" + + with patch.object(proxy_server, "llm_router", _video_router()): + await call_remix(harness, _video_id("deployment-b"), body={"prompt": "x", "metadata": '{"tag": "blue"}'}) + + assert harness.processor_data()["metadata"] == {"tag": "blue", "pinned_deployment_id": "deployment-b"} + + +async def _call_router_as_proxy(llm_router: Router, harness, key: UserAPIKeyAuth): + data = harness.processor_data() + metadata = {**data.get("metadata", {}), "user_api_key_auth": key, "user_api_key_team_id": key.team_id} + return await getattr(llm_router, harness.route_type())(**{**data, "metadata": metadata}) + + def _access_group_router() -> Router: return Router( model_list=[ @@ -975,17 +1018,18 @@ def _access_group_router() -> Router: @pytest.mark.asyncio @pytest.mark.parametrize("endpoint", sorted(FOLLOW_UP_CALLS)) async def test_follow_up__id_outside_the_key_access_group_is_rejected(harness, monkeypatch, endpoint): - monkeypatch.setitem(globals(), "_user", lambda: UserAPIKeyAuth(api_key="sk-test", models=["group-a"])) + key = UserAPIKeyAuth(api_key="sk-test", models=["group-a"]) + monkeypatch.setitem(globals(), "_user", lambda: key) + harness.base_process.return_value = b"video-bytes" + llm_router = _access_group_router() - with patch.object(proxy_server, "llm_router", _access_group_router()): - with pytest.raises(ProxyException) as rejected: - await FOLLOW_UP_CALLS[endpoint](harness, "deployment-b") + with patch.object(proxy_server, "llm_router", llm_router): + await FOLLOW_UP_CALLS[endpoint](harness, "deployment-b") + with pytest.raises(litellm.BadRequestError) as rejected: + await _call_router_as_proxy(llm_router, harness, key) - assert rejected.value.code == "400" - assert rejected.value.message == ( - "litellm.BadRequestError: You passed in model=sora-2. There are no healthy deployments for this model" - ) - assert harness.base_process.call_count == 0 + assert rejected.value.status_code == 400 + assert "You passed in model=sora-2. There are no healthy deployments for this model" in rejected.value.message @pytest.mark.asyncio @@ -1009,7 +1053,8 @@ async def test_status__id_inside_the_key_access_pins_the_deployment(harness, mon with patch.object(proxy_server, "llm_router", _access_group_router()): await call_status(harness, _video_id(model_id)) - assert harness.processor_data().get("model") == model_id + assert harness.processor_data().get("model") == "sora-2" + assert harness.processor_data()["metadata"]["pinned_deployment_id"] == model_id def _team_router() -> Router: @@ -1027,18 +1072,18 @@ def _team_router() -> Router: @pytest.mark.asyncio @pytest.mark.parametrize("endpoint", sorted(FOLLOW_UP_CALLS)) async def test_follow_up__id_of_another_team_deployment_is_rejected(harness, monkeypatch, endpoint): - monkeypatch.setitem(globals(), "_user", lambda: UserAPIKeyAuth(api_key="sk-test", team_id="team-a")) + key = UserAPIKeyAuth(api_key="sk-test", team_id="team-a") + monkeypatch.setitem(globals(), "_user", lambda: key) + harness.base_process.return_value = b"video-bytes" + llm_router = _team_router() - with patch.object(proxy_server, "llm_router", _team_router()): - with pytest.raises(ProxyException) as rejected: - await FOLLOW_UP_CALLS[endpoint](harness, "team-b-deployment") + with patch.object(proxy_server, "llm_router", llm_router): + await FOLLOW_UP_CALLS[endpoint](harness, "team-b-deployment") + with pytest.raises(litellm.BadRequestError) as rejected: + await _call_router_as_proxy(llm_router, harness, key) - assert rejected.value.code == "400" - assert rejected.value.message == ( - "litellm.BadRequestError: You passed in model=model_name_team-b_sora. " - "There are no healthy deployments for this model" - ) - assert harness.base_process.call_count == 0 + assert rejected.value.status_code == 400 + assert "You passed in model=model_name_team-b_sora. There are no healthy deployments" in rejected.value.message @pytest.mark.asyncio @@ -1056,4 +1101,5 @@ async def test_status__id_of_a_team_deployment_pins_it_for_that_team(harness, mo with patch.object(proxy_server, "llm_router", _team_router()): await call_status(harness, _video_id("team-b-deployment")) - assert harness.processor_data().get("model") == "team-b-deployment" + assert harness.processor_data().get("model") == "model_name_team-b_sora" + assert harness.processor_data()["metadata"]["pinned_deployment_id"] == "team-b-deployment" diff --git a/tests/unit/router_utils/test_router_utils_common_utils.py b/tests/unit/router_utils/test_router_utils_common_utils.py index ac18b4889dd..62384f2ed9e 100644 --- a/tests/unit/router_utils/test_router_utils_common_utils.py +++ b/tests/unit/router_utils/test_router_utils_common_utils.py @@ -4,12 +4,14 @@ from unittest.mock import Mock import pytest +import litellm from litellm import Router -from litellm.constants import ROUTER_FALLBACK_ERROR_DETAIL_MAX_CHARS +from litellm.constants import PINNED_DEPLOYMENT_ID_METADATA_KEY, ROUTER_FALLBACK_ERROR_DETAIL_MAX_CHARS from litellm.proxy._types import UserAPIKeyAuth from litellm.router_utils.common_utils import ( _deployment_supports_web_search, add_model_file_id_mappings, + filter_pinned_deployment, filter_team_based_models, filter_web_search_deployments, provider_for_generic_call, @@ -774,3 +776,69 @@ class TestWarnOnProviderCredentialMismatch: def test_provider_for_generic_call(litellm_params, expected, monkeypatch): monkeypatch.setenv("AZURE_AI_API_BASE", "https://unrelated.openai.azure.com") assert provider_for_generic_call(litellm_params) == expected + + +def _pinned_router(routing_strategy: str) -> Router: + return Router( + model_list=[ + { + "model_name": "sora-2", + "litellm_params": {"model": "openai/sora-2", "api_key": f"sk-mock-{name}", "weight": weight}, + "model_info": {"id": f"deployment-{name}"}, + } + for name, weight in (("a", 1), ("b", 0)) + ], + routing_strategy=routing_strategy, + ) + + +@pytest.mark.parametrize("bucket", ["metadata", "litellm_metadata"]) +def test_filter_pinned_deployment_keeps_only_the_pinned_id(bucket): + deployments = [{"model_info": {"id": "deployment-a"}}, {"model_info": {"id": "deployment-b"}}] + + kept = filter_pinned_deployment("sora-2", deployments, {bucket: {PINNED_DEPLOYMENT_ID_METADATA_KEY: "deployment-b"}}) + + assert kept == [{"model_info": {"id": "deployment-b"}}] + + +def test_filter_pinned_deployment_without_a_pin_keeps_every_deployment(): + deployments = [{"model_info": {"id": "deployment-a"}}, {"model_info": {"id": "deployment-b"}}] + + assert filter_pinned_deployment("sora-2", deployments, {"metadata": {}}) == deployments + + +def test_filter_pinned_deployment_rejects_a_pin_outside_the_candidates(): + with pytest.raises(litellm.BadRequestError) as rejected: + filter_pinned_deployment( + "sora-2", + [{"model_info": {"id": "deployment-a"}}], + {"metadata": {PINNED_DEPLOYMENT_ID_METADATA_KEY: "deployment-b"}}, + ) + + assert rejected.value.status_code == 400 + assert "You passed in model=sora-2. There are no healthy deployments for this model" in rejected.value.message + + +@pytest.mark.asyncio +async def test_async_routing_reaches_the_pinned_deployment(): + router = _pinned_router("simple-shuffle") + request_kwargs = {"metadata": {PINNED_DEPLOYMENT_ID_METADATA_KEY: "deployment-b"}} + + picked = [ + (await router.async_get_available_deployment(model="sora-2", request_kwargs=request_kwargs))["model_info"]["id"] + for _ in range(20) + ] + + assert picked == ["deployment-b"] * 20 + + +def test_sync_routing_reaches_the_pinned_deployment(): + router = _pinned_router("usage-based-routing") + request_kwargs = {"metadata": {PINNED_DEPLOYMENT_ID_METADATA_KEY: "deployment-b"}} + + picked = [ + router.get_available_deployment(model="sora-2", request_kwargs=request_kwargs)["model_info"]["id"] + for _ in range(20) + ] + + assert picked == ["deployment-b"] * 20 From 4137323f6d0c961099cdb6f1c7c1ce7aa8af7a8a Mon Sep 17 00:00:00 2001 From: Yash Raj Pandey Date: Wed, 30 Sep 2026 17:57:01 -0400 Subject: [PATCH 13/15] fix(proxy): keep the model group and pin the video deployment in metadata Follow-up video calls now keep the model group as the model, so guardrails, limits and budgets of the group apply. The endpoint passes the deployment id to the request processor. The processor sets the pin in the metadata after add_litellm_data_to_request, which strips any pin the client sent, and disables fallbacks for the call. The router retry path also narrows its candidates to the pinned deployment. --- litellm/proxy/common_request_processing.py | 13 +++- litellm/proxy/litellm_pre_call_utils.py | 2 + litellm/proxy/video_endpoints/endpoints.py | 63 ++++++++++++++----- litellm/proxy/video_endpoints/utils.py | 20 +++--- litellm/router.py | 8 ++- litellm/router_utils/common_utils.py | 23 ++++--- .../proxy/test_common_request_processing.py | 40 ++++++++++++ .../proxy/test_litellm_pre_call_utils.py | 2 + .../proxy/video_endpoints/test_endpoints.py | 33 ++++++---- .../test_router_utils_common_utils.py | 19 ++++++ 10 files changed, 171 insertions(+), 52 deletions(-) diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 97de2488b8d..6d1216a0373 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -41,6 +41,7 @@ from litellm.constants import ( MAX_LITELLM_CALL_ID_LENGTH, MAX_PAYLOAD_SIZE_FOR_DEBUG_LOG, NON_INFERENCE_CALL_TYPES, + PINNED_DEPLOYMENT_ID_METADATA_KEY, RETURN_RAW_MODEL_NAME_METADATA_KEY, STREAM_SSE_DATA_PREFIX, STREAM_SSE_KEEPALIVE_PING_BYTES, @@ -236,6 +237,7 @@ from litellm.proxy.anthropic_endpoints.streaming_model_restamp import ( AnthropicStreamModelRestamper, ) from litellm.proxy.litellm_pre_call_utils import ( + _get_metadata_variable_name, add_litellm_data_to_request, refresh_proxy_server_request_body_snapshot, reject_url_valued_destination, @@ -1684,8 +1686,9 @@ def _timing_values( class ProxyBaseLLMRequestProcessing: - def __init__(self, data: dict): + def __init__(self, data: dict, pinned_deployment_id: str | None = None): self.data = data + self.pinned_deployment_id: Final = pinned_deployment_id self._tags_before_guardrails: frozenset[str] | None = None @property @@ -2034,6 +2037,12 @@ class ProxyBaseLLMRequestProcessing: version=version, proxy_config=proxy_config, ) + if self.pinned_deployment_id is not None: + # Set after add_litellm_data_to_request, which strips any pin the client sent + self.data[_get_metadata_variable_name(request)][PINNED_DEPLOYMENT_ID_METADATA_KEY] = ( + self.pinned_deployment_id + ) + self.data["disable_fallbacks"] = True if not general_settings.get("expose_fallback_errors_to_caller"): self.data.pop("include_fallback_errors", None) if route_type in {"aresponses", "_aresponses_websocket"}: @@ -2056,8 +2065,6 @@ class ProxyBaseLLMRequestProcessing: # Store queue time in metadata after add_litellm_data_to_request to ensure it's preserved if queue_time_seconds is not None: - from litellm.proxy.litellm_pre_call_utils import _get_metadata_variable_name - _metadata_variable_name: Final = _get_metadata_variable_name(request) if _metadata_variable_name not in self.data: self.data[_metadata_variable_name] = {} diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index d48451de6b1..30f28f3a124 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -24,6 +24,7 @@ from litellm.constants import ( INTERNAL_CALL_ORIGIN_METADATA_KEY, LITELLM_PROXY_MASTER_KEY_ALIAS, OTEL_SERVICE_NAME_METADATA_KEYS, + PINNED_DEPLOYMENT_ID_METADATA_KEY, PRE_CALL_EXECUTED_GUARDRAILS_KEY, ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY, ROUTING_REQUEST_TAGS_METADATA_KEY, @@ -379,6 +380,7 @@ _ROUTER_RESERVED_METADATA_FIELDS: Final = frozenset( "request_retry_count", CLIENT_OUTPUT_CEILING_METADATA_KEY, ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY, + PINNED_DEPLOYMENT_ID_METADATA_KEY, } ) _ALLOW_CLIENT_PRICING_OVERRIDE_METADATA_KEY: Final = "allow_client_pricing_override" diff --git a/litellm/proxy/video_endpoints/endpoints.py b/litellm/proxy/video_endpoints/endpoints.py index 71bb21c4524..8638cc2e5e4 100644 --- a/litellm/proxy/video_endpoints/endpoints.py +++ b/litellm/proxy/video_endpoints/endpoints.py @@ -273,12 +273,16 @@ async def video_status( # Resolve model_name from model_id if available # This allows the router to automatically inject litellm_params from the model config - if model_id_from_decoded and llm_router: + pinned_deployment_id: Final = ( route_to_encoded_deployment(llm_router, model_id_from_decoded, data) + if model_id_from_decoded and llm_router + else None + ) + if model_id_from_decoded and llm_router: data["video_id"] = video_id_for_provider(llm_router, video_id) # Process request using ProxyBaseLLMRequestProcessing - processor: Final = ProxyBaseLLMRequestProcessing(data=data) + processor: Final = ProxyBaseLLMRequestProcessing(data=data, pinned_deployment_id=pinned_deployment_id) try: status: Final[object] = await processor.base_process_llm_request( request=request, @@ -298,7 +302,9 @@ async def video_status( user_api_base=user_api_base, version=version, ) - encoded_status: Final = encode_video_id_in_response(status, deployment_id_for_encoding(status, data)) + encoded_status: Final = encode_video_id_in_response( + status, deployment_id_for_encoding(status, data, pinned_deployment_id) + ) except Exception as e: raise await processor._handle_llm_api_exception( e=e, @@ -373,11 +379,15 @@ async def video_content( # Resolve model_name from model_id if available # This allows the router to automatically inject litellm_params from the model config - if model_id_from_decoded and llm_router: + pinned_deployment_id: Final = ( route_to_encoded_deployment(llm_router, model_id_from_decoded, data) + if model_id_from_decoded and llm_router + else None + ) + if model_id_from_decoded and llm_router: data["video_id"] = video_id_for_provider(llm_router, video_id) # Process request using ProxyBaseLLMRequestProcessing - processor: Final = ProxyBaseLLMRequestProcessing(data=data) + processor: Final = ProxyBaseLLMRequestProcessing(data=data, pinned_deployment_id=pinned_deployment_id) try: # Call the video content function directly to get raw bytes video_bytes: Final = await processor.base_process_llm_request( @@ -480,12 +490,16 @@ async def video_remix( # Resolve model_name from model_id if available # This allows the router to automatically inject litellm_params from the model config - if model_id_from_decoded and llm_router: + pinned_deployment_id: Final = ( route_to_encoded_deployment(llm_router, model_id_from_decoded, data) + if model_id_from_decoded and llm_router + else None + ) + if model_id_from_decoded and llm_router: data["video_id"] = video_id_for_provider(llm_router, video_id) # Process request using ProxyBaseLLMRequestProcessing - processor: Final = ProxyBaseLLMRequestProcessing(data=data) + processor: Final = ProxyBaseLLMRequestProcessing(data=data, pinned_deployment_id=pinned_deployment_id) try: remixed: Final[object] = await processor.base_process_llm_request( request=request, @@ -505,7 +519,9 @@ async def video_remix( user_api_base=user_api_base, version=version, ) - encoded_response: Final = encode_video_id_in_response(remixed, deployment_id_for_encoding(remixed, data)) + encoded_response: Final = encode_video_id_in_response( + remixed, deployment_id_for_encoding(remixed, data, pinned_deployment_id) + ) except Exception as e: raise await processor._handle_llm_api_exception( e=e, @@ -683,10 +699,13 @@ async def video_get_character( ) data["custom_llm_provider"] = custom_llm_provider - if model_id_from_decoded and llm_router: + pinned_deployment_id: Final = ( route_to_encoded_deployment(llm_router, model_id_from_decoded, data) + if model_id_from_decoded and llm_router + else None + ) - processor: Final = ProxyBaseLLMRequestProcessing(data=data) + processor: Final = ProxyBaseLLMRequestProcessing(data=data, pinned_deployment_id=pinned_deployment_id) try: response: object = await processor.base_process_llm_request( request=request, @@ -792,11 +811,15 @@ async def video_edit( ) data["custom_llm_provider"] = custom_llm_provider - if model_id_from_decoded and llm_router: + pinned_deployment_id: Final = ( route_to_encoded_deployment(llm_router, model_id_from_decoded, data) + if model_id_from_decoded and llm_router + else None + ) + if model_id_from_decoded and llm_router: data["video_id"] = video_id_for_provider(llm_router, data["video_id"]) - processor: Final = ProxyBaseLLMRequestProcessing(data=data) + processor: Final = ProxyBaseLLMRequestProcessing(data=data, pinned_deployment_id=pinned_deployment_id) try: edited: Final[object] = await processor.base_process_llm_request( request=request, @@ -816,7 +839,9 @@ async def video_edit( user_api_base=user_api_base, version=version, ) - encoded_response: Final = encode_video_id_in_response(edited, deployment_id_for_encoding(edited, data)) + encoded_response: Final = encode_video_id_in_response( + edited, deployment_id_for_encoding(edited, data, pinned_deployment_id) + ) except Exception as e: raise await processor._handle_llm_api_exception( e=e, @@ -889,11 +914,15 @@ async def video_extension( ) data["custom_llm_provider"] = custom_llm_provider - if model_id_from_decoded and llm_router: + pinned_deployment_id: Final = ( route_to_encoded_deployment(llm_router, model_id_from_decoded, data) + if model_id_from_decoded and llm_router + else None + ) + if model_id_from_decoded and llm_router: data["video_id"] = video_id_for_provider(llm_router, data["video_id"]) - processor: Final = ProxyBaseLLMRequestProcessing(data=data) + processor: Final = ProxyBaseLLMRequestProcessing(data=data, pinned_deployment_id=pinned_deployment_id) try: extended: Final[object] = await processor.base_process_llm_request( request=request, @@ -913,7 +942,9 @@ async def video_extension( user_api_base=user_api_base, version=version, ) - encoded_response: Final = encode_video_id_in_response(extended, deployment_id_for_encoding(extended, data)) + encoded_response: Final = encode_video_id_in_response( + extended, deployment_id_for_encoding(extended, data, pinned_deployment_id) + ) except Exception as e: raise await processor._handle_llm_api_exception( e=e, diff --git a/litellm/proxy/video_endpoints/utils.py b/litellm/proxy/video_endpoints/utils.py index 8f5cd8e1c85..94a5b986656 100644 --- a/litellm/proxy/video_endpoints/utils.py +++ b/litellm/proxy/video_endpoints/utils.py @@ -3,9 +3,7 @@ from typing import Any, Final import orjson -from litellm.constants import PINNED_DEPLOYMENT_ID_METADATA_KEY from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider -from litellm.litellm_core_utils.safe_json_loads import safe_json_loads from litellm.router import Router from litellm.types.videos.utils import ( decode_video_id_with_provider, @@ -64,7 +62,9 @@ def _hidden_param(response: object, key: str) -> str | None: return value if isinstance(value, str) else None -def deployment_id_for_encoding(response: object, data: Mapping[str, Any]) -> str | None: +def deployment_id_for_encoding( + response: object, data: Mapping[str, Any], pinned_deployment_id: str | None = None +) -> str | None: """The deployment the request actually routed to, not the public model group. The router leaves ``model_id`` out of ``_hidden_params`` on the generic path and @@ -74,12 +74,11 @@ def deployment_id_for_encoding(response: object, data: Mapping[str, Any]) -> str """ litellm_metadata: Final = data.get("litellm_metadata") or {} model_info: Final = litellm_metadata.get("model_info") or {} - pinned_id: Final = (data.get("metadata") or {}).get(PINNED_DEPLOYMENT_ID_METADATA_KEY) - return _hidden_param(response, "model_id") or model_info.get("id") or pinned_id or data.get("model") + return _hidden_param(response, "model_id") or model_info.get("id") or pinned_deployment_id or data.get("model") -def route_to_encoded_deployment(llm_router: Router, model_id: str, data: dict[str, Any]) -> None: - """Route by the model group, pinned to the deployment the id encodes. +def route_to_encoded_deployment(llm_router: Router, model_id: str, data: dict[str, Any]) -> str | None: + """Route by the model group and return the deployment id to pin, if the id encodes one. ``data["model"]`` stays the group so guardrails, limits and budgets keyed on it apply. The router's own access-group and team filters run first, then the pin narrows the result. @@ -88,11 +87,8 @@ def route_to_encoded_deployment(llm_router: Router, model_id: str, data: dict[st if resolved_model: data["model"] = resolved_model if model_id in llm_router.model_names or not llm_router.has_model_id(model_id): - return - raw_metadata: Final = data.get("metadata") or {} - metadata: Final = safe_json_loads(raw_metadata) if isinstance(raw_metadata, str) else raw_metadata - if isinstance(metadata, dict): - data["metadata"] = {**metadata, PINNED_DEPLOYMENT_ID_METADATA_KEY: model_id} + return None + return model_id def video_id_for_provider(llm_router: Router, video_id: str) -> str: diff --git a/litellm/router.py b/litellm/router.py index dabcdb3243b..78c8c9cd7e1 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -183,6 +183,7 @@ from litellm.router_utils.common_utils import ( format_fallback_outcome_message, format_no_fallback_group_message, get_request_team_id, + pinned_deployment_id, provider_for_generic_call, resolve_model_group_alias, truncate_fallback_error_detail, @@ -7678,6 +7679,7 @@ class Router: ) = await self._async_get_healthy_deployments( model=kwargs.get("model") or "", parent_otel_span=parent_otel_span, + request_kwargs=kwargs, ) # Check retry policy FIRST, before should_retry_this_error @@ -7770,6 +7772,7 @@ class Router: ) = await self._async_get_healthy_deployments( model=_model, parent_otel_span=parent_otel_span, + request_kwargs=kwargs, ) else: _healthy_deployments = [] @@ -8545,7 +8548,7 @@ class Router: return healthy_deployments, _all_deployments async def _async_get_healthy_deployments( - self, model: str, parent_otel_span: Span | None + self, model: str, parent_otel_span: Span | None, request_kwargs: Mapping[str, object] | None = None ) -> tuple[list[dict], list[dict]]: """ Returns Tuple of: @@ -8562,6 +8565,9 @@ class Router: return [], _all_deployments except Exception: pass + pinned_id: Final = pinned_deployment_id(request_kwargs) + if pinned_id is not None: + _all_deployments = [d for d in _all_deployments if d["model_info"]["id"] == pinned_id] unhealthy_deployments: Final = await _async_get_cooldown_deployments( litellm_router_instance=self, parent_otel_span=parent_otel_span diff --git a/litellm/router_utils/common_utils.py b/litellm/router_utils/common_utils.py index 51dbe693e4a..4141fb06e32 100644 --- a/litellm/router_utils/common_utils.py +++ b/litellm/router_utils/common_utils.py @@ -207,20 +207,25 @@ def filter_team_based_models( ] +def pinned_deployment_id(request_kwargs: Mapping[str, object] | None) -> str | None: + """The deployment a follow-up call must reach, such as the one that created a video.""" + if request_kwargs is None: + return None + buckets: Final = (request_kwargs.get("metadata"), request_kwargs.get("litellm_metadata")) + pinned_ids: Final = [ + bucket.get(PINNED_DEPLOYMENT_ID_METADATA_KEY) for bucket in buckets if isinstance(bucket, Mapping) + ] + return next((pinned_id for pinned_id in pinned_ids if isinstance(pinned_id, str) and pinned_id), None) + + def filter_pinned_deployment( model: str, healthy_deployments: list[dict] | dict, request_kwargs: Mapping[str, object] | None, ) -> list[dict] | dict: - """Keep only the deployment a follow-up call must reach, such as the one that created a video.""" - if request_kwargs is None or isinstance(healthy_deployments, dict): - return healthy_deployments - buckets: Final = (request_kwargs.get("metadata"), request_kwargs.get("litellm_metadata")) - pinned_ids: Final = [ - bucket.get(PINNED_DEPLOYMENT_ID_METADATA_KEY) for bucket in buckets if isinstance(bucket, Mapping) - ] - pinned_id: Final = next((pinned_id for pinned_id in pinned_ids if pinned_id), None) - if pinned_id is None: + """Keep only the pinned deployment, if the request has one.""" + pinned_id: Final = pinned_deployment_id(request_kwargs) + if pinned_id is None or isinstance(healthy_deployments, dict): return healthy_deployments pinned: Final = [d for d in healthy_deployments if (d.get("model_info") or {}).get("id") == pinned_id] if not pinned: diff --git a/tests/test_litellm/proxy/test_common_request_processing.py b/tests/test_litellm/proxy/test_common_request_processing.py index c17f41a8b8f..61e3fb424a4 100644 --- a/tests/test_litellm/proxy/test_common_request_processing.py +++ b/tests/test_litellm/proxy/test_common_request_processing.py @@ -2043,6 +2043,46 @@ class TestProxyBaseLLMRequestProcessing: logging_obj.start_time.timestamp(), abs=1e-6 ) + @pytest.mark.asyncio + async def test_pinned_deployment_is_set_after_the_client_metadata_is_stripped(self, monkeypatch): + processing_obj = ProxyBaseLLMRequestProcessing( + data={"model": "sora-2", "metadata": {"pinned_deployment_id": "deployment-a"}}, + pinned_deployment_id="deployment-b", + ) + mock_request = MagicMock(spec=Request) + mock_request.headers = {} + mock_request.url = MagicMock() + mock_request.url.path = "/v1/videos/video_123" + + async def strip_client_metadata(*args, **kwargs): + data = kwargs["data"] + data["metadata"] = {} + data["disable_fallbacks"] = False + return data + + async def mock_pre_call_hook(user_api_key_dict, data, call_type, skip_guardrails=False): + return copy.deepcopy(data) + + mock_proxy_logging_obj = MagicMock(spec=ProxyLogging) + mock_proxy_logging_obj.pre_call_hook = AsyncMock(side_effect=mock_pre_call_hook) + monkeypatch.setattr( + litellm.proxy.common_request_processing, + "add_litellm_data_to_request", + strip_client_metadata, + ) + + returned_data, _ = await processing_obj.common_processing_pre_call_logic( + request=mock_request, + general_settings={}, + user_api_key_dict=ProxyUserAPIKeyAuth(api_key="sk-test"), + proxy_logging_obj=mock_proxy_logging_obj, + proxy_config=MagicMock(spec=ProxyConfig), + route_type="avideo_status", + ) + + assert returned_data["metadata"]["pinned_deployment_id"] == "deployment-b" + assert returned_data["disable_fallbacks"] is True + @pytest.mark.asyncio class TestCommonRequestProcessingHelpers: diff --git a/tests/test_litellm/proxy/test_litellm_pre_call_utils.py b/tests/test_litellm/proxy/test_litellm_pre_call_utils.py index 84266325226..8631ebafb16 100644 --- a/tests/test_litellm/proxy/test_litellm_pre_call_utils.py +++ b/tests/test_litellm/proxy/test_litellm_pre_call_utils.py @@ -7791,6 +7791,7 @@ _PLANTED_STAMPS = { "request_retry_count": -100, "_client_output_ceiling": {"api_base": "https://attacker.example"}, ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY: 10**9, + "pinned_deployment_id": "deployment-b", "client_key": "client_value", } @@ -7824,6 +7825,7 @@ async def test_add_litellm_data_to_request_strips_router_reserved_stamps_from_bo assert "_client_output_ceiling" not in updated["metadata"] assert "request_retry_count" not in updated["metadata"] assert ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY not in updated["metadata"] + assert "pinned_deployment_id" not in updated["metadata"] assert updated["metadata"]["client_key"] == "client_value" diff --git a/tests/test_litellm/proxy/video_endpoints/test_endpoints.py b/tests/test_litellm/proxy/video_endpoints/test_endpoints.py index ed4b6ee1b8b..5005282813e 100644 --- a/tests/test_litellm/proxy/video_endpoints/test_endpoints.py +++ b/tests/test_litellm/proxy/video_endpoints/test_endpoints.py @@ -111,6 +111,9 @@ class Harness: assert self.base_process.call_count == 1 return dict(self.base_process.call_args.args[0].data) + def pinned_deployment_id(self) -> str | None: + return self.base_process.call_args.args[0].pinned_deployment_id + def route_type(self) -> str: return self.base_process.call_args.kwargs["route_type"] @@ -892,9 +895,7 @@ async def test_follow_up__model_comes_from_the_id(harness, endpoint, model_id, e await FOLLOW_UP_CALLS[endpoint](harness, model_id) assert harness.processor_data().get("model") == expected_model - assert (harness.processor_data().get("metadata") or {}).get("pinned_deployment_id") == ( - "deployment-b" if model_id == "deployment-b" else None - ) + assert harness.pinned_deployment_id() == ("deployment-b" if model_id == "deployment-b" else None) @pytest.mark.asyncio @@ -909,7 +910,10 @@ async def test_status__real_router_reaches_the_creating_deployment(harness, monk deployment_b = respx_mock.get("http://b.localhost/v1/videos/video_orig").respond( json={"id": "video_orig", "object": "video", "status": "completed", "created_at": 0} ) - status = await llm_router.avideo_status(**harness.processor_data()) + data = harness.processor_data() + status = await llm_router.avideo_status( + **{**data, "metadata": {**data.get("metadata", {}), "pinned_deployment_id": harness.pinned_deployment_id()}} + ) assert deployment_b.call_count == 1 assert status.status == "completed" @@ -945,7 +949,7 @@ async def test_status__response_id_keeps_the_deployment_for_content(harness): assert decode_video_id_with_provider(status.id)["model_id"] == "deployment-b" assert harness.processor_data().get("model") == "sora-2" - assert harness.processor_data()["metadata"]["pinned_deployment_id"] == "deployment-b" + assert harness.pinned_deployment_id() == "deployment-b" @pytest.mark.asyncio @@ -983,18 +987,25 @@ async def test_follow_up__pinned_deployment_keeps_its_model_level_guardrails(har @pytest.mark.asyncio -async def test_remix__pin_keeps_the_metadata_sent_as_a_form_string(harness): +@pytest.mark.parametrize("metadata", [False, [], "", '{"tag": "blue"}']) +async def test_remix__pinned_id_leaves_the_client_metadata_for_the_proxy_to_validate(harness, metadata): harness.base_process.return_value = b"video-bytes" with patch.object(proxy_server, "llm_router", _video_router()): - await call_remix(harness, _video_id("deployment-b"), body={"prompt": "x", "metadata": '{"tag": "blue"}'}) + await call_remix(harness, _video_id("deployment-b"), body={"prompt": "x", "metadata": metadata}) - assert harness.processor_data()["metadata"] == {"tag": "blue", "pinned_deployment_id": "deployment-b"} + assert harness.processor_data()["metadata"] == metadata + assert harness.pinned_deployment_id() == "deployment-b" async def _call_router_as_proxy(llm_router: Router, harness, key: UserAPIKeyAuth): data = harness.processor_data() - metadata = {**data.get("metadata", {}), "user_api_key_auth": key, "user_api_key_team_id": key.team_id} + metadata = { + **data.get("metadata", {}), + "pinned_deployment_id": harness.pinned_deployment_id(), + "user_api_key_auth": key, + "user_api_key_team_id": key.team_id, + } return await getattr(llm_router, harness.route_type())(**{**data, "metadata": metadata}) @@ -1054,7 +1065,7 @@ async def test_status__id_inside_the_key_access_pins_the_deployment(harness, mon await call_status(harness, _video_id(model_id)) assert harness.processor_data().get("model") == "sora-2" - assert harness.processor_data()["metadata"]["pinned_deployment_id"] == model_id + assert harness.pinned_deployment_id() == model_id def _team_router() -> Router: @@ -1102,4 +1113,4 @@ async def test_status__id_of_a_team_deployment_pins_it_for_that_team(harness, mo await call_status(harness, _video_id("team-b-deployment")) assert harness.processor_data().get("model") == "model_name_team-b_sora" - assert harness.processor_data()["metadata"]["pinned_deployment_id"] == "team-b-deployment" + assert harness.pinned_deployment_id() == "team-b-deployment" diff --git a/tests/unit/router_utils/test_router_utils_common_utils.py b/tests/unit/router_utils/test_router_utils_common_utils.py index 62384f2ed9e..88b335e27fa 100644 --- a/tests/unit/router_utils/test_router_utils_common_utils.py +++ b/tests/unit/router_utils/test_router_utils_common_utils.py @@ -842,3 +842,22 @@ def test_sync_routing_reaches_the_pinned_deployment(): ] assert picked == ["deployment-b"] * 20 + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("request_kwargs", "expected_ids"), + [ + ({"metadata": {PINNED_DEPLOYMENT_ID_METADATA_KEY: "deployment-b"}}, ["deployment-b"]), + ({"metadata": {}}, ["deployment-a", "deployment-b"]), + ], +) +async def test_retry_candidates_of_a_pinned_request_are_only_the_pinned_deployment(request_kwargs, expected_ids): + router = _pinned_router("simple-shuffle") + + healthy, all_deployments = await router._async_get_healthy_deployments( + model="sora-2", parent_otel_span=None, request_kwargs=request_kwargs + ) + + assert [d["model_info"]["id"] for d in healthy] == expected_ids + assert [d["model_info"]["id"] for d in all_deployments] == expected_ids From 78cb4408f5681121fd286627d192083826e03ee9 Mon Sep 17 00:00:00 2001 From: Yash Raj Pandey Date: Fri, 2 Oct 2026 12:11:16 -0400 Subject: [PATCH 14/15] fix: satisfy type checks for video deployment routing --- litellm/proxy/video_endpoints/endpoints.py | 32 +++++++++++++++++----- litellm/proxy/video_endpoints/utils.py | 6 ++-- litellm/router.py | 21 +++++++++++--- 3 files changed, 46 insertions(+), 13 deletions(-) diff --git a/litellm/proxy/video_endpoints/endpoints.py b/litellm/proxy/video_endpoints/endpoints.py index 8638cc2e5e4..5e16d347c95 100644 --- a/litellm/proxy/video_endpoints/endpoints.py +++ b/litellm/proxy/video_endpoints/endpoints.py @@ -1,6 +1,6 @@ #### Video Endpoints ##### -from typing import Final +from typing import Any, Final, cast from fastapi import APIRouter, Depends, File, Form, Request, Response, UploadFile from fastapi.responses import ORJSONResponse @@ -84,7 +84,10 @@ async def video_generation( ) # Read request body - data: Final = await _read_request_body(request=request) + data: Final = cast( # cast-ok: _read_request_body is legacy-typed as bare dict + dict[str, Any], + await _read_request_body(request=request), + ) if input_reference is not None: input_reference_file: Final = await batch_to_bytesio([input_reference]) if input_reference_file: @@ -472,7 +475,10 @@ async def video_remix( version, ) - data: Final = await _read_request_body(request=request) + data: Final = cast( # cast-ok: _read_request_body is legacy-typed as bare dict + dict[str, Any], + await _read_request_body(request=request), + ) data["video_id"] = video_id decoded: Final = decode_video_id_with_provider(video_id) @@ -580,7 +586,10 @@ async def video_create_character( version, ) - data: Final = await _read_request_body(request=request) + data: Final = cast( # cast-ok: _read_request_body is legacy-typed as bare dict + dict[str, Any], + await _read_request_body(request=request), + ) video_file: Final = await batch_to_bytesio([video]) if video_file: data["video"] = video_file[0] @@ -685,7 +694,10 @@ async def video_get_character( decoded: Final = decode_character_id_with_provider(character_id) provider_from_id: Final = decoded.get("custom_llm_provider") - model_id_from_decoded: Final = decoded.get("model_id") + raw_model_id_from_decoded: Final = decoded.get("model_id") + model_id_from_decoded: Final[str | None] = ( + raw_model_id_from_decoded if isinstance(raw_model_id_from_decoded, str) else None + ) decoded_character_id: Final = decoded.get("character_id") if decoded_character_id: data["character_id"] = decoded_character_id @@ -788,7 +800,10 @@ async def video_edit( version, ) - data: Final = await _read_request_body(request=request) + data: Final = cast( # cast-ok: _read_request_body is legacy-typed as bare dict + dict[str, Any], + await _read_request_body(request=request), + ) uploaded_video: Final = data.pop("video", None) if isinstance(uploaded_video, StarletteUploadFile): video_files: Final = await batch_to_bytesio((uploaded_video,)) @@ -898,7 +913,10 @@ async def video_extension( version, ) - data: Final = await _read_request_body(request=request) + data: Final = cast( # cast-ok: _read_request_body is legacy-typed as bare dict + dict[str, Any], + await _read_request_body(request=request), + ) data["video_id"] = video_reference_to_id(data.pop("video", None)) decoded: Final = decode_video_id_with_provider(data["video_id"]) diff --git a/litellm/proxy/video_endpoints/utils.py b/litellm/proxy/video_endpoints/utils.py index 94a5b986656..b2feb245e53 100644 --- a/litellm/proxy/video_endpoints/utils.py +++ b/litellm/proxy/video_endpoints/utils.py @@ -107,6 +107,8 @@ def video_id_for_provider(llm_router: Router, video_id: str) -> str: def encode_video_id_in_response(response: object, fallback_model: str | None) -> object: + hidden_provider: Final = _hidden_param(response, "custom_llm_provider") + hidden_model_id: Final = _hidden_param(response, "model_id") video_id: Final = response.get("id") if isinstance(response, dict) else getattr(response, "id", None) if not isinstance(video_id, str) or not video_id: return response @@ -114,8 +116,8 @@ def encode_video_id_in_response(response: object, fallback_model: str | None) -> decoded: Final = decode_video_id_with_provider(video_id) encoded: Final = encode_video_id_with_provider( video_id=decoded.get("video_id", ""), - provider=_hidden_param(response, "custom_llm_provider") or decoded.get("custom_llm_provider") or "openai", - model_id=_hidden_param(response, "model_id") or fallback_model or decoded.get("model_id"), + provider=hidden_provider or decoded.get("custom_llm_provider") or "openai", + model_id=hidden_model_id or fallback_model or decoded.get("model_id"), ) if isinstance(response, dict): response["id"] = encoded # rebind-ok: in-place id rewrite, matching encode_character_id_in_response diff --git a/litellm/router.py b/litellm/router.py index 8781a3c1258..5d175628997 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -7659,6 +7659,9 @@ class Router: @tracer.wrap() async def async_function_with_retries(self, *args, **kwargs): + typed_request_kwargs: Final = cast( # cast-ok: Python kwargs always have str keys and arbitrary object values + Mapping[str, object], kwargs + ) verbose_router_logger.debug("Inside async function with retries.") original_function: Final = kwargs.pop("original_function") fallbacks: Final = kwargs.pop("fallbacks", self.fallbacks) @@ -7716,7 +7719,7 @@ class Router: ) = await self._async_get_healthy_deployments( model=kwargs.get("model") or "", parent_otel_span=parent_otel_span, - request_kwargs=kwargs, + request_kwargs=typed_request_kwargs, ) # Check retry policy FIRST, before should_retry_this_error @@ -7809,7 +7812,7 @@ class Router: ) = await self._async_get_healthy_deployments( model=_model, parent_otel_span=parent_otel_span, - request_kwargs=kwargs, + request_kwargs=typed_request_kwargs, ) else: _healthy_deployments = [] @@ -13026,7 +13029,10 @@ class Router: if verbose_router_logger.isEnabledFor(logging.DEBUG): verbose_router_logger.debug("healthy_deployments after web search filter: %s", healthy_deployments) - healthy_deployments = filter_pinned_deployment(model, healthy_deployments, request_kwargs) + pinned_request_kwargs: Final = cast( # cast-ok: this API's legacy bare dict is a str-keyed request mapping + Mapping[str, object], request_kwargs + ) + healthy_deployments = filter_pinned_deployment(model, healthy_deployments, pinned_request_kwargs) if isinstance(healthy_deployments, dict): if (healthy_deployments.get("model_info") or {}).get("blocked") is True: @@ -14174,7 +14180,14 @@ class Router: specific_deployment=specific_deployment, request_kwargs=request_kwargs, ) - healthy_deployments = filter_pinned_deployment(model, healthy_deployments, request_kwargs) + pinned_request_kwargs: Final = ( + cast( # cast-ok: this API's legacy bare dict is a str-keyed request mapping + Mapping[str, object], request_kwargs + ) + if request_kwargs is not None + else None + ) + healthy_deployments = filter_pinned_deployment(model, healthy_deployments, pinned_request_kwargs) strategy, strategy_selector = self._get_routing_context(model, request_kwargs) if isinstance(healthy_deployments, dict): From 10cd26d811c6d5c9c0790df8774c635bc04c073f Mon Sep 17 00:00:00 2001 From: Yash Raj Pandey Date: Fri, 2 Oct 2026 12:23:50 -0400 Subject: [PATCH 15/15] fix: satisfy strict lint for video endpoint typing --- litellm/proxy/video_endpoints/endpoints.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/proxy/video_endpoints/endpoints.py b/litellm/proxy/video_endpoints/endpoints.py index 5e16d347c95..4e9809c327d 100644 --- a/litellm/proxy/video_endpoints/endpoints.py +++ b/litellm/proxy/video_endpoints/endpoints.py @@ -1,6 +1,6 @@ #### Video Endpoints ##### -from typing import Any, Final, cast +from typing import Any, Final, cast # noqa: TID251 # request-body boundary needs Any and a checked-by-source cast from fastapi import APIRouter, Depends, File, Form, Request, Response, UploadFile from fastapi.responses import ORJSONResponse