diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/openai_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/openai_passthrough_logging_handler.py index 1c8bce28454..5f6489a69ca 100644 --- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/openai_passthrough_logging_handler.py +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/openai_passthrough_logging_handler.py @@ -229,6 +229,25 @@ class OpenAIPassthroughLoggingHandler(BasePassthroughLoggingHandler): verbose_proxy_logger.warning("Error calculating image editing cost: %s", e) return 0.0 + @staticmethod + def _calculate_embeddings_cost( + litellm_model_response: EmbeddingResponse, + model: str, + custom_llm_provider: str, + ) -> float: + try: + return litellm.completion_cost( + completion_response=litellm_model_response, + model=model, + custom_llm_provider=custom_llm_provider, + call_type="aembedding", + ) + except Exception as e: # noqa: BLE001 # completion_cost raises bare Exception for unmapped models; cost failure must never drop the spend log + verbose_proxy_logger.warning( + "Error calculating embeddings cost for model %s, logging spend with cost 0: %s", model, e + ) + return 0.0 + @staticmethod def _build_responses_api_response_and_cost( model: str, @@ -351,11 +370,10 @@ class OpenAIPassthroughLoggingHandler(BasePassthroughLoggingHandler): model_response_object=EmbeddingResponse(), response_type="embedding", ) - response_cost = litellm.completion_cost( - completion_response=litellm_model_response, + response_cost = OpenAIPassthroughLoggingHandler._calculate_embeddings_cost( + litellm_model_response=litellm_model_response, model=model, custom_llm_provider=custom_llm_provider, - call_type="aembedding", ) litellm_model_response._hidden_params["response_cost"] = response_cost elif is_image_generation: @@ -471,6 +489,12 @@ class OpenAIPassthroughLoggingHandler(BasePassthroughLoggingHandler): except Exception as e: verbose_proxy_logger.error("Error in OpenAI passthrough cost tracking: %s", e) + if not is_chat_completions: + unbilled_result: Final[PassThroughEndpointLoggingTypedDict] = { + "result": None, + "kwargs": kwargs, + } + return unbilled_result # Fall back to base handler without cost tracking base_handler = OpenAIPassthroughLoggingHandler() return base_handler.passthrough_chat_handler( diff --git a/litellm/router_utils/fallback_event_handlers.py b/litellm/router_utils/fallback_event_handlers.py index 63bc5203417..3c9a4097321 100644 --- a/litellm/router_utils/fallback_event_handlers.py +++ b/litellm/router_utils/fallback_event_handlers.py @@ -253,6 +253,7 @@ def get_fallback_model_group(fallbacks: list[Any], model_group: str) -> tuple[li PROVIDER_SCOPED_RESOURCE_KEYS: Final = ("input_file_id", "training_file") +PROVIDER_SCOPED_CREATION_FUNCTION_NAMES: Final = frozenset({"_acreate_file"}) def _get_fallback_target_model_group(fallback_entry: str | Mapping[str, object]) -> str | None: @@ -274,6 +275,18 @@ def references_provider_scoped_resource(kwargs: Mapping[str, object]) -> bool: return any(kwargs.get(key) for key in PROVIDER_SCOPED_RESOURCE_KEYS) +def creates_provider_scoped_resource(kwargs: Mapping[str, object]) -> bool: + """ + True when the request creates a resource that will live under one provider's credentials. + + A file uploaded for batches or fine-tuning is stored in the account of the deployment + that handled it, and its id is only usable against the model group the caller named. + Letting the upload fall back to a different model group silently stores the file with + the wrong provider, and every later use of the returned id fails. + """ + return getattr(kwargs.get("original_function"), "__name__", None) in PROVIDER_SCOPED_CREATION_FUNCTION_NAMES + + async def run_async_fallback( *args: tuple[Any], litellm_router: LitellmRouter, @@ -322,7 +335,9 @@ async def run_async_fallback( metadata_variable_name: Final = _get_router_metadata_variable_name( function_name=getattr(kwargs.get("original_function"), "__name__", None) ) - same_model_group_only: Final = references_provider_scoped_resource(kwargs) + same_model_group_only: Final = references_provider_scoped_resource(kwargs) or creates_provider_scoped_resource( + kwargs + ) # Read out of kwargs and narrowed here rather than declared as a parameter: every caller # reaches this function by spreading a loosely-typed kwargs dict, so a declared parameter # would carry an annotation that no call site can actually be checked against. diff --git a/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_openai_passthrough_logging_handler.py b/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_openai_passthrough_logging_handler.py index 664015003e4..69819318800 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_openai_passthrough_logging_handler.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_openai_passthrough_logging_handler.py @@ -1286,6 +1286,75 @@ class TestOpenAIPassthroughIntegration: mock_chat_handler.assert_called_once() assert result == {"result": None, "kwargs": {}} + def test_openai_passthrough_handler_embeddings_unmapped_model_logs_zero_cost(self): + response_body = { + "object": "list", + "model": "lit5787-unmapped-embeddings-deployment", + "data": [{"object": "embedding", "index": 0, "embedding": [0.1, 0.2]}], + "usage": {"prompt_tokens": 9, "total_tokens": 9}, + } + mock_logging_obj = self._create_mock_logging_obj() + result = OpenAIPassthroughLoggingHandler.openai_passthrough_handler( + httpx_response=self._create_mock_httpx_response(response_body), + response_body=response_body, + logging_obj=mock_logging_obj, + url_route="https://my-resource.openai.azure.com/openai/v1/embeddings", + result="", + start_time=self.start_time, + end_time=self.end_time, + cache_hit=False, + request_body={ + "model": "lit5787-unmapped-embeddings-deployment", + "input": "spend probe", + }, + passthrough_logging_payload=PassthroughStandardLoggingPayload( + url="https://my-resource.openai.azure.com/openai/v1/embeddings", + request_body={ + "model": "lit5787-unmapped-embeddings-deployment", + "input": "spend probe", + }, + request_method="POST", + ), + litellm_params={}, + ) + + assert result["result"] is not None + assert result["result"].usage.prompt_tokens == 9 + assert result["kwargs"]["response_cost"] == 0.0 + assert result["kwargs"]["model"] == "lit5787-unmapped-embeddings-deployment" + assert result["result"]._hidden_params["response_cost"] == 0.0 + assert mock_logging_obj.model_call_details["response_cost"] == 0.0 + + def test_openai_passthrough_handler_embeddings_error_skips_chat_fallback(self): + response_body = { + "object": "list", + "model": "text-embedding-3-small", + "usage": {"prompt_tokens": 9, "total_tokens": 9}, + } + kwargs_in = { + "passthrough_logging_payload": PassthroughStandardLoggingPayload( + url="https://api.openai.com/v1/embeddings", + request_body={"model": "text-embedding-3-small", "input": "spend probe"}, + request_method="POST", + ), + "litellm_params": {}, + } + result = OpenAIPassthroughLoggingHandler.openai_passthrough_handler( + httpx_response=self._create_mock_httpx_response(response_body), + response_body=response_body, + logging_obj=self._create_mock_logging_obj(), + url_route="https://api.openai.com/v1/embeddings", + result="", + start_time=self.start_time, + end_time=self.end_time, + cache_hit=False, + request_body={"model": "text-embedding-3-small", "input": "spend probe"}, + **kwargs_in, + ) + + assert result["result"] is None + assert result["kwargs"]["passthrough_logging_payload"] == kwargs_in["passthrough_logging_payload"] + @patch( "litellm.proxy.pass_through_endpoints.llm_provider_handlers.openai_passthrough_logging_handler.OpenAIPassthroughLoggingHandler.openai_passthrough_handler" ) diff --git a/tests/test_litellm/router_utils/test_fallback_event_handlers.py b/tests/test_litellm/router_utils/test_fallback_event_handlers.py index 68395737469..24477248a8a 100644 --- a/tests/test_litellm/router_utils/test_fallback_event_handlers.py +++ b/tests/test_litellm/router_utils/test_fallback_event_handlers.py @@ -1,4 +1,5 @@ import json +from typing import NoReturn from unittest.mock import MagicMock, patch import httpx @@ -167,6 +168,10 @@ async def _acreate_batch(*args, **kwargs): raise AssertionError("only used for its __name__") +async def _acreate_file(*args: object, **kwargs: object) -> NoReturn: + raise AssertionError("only used for its __name__") + + @pytest.mark.asyncio async def test_run_async_fallback_keeps_uploaded_file_requests_in_their_model_group(): """An input_file_id only exists under the credentials of the group it was uploaded @@ -229,6 +234,46 @@ async def test_run_async_fallback_allows_same_model_group_retry_for_uploaded_fil assert router.attempted_model_groups == ["openai-group"] +@pytest.mark.asyncio +async def test_run_async_fallback_keeps_file_creation_in_its_model_group(): + """A file created for batches lands in the account of the deployment that stored it, + and its id is only usable against the model group the caller named. A cross-group + fallback silently stores the file with the wrong provider.""" + router = AttemptRecordingRouter() + + with pytest.raises(RuntimeError, match="azure connection error"): + await run_async_fallback( + litellm_router=router, + fallback_model_group=["openai-group"], + original_model_group="azure-group", + original_exception=RuntimeError("azure connection error"), + max_fallbacks=3, + fallback_depth=0, + model="azure-group", + original_function=_acreate_file, + ) + + assert router.attempted_model_groups == [] + + +@pytest.mark.asyncio +async def test_run_async_fallback_allows_same_model_group_retry_for_file_creation(): + router = AttemptRecordingRouter() + + await run_async_fallback( + litellm_router=router, + fallback_model_group=[{"model": "azure-group", "_target_order": 2}], + original_model_group="azure-group", + original_exception=RuntimeError("first deployment failed"), + max_fallbacks=3, + fallback_depth=0, + model="azure-group", + original_function=_acreate_file, + ) + + assert router.attempted_model_groups == ["azure-group"] + + @pytest.mark.asyncio async def test_run_async_fallback_still_crosses_model_groups_without_an_uploaded_file(): router = AttemptRecordingRouter() diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index fb8438ccb01..d2d5d9268be 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -492,6 +492,54 @@ async def test_async_router_acreate_file_with_jsonl(): assert first_call_content == non_jsonl_content +@pytest.mark.asyncio +async def test_async_router_acreate_file_does_not_fall_back_across_model_groups(): + """A file created for batches only exists under the credentials of the model group + the caller named. A cross-group fallback silently stores it with the wrong provider + and the later batch create against the named group permanently fails.""" + from unittest.mock import MagicMock, patch + + router = litellm.Router( + model_list=[ + { + "model_name": "azure-gpt", + "litellm_params": { + "model": "azure/my-azure-deployment", + "api_base": "http://127.0.0.1:9", + "api_key": "dummy-key", + "api_version": "2024-06-01", + }, + }, + { + "model_name": "openai-gpt", + "litellm_params": {"model": "gpt-4o-mini"}, + }, + ], + fallbacks=[{"azure-gpt": ["openai-gpt"]}], + ) + + def fail_azure(*args: object, **kwargs: object) -> MagicMock: + if kwargs.get("model") == "azure/my-azure-deployment": + raise litellm.APIConnectionError( + message="Connection error.", + llm_provider="azure", + model="azure/my-azure-deployment", + ) + return MagicMock() + + with patch("litellm.acreate_file", side_effect=fail_azure) as mock_acreate_file: + with pytest.raises(litellm.APIConnectionError): + await router.acreate_file( + model="azure-gpt", + purpose="batch", + file=MagicMock(), + ) + + called_models = [call.kwargs.get("model") for call in mock_acreate_file.call_args_list] + assert "azure/my-azure-deployment" in called_models + assert "gpt-4o-mini" not in called_models + + @pytest.mark.asyncio async def test_async_router_acreate_file_uses_deployment_custom_llm_provider(): """