Merge remote-tracking branch 'origin/litellm_internal_staging' into litellm_/inspiring-gauss-6fa7a8

litellm/proxy/pass_through_endpoints/llm_provider_handlers/openai_passthrough_logging_handler.py

@@ -229,6 +229,25 @@ class OpenAIPassthroughLoggingHandler(BasePassthroughLoggingHandler):
             verbose_proxy_logger.warning("Error calculating image editing cost: %s", e)
             return 0.0
 
+    @staticmethod
+    def _calculate_embeddings_cost(
+        litellm_model_response: EmbeddingResponse,
+        model: str,
+        custom_llm_provider: str,
+    ) -> float:
+        try:
+            return litellm.completion_cost(
+                completion_response=litellm_model_response,
+                model=model,
+                custom_llm_provider=custom_llm_provider,
+                call_type="aembedding",
+            )
+        except Exception as e:  # noqa: BLE001  # completion_cost raises bare Exception for unmapped models; cost failure must never drop the spend log
+            verbose_proxy_logger.warning(
+                "Error calculating embeddings cost for model %s, logging spend with cost 0: %s", model, e
+            )
+            return 0.0
+
     @staticmethod
     def _build_responses_api_response_and_cost(
         model: str,
@@ -351,11 +370,10 @@ class OpenAIPassthroughLoggingHandler(BasePassthroughLoggingHandler):
                     model_response_object=EmbeddingResponse(),
                     response_type="embedding",
                 )
-                response_cost = litellm.completion_cost(
-                    completion_response=litellm_model_response,
+                response_cost = OpenAIPassthroughLoggingHandler._calculate_embeddings_cost(
+                    litellm_model_response=litellm_model_response,
                     model=model,
                     custom_llm_provider=custom_llm_provider,
-                    call_type="aembedding",
                 )
                 litellm_model_response._hidden_params["response_cost"] = response_cost
             elif is_image_generation:
@@ -471,6 +489,12 @@ class OpenAIPassthroughLoggingHandler(BasePassthroughLoggingHandler):
 
         except Exception as e:
             verbose_proxy_logger.error("Error in OpenAI passthrough cost tracking: %s", e)
+            if not is_chat_completions:
+                unbilled_result: Final[PassThroughEndpointLoggingTypedDict] = {
+                    "result": None,
+                    "kwargs": kwargs,
+                }
+                return unbilled_result
             # Fall back to base handler without cost tracking
             base_handler = OpenAIPassthroughLoggingHandler()
             return base_handler.passthrough_chat_handler(

litellm/router_utils/fallback_event_handlers.py

@@ -253,6 +253,7 @@ def get_fallback_model_group(fallbacks: list[Any], model_group: str) -> tuple[li
 
 
 PROVIDER_SCOPED_RESOURCE_KEYS: Final = ("input_file_id", "training_file")
+PROVIDER_SCOPED_CREATION_FUNCTION_NAMES: Final = frozenset({"_acreate_file"})
 
 
 def _get_fallback_target_model_group(fallback_entry: str | Mapping[str, object]) -> str | None:
@@ -274,6 +275,18 @@ def references_provider_scoped_resource(kwargs: Mapping[str, object]) -> bool:
     return any(kwargs.get(key) for key in PROVIDER_SCOPED_RESOURCE_KEYS)
 
 
+def creates_provider_scoped_resource(kwargs: Mapping[str, object]) -> bool:
+    """
+    True when the request creates a resource that will live under one provider's credentials.
+
+    A file uploaded for batches or fine-tuning is stored in the account of the deployment
+    that handled it, and its id is only usable against the model group the caller named.
+    Letting the upload fall back to a different model group silently stores the file with
+    the wrong provider, and every later use of the returned id fails.
+    """
+    return getattr(kwargs.get("original_function"), "__name__", None) in PROVIDER_SCOPED_CREATION_FUNCTION_NAMES
+
+
 async def run_async_fallback(
     *args: tuple[Any],
     litellm_router: LitellmRouter,
@@ -322,7 +335,9 @@ async def run_async_fallback(
     metadata_variable_name: Final = _get_router_metadata_variable_name(
         function_name=getattr(kwargs.get("original_function"), "__name__", None)
     )
-    same_model_group_only: Final = references_provider_scoped_resource(kwargs)
+    same_model_group_only: Final = references_provider_scoped_resource(kwargs) or creates_provider_scoped_resource(
+        kwargs
+    )
     # Read out of kwargs and narrowed here rather than declared as a parameter: every caller
     # reaches this function by spreading a loosely-typed kwargs dict, so a declared parameter
     # would carry an annotation that no call site can actually be checked against.

tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_openai_passthrough_logging_handler.py

@@ -1286,6 +1286,75 @@ class TestOpenAIPassthroughIntegration:
         mock_chat_handler.assert_called_once()
         assert result == {"result": None, "kwargs": {}}
 
+    def test_openai_passthrough_handler_embeddings_unmapped_model_logs_zero_cost(self):
+        response_body = {
+            "object": "list",
+            "model": "lit5787-unmapped-embeddings-deployment",
+            "data": [{"object": "embedding", "index": 0, "embedding": [0.1, 0.2]}],
+            "usage": {"prompt_tokens": 9, "total_tokens": 9},
+        }
+        mock_logging_obj = self._create_mock_logging_obj()
+        result = OpenAIPassthroughLoggingHandler.openai_passthrough_handler(
+            httpx_response=self._create_mock_httpx_response(response_body),
+            response_body=response_body,
+            logging_obj=mock_logging_obj,
+            url_route="https://my-resource.openai.azure.com/openai/v1/embeddings",
+            result="",
+            start_time=self.start_time,
+            end_time=self.end_time,
+            cache_hit=False,
+            request_body={
+                "model": "lit5787-unmapped-embeddings-deployment",
+                "input": "spend probe",
+            },
+            passthrough_logging_payload=PassthroughStandardLoggingPayload(
+                url="https://my-resource.openai.azure.com/openai/v1/embeddings",
+                request_body={
+                    "model": "lit5787-unmapped-embeddings-deployment",
+                    "input": "spend probe",
+                },
+                request_method="POST",
+            ),
+            litellm_params={},
+        )
+
+        assert result["result"] is not None
+        assert result["result"].usage.prompt_tokens == 9
+        assert result["kwargs"]["response_cost"] == 0.0
+        assert result["kwargs"]["model"] == "lit5787-unmapped-embeddings-deployment"
+        assert result["result"]._hidden_params["response_cost"] == 0.0
+        assert mock_logging_obj.model_call_details["response_cost"] == 0.0
+
+    def test_openai_passthrough_handler_embeddings_error_skips_chat_fallback(self):
+        response_body = {
+            "object": "list",
+            "model": "text-embedding-3-small",
+            "usage": {"prompt_tokens": 9, "total_tokens": 9},
+        }
+        kwargs_in = {
+            "passthrough_logging_payload": PassthroughStandardLoggingPayload(
+                url="https://api.openai.com/v1/embeddings",
+                request_body={"model": "text-embedding-3-small", "input": "spend probe"},
+                request_method="POST",
+            ),
+            "litellm_params": {},
+        }
+        result = OpenAIPassthroughLoggingHandler.openai_passthrough_handler(
+            httpx_response=self._create_mock_httpx_response(response_body),
+            response_body=response_body,
+            logging_obj=self._create_mock_logging_obj(),
+            url_route="https://api.openai.com/v1/embeddings",
+            result="",
+            start_time=self.start_time,
+            end_time=self.end_time,
+            cache_hit=False,
+            request_body={"model": "text-embedding-3-small", "input": "spend probe"},
+            **kwargs_in,
+        )
+
+        assert result["result"] is None
+        assert result["kwargs"]["passthrough_logging_payload"] == kwargs_in["passthrough_logging_payload"]
+
     @patch(
         "litellm.proxy.pass_through_endpoints.llm_provider_handlers.openai_passthrough_logging_handler.OpenAIPassthroughLoggingHandler.openai_passthrough_handler"
     )

tests/test_litellm/router_utils/test_fallback_event_handlers.py

@@ -1,4 +1,5 @@
 import json
+from typing import NoReturn
 from unittest.mock import MagicMock, patch
 
 import httpx
@@ -167,6 +168,10 @@ async def _acreate_batch(*args, **kwargs):
     raise AssertionError("only used for its __name__")
 
 
+async def _acreate_file(*args: object, **kwargs: object) -> NoReturn:
+    raise AssertionError("only used for its __name__")
+
+
 @pytest.mark.asyncio
 async def test_run_async_fallback_keeps_uploaded_file_requests_in_their_model_group():
     """An input_file_id only exists under the credentials of the group it was uploaded
@@ -229,6 +234,46 @@ async def test_run_async_fallback_allows_same_model_group_retry_for_uploaded_fil
     assert router.attempted_model_groups == ["openai-group"]
 
 
+@pytest.mark.asyncio
+async def test_run_async_fallback_keeps_file_creation_in_its_model_group():
+    """A file created for batches lands in the account of the deployment that stored it,
+    and its id is only usable against the model group the caller named. A cross-group
+    fallback silently stores the file with the wrong provider."""
+    router = AttemptRecordingRouter()
+
+    with pytest.raises(RuntimeError, match="azure connection error"):
+        await run_async_fallback(
+            litellm_router=router,
+            fallback_model_group=["openai-group"],
+            original_model_group="azure-group",
+            original_exception=RuntimeError("azure connection error"),
+            max_fallbacks=3,
+            fallback_depth=0,
+            model="azure-group",
+            original_function=_acreate_file,
+        )
+
+    assert router.attempted_model_groups == []
+
+
+@pytest.mark.asyncio
+async def test_run_async_fallback_allows_same_model_group_retry_for_file_creation():
+    router = AttemptRecordingRouter()
+
+    await run_async_fallback(
+        litellm_router=router,
+        fallback_model_group=[{"model": "azure-group", "_target_order": 2}],
+        original_model_group="azure-group",
+        original_exception=RuntimeError("first deployment failed"),
+        max_fallbacks=3,
+        fallback_depth=0,
+        model="azure-group",
+        original_function=_acreate_file,
+    )
+
+    assert router.attempted_model_groups == ["azure-group"]
+
+
 @pytest.mark.asyncio
 async def test_run_async_fallback_still_crosses_model_groups_without_an_uploaded_file():
     router = AttemptRecordingRouter()

tests/test_litellm/test_router.py

@@ -492,6 +492,54 @@ async def test_async_router_acreate_file_with_jsonl():
         assert first_call_content == non_jsonl_content
 
 
+@pytest.mark.asyncio
+async def test_async_router_acreate_file_does_not_fall_back_across_model_groups():
+    """A file created for batches only exists under the credentials of the model group
+    the caller named. A cross-group fallback silently stores it with the wrong provider
+    and the later batch create against the named group permanently fails."""
+    from unittest.mock import MagicMock, patch
+
+    router = litellm.Router(
+        model_list=[
+            {
+                "model_name": "azure-gpt",
+                "litellm_params": {
+                    "model": "azure/my-azure-deployment",
+                    "api_base": "http://127.0.0.1:9",
+                    "api_key": "dummy-key",
+                    "api_version": "2024-06-01",
+                },
+            },
+            {
+                "model_name": "openai-gpt",
+                "litellm_params": {"model": "gpt-4o-mini"},
+            },
+        ],
+        fallbacks=[{"azure-gpt": ["openai-gpt"]}],
+    )
+
+    def fail_azure(*args: object, **kwargs: object) -> MagicMock:
+        if kwargs.get("model") == "azure/my-azure-deployment":
+            raise litellm.APIConnectionError(
+                message="Connection error.",
+                llm_provider="azure",
+                model="azure/my-azure-deployment",
+            )
+        return MagicMock()
+
+    with patch("litellm.acreate_file", side_effect=fail_azure) as mock_acreate_file:
+        with pytest.raises(litellm.APIConnectionError):
+            await router.acreate_file(
+                model="azure-gpt",
+                purpose="batch",
+                file=MagicMock(),
+            )
+
+        called_models = [call.kwargs.get("model") for call in mock_acreate_file.call_args_list]
+        assert "azure/my-azure-deployment" in called_models
+        assert "gpt-4o-mini" not in called_models
+
+
 @pytest.mark.asyncio
 async def test_async_router_acreate_file_uses_deployment_custom_llm_provider():
     """