diff --git a/litellm/constants.py b/litellm/constants.py index cc6db6c10cc..0f1fa2ee3a7 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -806,6 +806,7 @@ openai_compatible_endpoints: Final[list] = [ "https://api.meta.ai/v1", "https://api.cognition.ai/v1", "https://api.scx.ai/v1", + "https://gigachat.devices.sberbank.ru/api/v1", ] diff --git a/litellm/litellm_core_utils/get_llm_provider_logic.py b/litellm/litellm_core_utils/get_llm_provider_logic.py index b35ef659d67..9b53b79bbe6 100644 --- a/litellm/litellm_core_utils/get_llm_provider_logic.py +++ b/litellm/litellm_core_utils/get_llm_provider_logic.py @@ -496,8 +496,6 @@ def get_llm_provider( custom_llm_provider = "amazon_nova" elif model.startswith("sap/"): custom_llm_provider = "sap" - elif model in litellm.gigachat_models or model.startswith("gigachat/"): - custom_llm_provider = "gigachat" # Last resort for an otherwise-unknown model: a declarative # fallback-generalization routing rule (e.g. routes future claude-* to anthropic). diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index e34c647efc2..350e5403e4c 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -2141,7 +2141,7 @@ class Logging(LiteLLMLoggingBaseClass): logging_result: Final = self.normalize_logging_result(result=result) - if isinstance(result, Response) and isinstance(logging_result, ModelResponse): + if isinstance(result, Response) and isinstance(logging_result, (ModelResponse, EmbeddingResponse)): result = logging_result if standard_logging_object is None and result is not None and self.stream is not True: diff --git a/litellm/passthrough/main.py b/litellm/passthrough/main.py index 2780f510a76..9095cee15a9 100644 --- a/litellm/passthrough/main.py +++ b/litellm/passthrough/main.py @@ -54,6 +54,7 @@ class AsyncPassthroughStreamingResponse(AsyncGenerator[Any, Any]): self._raw_bytes: list[bytes] = [] # mutable-ok: instance buffer for streaming chunks self._flush_scheduled = False self._background_tasks: set[asyncio.Task] = set() # mutable-ok: instance set for background task tracking + self._hidden_params: dict[str, object] = {} # mutable-ok: router attaches response headers here in place @property def status_code(self) -> int: @@ -127,6 +128,9 @@ class AsyncPassthroughStreamingResponse(AsyncGenerator[Any, Any]): def __aiter__(self) -> AsyncPassthroughStreamingResponse: return self + def aiter_bytes(self) -> AsyncPassthroughStreamingResponse: + return self + async def __anext__(self) -> bytes: if not self._initialized: await self # pyright: ignore[reportGeneralTypeIssues] # structural type check misses __await__ @@ -556,7 +560,18 @@ def llm_passthrough_route( else: # Sync path - client.client.send returns Response directly response: httpx.Response = client.client.send(request=request, stream=is_streaming_request) - response.raise_for_status() + try: + response.raise_for_status() + except Exception: # noqa: BLE001 # Safe catch-all for cleanup logic + try: + response.read() + except Exception: # noqa: BLE001 S110 # Safe catch-all for cleanup logic + pass + try: + response.close() + except Exception: # noqa: BLE001 S110 # Safe catch-all for cleanup logic + pass + raise if hasattr(response, "iter_bytes") and is_streaming_request: return PassthroughStreamingResponse(response, litellm_logging_obj, provider_config) diff --git a/tests/test_litellm/litellm_core_utils/test_get_llm_provider_endpoint_match.py b/tests/test_litellm/litellm_core_utils/test_get_llm_provider_endpoint_match.py index 6cacd119030..419ca104bb1 100644 --- a/tests/test_litellm/litellm_core_utils/test_get_llm_provider_endpoint_match.py +++ b/tests/test_litellm/litellm_core_utils/test_get_llm_provider_endpoint_match.py @@ -184,3 +184,26 @@ class TestTogetherApiBaseResolvesProvider: assert provider == "together_ai" assert api_base == "https://api.together.ai/v1" + + +class TestGigachatApiBaseResolvesProvider: + """ + Regression for the GigaChat api_base branch: the provider-mapping chain + carried an ``endpoint == "https://gigachat.devices.sberbank.ru/api/v1"`` + elif, but the URL was never added to ``openai_compatible_endpoints``, so + the endpoint loop never fired the branch and a caller-supplied GigaChat + api_base raised BadRequestError instead of resolving to ``gigachat``. + """ + + def test_gigachat_api_base_resolves_to_gigachat(self, monkeypatch): + monkeypatch.setenv("GIGACHAT_API_KEY", "gigachat-key-from-env") + + model, provider, dynamic_api_key, returned_api_base = get_llm_provider( + model="GigaChat-2", + api_base="https://gigachat.devices.sberbank.ru/api/v1", + ) + + assert provider == "gigachat" + assert dynamic_api_key == "gigachat-key-from-env" + assert returned_api_base == "https://gigachat.devices.sberbank.ru/api/v1" + assert model == "GigaChat-2" diff --git a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py index 947a55410ef..c7328adb0b3 100644 --- a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py +++ b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py @@ -6101,3 +6101,64 @@ def test_response_timing_metrics_survive_deepcopy(logging_obj): logging_obj.set_response_timing_metrics({"_response_ms": 12.5}) assert copy.deepcopy(logging_obj).response_timing_metrics == {"_response_ms": 12.5} + + +def test_passthrough_embeddings_result_swapped_for_callbacks(): + """ + Regression: for gigachat passthrough /embeddings, normalize_logging_result + produces an EmbeddingResponse, but the result swap only accepted + ModelResponse, so callbacks kept receiving the raw httpx.Response (which + crashes attribute readers like OTEL). The swap must cover + EmbeddingResponse too. + """ + import datetime as dt + + from litellm.types.utils import EmbeddingResponse + + logging_obj = LitellmLogging( + model="EmbeddingsGigaR", + messages=[], + stream=False, + call_type="allm_passthrough_route", + start_time=time.time(), + litellm_call_id="passthrough-embed-call-id", + function_id="passthrough-embed-fn-id", + ) + logging_obj.update_environment_variables( + litellm_params={}, + optional_params={}, + model="EmbeddingsGigaR", + custom_llm_provider="gigachat", + endpoint="/embeddings", + request_data={"model": "EmbeddingsGigaR", "input": ["hello"]}, + input=["hello"], + ) + + httpx_response = httpx.Response( + 200, + json={ + "object": "list", + "data": [ + { + "object": "embedding", + "embedding": [0.1, 0.2, 0.3], + "index": 0, + "usage": {"prompt_tokens": 5}, + } + ], + "model": "EmbeddingsGigaR", + }, + request=httpx.Request( + "POST", "https://gigachat.devices.sberbank.ru/api/v1/embeddings" + ), + ) + + _, _, swapped_result = logging_obj._success_handler_helper_fn( + result=httpx_response, + start_time=dt.datetime.now(), + end_time=dt.datetime.now(), + cache_hit=False, + ) + + assert isinstance(swapped_result, EmbeddingResponse) + assert swapped_result.data[0]["embedding"] == [0.1, 0.2, 0.3] diff --git a/tests/test_litellm/passthrough/test_passthrough_main.py b/tests/test_litellm/passthrough/test_passthrough_main.py index fb67aa6f3e4..1950c37a12e 100644 --- a/tests/test_litellm/passthrough/test_passthrough_main.py +++ b/tests/test_litellm/passthrough/test_passthrough_main.py @@ -719,6 +719,87 @@ async def test_allm_passthrough_route_429_streaming_raises(): assert exc_info.value.response.status_code == 429 +def test_llm_passthrough_route_sync_streaming_error_maps_upstream_status(): + """ + Regression test: a sync streaming passthrough whose upstream answers an + error status must surface the mapped provider error, not + httpx.ResponseNotRead. + + Before the fix, raise_for_status() raised on the still-unread streamed + response, and _handle_error then touched e.response.text, which raises + ResponseNotRead on a streamed-but-unread body, masking the real upstream + error entirely. + """ + from litellm.llms.base_llm.chat.transformation import BaseLLMException + + error_body = json.dumps( + { + "error": { + "code": "429", + "message": "Rate limit exceeded. Retry after 10 seconds.", + } + } + ).encode() + + class _UnreadErrorStream(httpx.SyncByteStream): + def __iter__(self): + yield error_body + + def _handler(request: httpx.Request) -> httpx.Response: + return httpx.Response( + 429, + stream=_UnreadErrorStream(), + headers={"content-type": "application/json"}, + ) + + sync_client = HTTPHandler( + client=httpx.Client(transport=httpx.MockTransport(_handler)) + ) + + mock_provider_config = MagicMock() + mock_provider_config.get_complete_url.return_value = ( + httpx.URL("https://gigachat.devices.sberbank.ru/api/v1/chat/completions"), + "https://gigachat.devices.sberbank.ru/api/v1", + ) + mock_provider_config.get_api_key.return_value = "fake-key" + mock_provider_config.validate_environment.return_value = { + "Authorization": "Bearer fake-key" + } + mock_provider_config.sign_request.return_value = ( + {"Authorization": "Bearer fake-key"}, + None, + ) + mock_provider_config.is_streaming_request.return_value = True + mock_provider_config.get_error_class.side_effect = ( + lambda error_message, status_code, headers: BaseLLMException( + status_code=status_code, message=error_message, headers=headers + ) + ) + + mock_logging_obj = MagicMock() + + with pytest.raises(BaseLLMException) as exc_info: + llm_passthrough_route( + model="gigachat/GigaChat-2", + endpoint="chat/completions", + method="POST", + custom_llm_provider="gigachat", + api_base="https://gigachat.devices.sberbank.ru/api/v1", + api_key="fake-key", + json={ + "model": "GigaChat-2", + "messages": [{"role": "user", "content": "hi"}], + "stream": True, + }, + client=sync_client, + litellm_logging_obj=mock_logging_obj, + provider_config=mock_provider_config, + ) + + assert exc_info.value.status_code == 429 + assert "Rate limit exceeded" in str(exc_info.value) + + def test_llm_passthrough_route_propagates_allm_passthrough_route_to_logging_obj(): """ Regression guard for LIT-4192: `allm_passthrough_route` sets diff --git a/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py b/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py index 225e1b3d998..303c71a1630 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py @@ -4865,3 +4865,76 @@ class TestPassthroughRouterModelBudgetReservation: ) self._assert_metadata_carries_attribution(captured, user_api_key_dict) + + +class TestAzureRouterModelStreamingDispatch: + """ + Regression: ``llm_router.allm_passthrough_route`` returns an awaited + ``AsyncPassthroughStreamingResponse`` for streaming calls, which is no + longer an async generator under ``inspect.isasyncgen``. The dispatch's + else branch therefore calls ``.aiter_bytes()`` / ``.status_code`` / + ``.headers`` on it. The router's ``set_response_headers`` also runs the + result through ``prepare_response_for_header_attachment``, which used to + wrap it in ``HiddenParamsAsyncIteratorWrapper`` (no ``aiter_bytes``), so + every streaming Azure router-model request 500'd with + ``AttributeError: aiter_bytes``; ``_hidden_params`` on the streaming + response keeps it unwrapped. + """ + + @pytest.mark.asyncio + async def test_azure_router_model_streaming_returns_streaming_response(self, monkeypatch): + import litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints as ep + import litellm.proxy.proxy_server as proxy_server + from litellm.passthrough.main import AsyncPassthroughStreamingResponse + + upstream_body = b"data: hello\n\n" + + async def _upstream_response() -> httpx.Response: + upstream_request = httpx.Request( + "POST", + "https://my-azure.openai.azure.com/openai/deployments/gpt-5/chat/completions", + ) + return httpx.Response( + 200, + headers={"content-type": "text/event-stream"}, + content=upstream_body, + request=upstream_request, + ) + + logging_obj = MagicMock() + logging_obj.async_flush_passthrough_collected_chunks = AsyncMock() + + from litellm.router_utils.add_retry_fallback_headers import prepare_response_for_header_attachment + + class StreamingRouter: + async def allm_passthrough_route(self, **kwargs): + streaming_response = await AsyncPassthroughStreamingResponse( + response=_upstream_response(), + litellm_logging_obj=logging_obj, + provider_config=MagicMock(), + ) + return prepare_response_for_header_attachment(streaming_response) + + async def fake_get_request_body(_request): + return {"model": "gpt-5", "stream": True} + + monkeypatch.setattr(proxy_server, "llm_router", StreamingRouter()) + monkeypatch.setattr(ep, "get_request_body", fake_get_request_body) + monkeypatch.setattr(ep, "is_passthrough_request_using_router_model", lambda *a, **k: True) + + request = MagicMock(spec=Request) + request.method = "POST" + request.headers = {"content-type": "application/json"} + request.query_params = {} + + result = await azure_proxy_route( + endpoint="openai/deployments/gpt-5/chat/completions", + request=request, + fastapi_response=MagicMock(spec=Response), + user_api_key_dict=UserAPIKeyAuth(api_key="hashed-token"), + ) + + assert isinstance(result, StreamingResponse) + assert result.status_code == 200 + body = b"".join([chunk async for chunk in result.body_iterator]) + assert body == upstream_body