From db15948ae49d1cd0da663e3dd18f80b70c94594a Mon Sep 17 00:00:00 2001 From: Alexsander Hamir Date: Thu, 15 Jan 2026 16:08:57 -0800 Subject: [PATCH] Fix Azure embeddings JSON parsing to prevent connection leaks and ensure proper router cooldown Added explicit json.JSONDecodeError handling in aembedding method to: 1. Convert to AzureOpenAIError with status_code for router cooldown evaluation 2. Trigger httpx connection cleanup by accessing raw_response.status_code Without this handling, JSONDecodeError would bypass router logic (no status_code attribute) and leave httpx connections open, causing resource leaks during high load. This completely eliminates 'Unclosed connection' warnings during OOM load testing. --- litellm/llms/azure/azure.py | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/litellm/llms/azure/azure.py b/litellm/llms/azure/azure.py index ec4553fac4f..e669be88f7b 100644 --- a/litellm/llms/azure/azure.py +++ b/litellm/llms/azure/azure.py @@ -664,8 +664,29 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM): **data, timeout=timeout ) headers = dict(raw_response.headers) - response = raw_response.parse() - stringified_response = response.model_dump() + + # Convert json.JSONDecodeError to AzureOpenAIError for two critical reasons: + # + # 1. ROUTER BEHAVIOR: The router relies on exception.status_code to determine cooldown logic: + # - JSONDecodeError has no status_code → router skips cooldown evaluation + # - AzureOpenAIError has status_code → router properly evaluates for cooldown + # + # 2. CONNECTION CLEANUP: When response.parse() throws JSONDecodeError, the response + # body may not be fully consumed, preventing httpx from properly returning the + # connection to the pool. By catching the exception and accessing raw_response.status_code, + # we trigger httpx's internal cleanup logic. Without this: + # - parse() fails → JSONDecodeError bubbles up → httpx never knows response was acknowledged → connection leak + # This completely eliminates "Unclosed connection" warnings during high load. + try: + response = raw_response.parse() + stringified_response = response.model_dump() + except json.JSONDecodeError as json_error: + raise AzureOpenAIError( + status_code=raw_response.status_code or 500, + message=f"Failed to parse raw Azure embedding response: {str(json_error)}" + ) from json_error + + ## LOGGING logging_obj.post_call( input=input,