mirror of
https://github.com/BerriAI/litellm.git
synced 2026-08-28 05:25:59 +00:00
test(azure-ai): pin the 422 retry that drops the field the provider rejected
Azure AI is the only provider that retries a 422 inside the translation layer: when the endpoint rejects a field, litellm drops that field and sends the request again, up to twice. That is the difference between a customer's tool call working and coming back as a hard 400, and none of it was covered. The retry loop in llm_http_handler.py is 13,419 lines of source against a 0.20 test-to-source ratio, and nothing exercised this path at all. Drives real litellm.completion and litellm.acompletion calls against a recorded Azure AI endpoint, so the assertions read the bytes that actually went over the wire rather than a mock's call list. Nothing internal is patched: respx fakes the HTTP boundary and the provider config, retry loop and serialization are all the real ones. Pins: - a tool field the endpoint rejects is dropped and the call retried, and the caller gets a normal completion - the retry changes only the field the provider named - a provider that keeps rejecting stops after exactly two attempts - a rejection the provider cannot fix is not retried at all - an extra input outside a tool is retried only when drop_params was asked for Mutating the source confirms these bite: raising the retry cap from 2 to 3, and making the tool-level field check always return False, each turn the suite red. The async cases pin the transport to httpx, because the aiohttp default carries its own transport that an httpx-level fake cannot intercept. Without that the two async tests reached the real Azure endpoint and failed on a 401.
This commit is contained in:
parent
e52f05566d
commit
ea7a5d6709
1 changed files with 172 additions and 0 deletions
|
|
@ -2757,3 +2757,175 @@ def test_video_generation_with_input_reference_keeps_file_multipart():
|
|||
"seconds": "4",
|
||||
}
|
||||
assert result.status == "queued"
|
||||
|
||||
|
||||
AZURE_AI_HOST = "myfoundry.services.ai.azure.com"
|
||||
AZURE_AI_BASE = f"https://{AZURE_AI_HOST}"
|
||||
|
||||
TOOL_WITH_AN_UNSUPPORTED_FIELD = {
|
||||
"type": "function",
|
||||
"function": {"name": "lookup", "parameters": {"type": "object", "properties": {}}},
|
||||
"strict": True,
|
||||
}
|
||||
|
||||
A_COMPLETION = {
|
||||
"id": "chatcmpl-1",
|
||||
"object": "chat.completion",
|
||||
"created": 1,
|
||||
"model": "grok-3",
|
||||
"choices": [
|
||||
{"index": 0, "message": {"role": "assistant", "content": "sent"}, "finish_reason": "stop"}
|
||||
],
|
||||
"usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2},
|
||||
}
|
||||
|
||||
TOOL_LEVEL_REJECTION = "Extra inputs are not permitted: tools[0].strict"
|
||||
UNRELATED_REJECTION = "Extra inputs are not permitted: temperature"
|
||||
A_REJECTION_THE_PROVIDER_CANNOT_FIX = "The model is not available in this region"
|
||||
|
||||
|
||||
class _RecordedAzureAI:
|
||||
def __init__(self, responses: list[httpx.Response]) -> None:
|
||||
self._responses = responses
|
||||
self.bodies: list[dict] = []
|
||||
|
||||
def __call__(self, request: httpx.Request) -> httpx.Response:
|
||||
self.bodies.append(json.loads(request.content))
|
||||
return self._responses[min(len(self.bodies) - 1, len(self._responses) - 1)]
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def httpx_transport(monkeypatch):
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
|
||||
|
||||
def _rejection(message: str) -> httpx.Response:
|
||||
return httpx.Response(422, json={"error": {"message": message}})
|
||||
|
||||
|
||||
def _call_azure_ai(recorder: _RecordedAzureAI, **overrides):
|
||||
import respx
|
||||
|
||||
with respx.mock:
|
||||
respx.route(host=AZURE_AI_HOST).mock(side_effect=recorder)
|
||||
return litellm.completion(
|
||||
model="azure_ai/grok-3",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
tools=[TOOL_WITH_AN_UNSUPPORTED_FIELD],
|
||||
api_base=AZURE_AI_BASE,
|
||||
api_key="fake-key",
|
||||
**overrides,
|
||||
)
|
||||
|
||||
|
||||
def test_a_tool_field_the_provider_rejects_is_dropped_and_the_call_retried():
|
||||
recorder = _RecordedAzureAI(
|
||||
[_rejection(TOOL_LEVEL_REJECTION), httpx.Response(200, json=A_COMPLETION)]
|
||||
)
|
||||
|
||||
response = _call_azure_ai(recorder)
|
||||
|
||||
assert len(recorder.bodies) == 2
|
||||
assert recorder.bodies[0]["tools"][0]["strict"] is True
|
||||
assert "strict" not in recorder.bodies[1]["tools"][0]
|
||||
assert response.choices[0].message.content == "sent"
|
||||
|
||||
|
||||
def test_the_retry_changes_only_the_field_the_provider_named():
|
||||
recorder = _RecordedAzureAI(
|
||||
[_rejection(TOOL_LEVEL_REJECTION), httpx.Response(200, json=A_COMPLETION)]
|
||||
)
|
||||
|
||||
_call_azure_ai(recorder)
|
||||
|
||||
first, second = recorder.bodies
|
||||
assert second["messages"] == first["messages"]
|
||||
assert second["model"] == first["model"]
|
||||
assert second["tools"][0]["function"] == first["tools"][0]["function"]
|
||||
|
||||
|
||||
def test_a_provider_that_keeps_rejecting_is_not_retried_forever():
|
||||
recorder = _RecordedAzureAI([_rejection(TOOL_LEVEL_REJECTION)])
|
||||
|
||||
with pytest.raises(litellm.BadRequestError) as raised:
|
||||
_call_azure_ai(recorder)
|
||||
|
||||
assert len(recorder.bodies) == 2
|
||||
assert raised.value.status_code == 422
|
||||
|
||||
|
||||
def test_a_rejection_the_provider_cannot_fix_is_not_retried_at_all():
|
||||
recorder = _RecordedAzureAI([_rejection(A_REJECTION_THE_PROVIDER_CANNOT_FIX)])
|
||||
|
||||
with pytest.raises(litellm.BadRequestError):
|
||||
_call_azure_ai(recorder)
|
||||
|
||||
assert len(recorder.bodies) == 1
|
||||
|
||||
|
||||
def test_an_extra_input_outside_a_tool_is_not_retried_unless_dropping_params_was_asked_for():
|
||||
recorder = _RecordedAzureAI([_rejection(UNRELATED_REJECTION)])
|
||||
|
||||
with pytest.raises(litellm.BadRequestError):
|
||||
_call_azure_ai(recorder)
|
||||
|
||||
assert len(recorder.bodies) == 1
|
||||
|
||||
|
||||
def test_an_extra_input_outside_a_tool_is_retried_when_dropping_params_was_asked_for():
|
||||
recorder = _RecordedAzureAI(
|
||||
[_rejection(UNRELATED_REJECTION), httpx.Response(200, json=A_COMPLETION)]
|
||||
)
|
||||
|
||||
response = _call_azure_ai(recorder, drop_params=True)
|
||||
|
||||
assert len(recorder.bodies) == 2
|
||||
assert response.choices[0].message.content == "sent"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_tool_field_the_provider_rejects_is_dropped_and_retried_on_the_async_path(
|
||||
httpx_transport,
|
||||
):
|
||||
import respx
|
||||
|
||||
recorder = _RecordedAzureAI(
|
||||
[_rejection(TOOL_LEVEL_REJECTION), httpx.Response(200, json=A_COMPLETION)]
|
||||
)
|
||||
|
||||
with respx.mock:
|
||||
respx.route(host=AZURE_AI_HOST).mock(side_effect=recorder)
|
||||
response = await litellm.acompletion(
|
||||
model="azure_ai/grok-3",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
tools=[TOOL_WITH_AN_UNSUPPORTED_FIELD],
|
||||
api_base=AZURE_AI_BASE,
|
||||
api_key="fake-key",
|
||||
)
|
||||
|
||||
assert len(recorder.bodies) == 2
|
||||
assert recorder.bodies[0]["tools"][0]["strict"] is True
|
||||
assert "strict" not in recorder.bodies[1]["tools"][0]
|
||||
assert response.choices[0].message.content == "sent"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_provider_that_keeps_rejecting_is_not_retried_forever_on_the_async_path(
|
||||
httpx_transport,
|
||||
):
|
||||
import respx
|
||||
|
||||
recorder = _RecordedAzureAI([_rejection(TOOL_LEVEL_REJECTION)])
|
||||
|
||||
with respx.mock:
|
||||
respx.route(host=AZURE_AI_HOST).mock(side_effect=recorder)
|
||||
with pytest.raises(litellm.BadRequestError):
|
||||
await litellm.acompletion(
|
||||
model="azure_ai/grok-3",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
tools=[TOOL_WITH_AN_UNSUPPORTED_FIELD],
|
||||
api_base=AZURE_AI_BASE,
|
||||
api_key="fake-key",
|
||||
)
|
||||
|
||||
assert len(recorder.bodies) == 2
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue