scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-08-28 05:25:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 36

test(responses): replace perma-skip azure shell e2e with offline coverage (#32444)

Browse source
This commit is contained in:
Mateo Wang 2026-07-08 10:01:41 -07:00 • committed by GitHub
parent bfff5e8d86
commit c2d8a17692
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
4 changed files with 142 additions and 36 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
tests/llm_responses_api_testing/base_responses_api.py
View file

@ -746,7 +746,8 @@ class BaseResponsesAPITest(ABC):
E2E test for Shell tool on OpenAI Responses API.
Passes tools=[{"type": "shell", "environment": {"type": "container_auto"}}];
validates that the request is accepted and returns a valid response.
Only runs for OpenAI/Azure (Responses API with shell support).
Only runs for OpenAI; offline coverage for the Azure route lives in
tests/test_litellm/responses/test_responses_api_request_body.py.
"""
base_completion_call_args = self.get_base_completion_call_args()
model = (
@ -754,8 +755,10 @@ class BaseResponsesAPITest(ABC):
or base_completion_call_args.get("model")
or ""
)
if "openai/" not in str(model) and "azure/" not in str(model):
pytest.skip("Shell tool e2e is only run for OpenAI/Azure Responses API")
if "openai/" not in str(model):
pytest.skip(
"Shell tool e2e is OpenAI-only; no Azure deployment supports the shell tool yet, re-enable once one exists"
)
tools = [{"type": "shell", "environment": {"type": "container_auto"}}]
input_msg = "List files in /mnt/data and show python --version."
try:

5
tests/llm_responses_api_testing/test_azure_responses_api.py
View file

@ -2,7 +2,6 @@ import os
import sys
import pytest
import asyncio
from typing import Optional
from unittest.mock import patch, AsyncMock
sys.path.insert(0, os.path.abspath("../.."))
@ -30,10 +29,6 @@ class TestAzureResponsesAPITest(BaseResponsesAPITest):
"api_version": "2025-03-01-preview",
}
def get_advanced_model_for_shell_tool(self) -> Optional[str]:
"""If specified, overrides the model used by test_responses_api_shell_tool_streaming_sees_shell_output (e.g. openai/gpt-5.2 for shell support)."""
return "azure/gpt-5-mini"
@pytest.mark.asyncio
async def test_azure_responses_api_preview_api_version():

14
tests/test_litellm/expected_responses_api_request/azure_shell_tool.json Normal file
View file

@ -0,0 +1,14 @@
{
"model": "gpt-5-mini",
"input": "List files in /mnt/data and run python --version.",
"tools": [
{
"type": "shell",
"environment": {
"type": "container_auto"
}
}
],
"tool_choice": "auto",
"max_output_tokens": 256
}

150
tests/test_litellm/responses/test_responses_api_request_body.py
View file

@ -1,6 +1,7 @@
"""
Test that litellm.responses() / litellm.aresponses() send the expected request body
over the wire. Expected JSON bodies are stored in expected_responses_api_request/.
over the wire and surface provider errors correctly. Expected JSON bodies are stored
in expected_responses_api_request/.
"""
import json
@ -18,24 +19,20 @@ def _expected_dir() -> Path:
return Path(__file__).resolve().parent.parent / "expected_responses_api_request"
@pytest.mark.asyncio
async def test_aresponses_context_management_and_shell_request_body_matches_expected():
"""
Call litellm.aresponses() with context_management and shell tool;
assert the httpx POST request body matches the expected JSON.
"""
expected_path = _expected_dir() / "context_management_and_shell.json"
def _load_expected_body(filename: str) -> dict:
expected_path = _expected_dir() / filename
assert expected_path.exists(), f"Expected file not found: {expected_path}"
with open(expected_path) as f:
expected_body = json.load(f)
return json.load(f)
# Minimal Responses API response so parsing succeeds
mock_response = {
"id": "resp_ctx_shell_test",
def _minimal_responses_api_payload(response_id: str, model: str) -> dict:
return {
"id": response_id,
"object": "response",
"created_at": 1734366691,
"status": "completed",
"model": "gpt-4o",
"model": model,
"output": [
{
"type": "message",
@ -69,21 +66,41 @@ async def test_aresponses_context_management_and_shell_request_body_matches_expe
"user": None,
}
class MockResponse:
def __init__(self, json_data, status_code=200):
self._json_data = json_data
self.status_code = status_code
self.text = json.dumps(json_data)
self.headers = httpx.Headers({})
def json(self):
return self._json_data
class MockResponse:
def __init__(self, json_data, status_code=200):
self._json_data = json_data
self.status_code = status_code
self.text = json.dumps(json_data)
self.headers = httpx.Headers({})
def json(self):
return self._json_data
def _assert_request_body_matches(request_body: dict, expected_body: dict) -> None:
for key, expected_value in expected_body.items():
assert key in request_body, f"Missing key in request body: {key}"
assert (
request_body[key] == expected_value
), f"Mismatch for key {key}: got {request_body[key]!r}, expected {expected_value!r}"
@pytest.mark.asyncio
async def test_aresponses_context_management_and_shell_request_body_matches_expected():
"""
Call litellm.aresponses() with context_management and shell tool;
assert the httpx POST request body matches the expected JSON.
"""
expected_body = _load_expected_body("context_management_and_shell.json")
with patch(
"litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post",
new_callable=AsyncMock,
) as mock_post:
mock_post.return_value = MockResponse(mock_response, 200)
mock_post.return_value = MockResponse(
_minimal_responses_api_payload("resp_ctx_shell_test", "gpt-4o"), 200
)
await litellm.aresponses(
model="openai/gpt-4o",
@ -95,10 +112,87 @@ async def test_aresponses_context_management_and_shell_request_body_matches_expe
)
mock_post.assert_called_once()
request_body = mock_post.call_args.kwargs["json"]
_assert_request_body_matches(mock_post.call_args.kwargs["json"], expected_body)
for key, expected_value in expected_body.items():
assert key in request_body, f"Missing key in request body: {key}"
assert (
request_body[key] == expected_value
), f"Mismatch for key {key}: got {request_body[key]!r}, expected {expected_value!r}"
@pytest.mark.asyncio
async def test_aresponses_azure_shell_tool_request_body_matches_expected():
"""
Call litellm.aresponses() on the Azure route with the shell tool;
assert the httpx POST request body carries the shell tool verbatim.
"""
expected_body = _load_expected_body("azure_shell_tool.json")
with patch(
"litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post",
new_callable=AsyncMock,
) as mock_post:
mock_post.return_value = MockResponse(
_minimal_responses_api_payload("resp_azure_shell_test", "gpt-5-mini"), 200
)
await litellm.aresponses(
model="azure/gpt-5-mini",
api_base="https://fake-resource.openai.azure.com",
api_key="fake-api-key",
api_version="2025-03-01-preview",
input=expected_body["input"],
tools=expected_body["tools"],
tool_choice=expected_body["tool_choice"],
max_output_tokens=expected_body["max_output_tokens"],
)
mock_post.assert_called_once()
_assert_request_body_matches(mock_post.call_args.kwargs["json"], expected_body)
@pytest.mark.asyncio
async def test_aresponses_azure_shell_tool_400_maps_to_bad_request_error():
"""
Azure rejects the shell tool for unsupported deployments with a 400;
litellm must surface that as litellm.BadRequestError carrying the provider message.
"""
error_body = {
"error": {
"message": "Tool of type 'shell' is not supported with this model.",
"type": "invalid_request_error",
"param": "tools",
"code": None,
}
}
def _raise_azure_400(*args, **kwargs):
response = httpx.Response(
status_code=400,
json=error_body,
request=httpx.Request(
"POST",
kwargs.get(
"url",
"https://fake-resource.openai.azure.com/openai/responses",
),
),
)
response.raise_for_status()
with patch(
"litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post",
new_callable=AsyncMock,
) as mock_post:
mock_post.side_effect = _raise_azure_400
with pytest.raises(litellm.BadRequestError) as excinfo:
await litellm.aresponses(
model="azure/gpt-5-mini",
api_base="https://fake-resource.openai.azure.com",
api_key="fake-api-key",
api_version="2025-03-01-preview",
input="List files in /mnt/data and run python --version.",
tools=[{"type": "shell", "environment": {"type": "container_auto"}}],
tool_choice="auto",
max_output_tokens=256,
)
assert excinfo.value.status_code == 400
assert "shell" in str(excinfo.value).lower()
assert "not supported" in str(excinfo.value).lower()