scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-06 08:16:43 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 78

test_responses_api_shell_tool

Browse source
This commit is contained in:
Ishaan Jaffer 2026-02-12 10:01:12 -08:00
parent b71cc9cc20
commit c8cc0bd8dc
1 changed files with 43 additions and 16 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

59
tests/llm_responses_api_testing/base_responses_api.py
View file

@ -681,9 +681,8 @@ class BaseResponsesAPITest(ABC):
response_id="invalid_response_id_12345", **base_completion_call_args
)
@pytest.mark.parametrize("sync_mode", [True, False])
@pytest.mark.asyncio
async def test_responses_api_context_management_server_side_compaction(self, sync_mode):
async def test_responses_api_context_management_server_side_compaction(self):
"""
E2E test for server-side compaction (context_management) on OpenAI Responses API.
Passes context_management with compact_threshold; validates that the request is
@ -698,22 +697,50 @@ class BaseResponsesAPITest(ABC):
)
context_management = [{"type": "compaction", "compact_threshold": 200000}]
try:
if sync_mode:
response = litellm.responses(
input="Short ping to verify context_management is accepted.",
max_output_tokens=20,
context_management=context_management,
**base_completion_call_args,
)
else:
response = await litellm.aresponses(
input="Short ping to verify context_management is accepted.",
max_output_tokens=20,
context_management=context_management,
**base_completion_call_args,
)
response = await litellm.aresponses(
input="Short ping to verify context_management is accepted.",
max_output_tokens=20,
context_management=context_management,
**base_completion_call_args,
)
except litellm.InternalServerError:
pytest.skip("Skipping test due to litellm.InternalServerError")
validate_responses_api_response(response, final_chunk=True)
assert response.get("id") is not None
assert response.get("status") is not None
@pytest.mark.asyncio
async def test_responses_api_shell_tool(self):
"""
E2E test for Shell tool on OpenAI Responses API.
Passes tools=[{"type": "shell", "environment": {"type": "container_auto"}}];
validates that the request is accepted and returns a valid response.
Only runs for OpenAI/Azure (Responses API with shell support).
"""
base_completion_call_args = self.get_base_completion_call_args()
model = base_completion_call_args.get("model") or ""
if "openai/" not in str(model) and "azure/" not in str(model):
pytest.skip(
"Shell tool e2e is only run for OpenAI/Azure Responses API"
)
tools = [{"type": "shell", "environment": {"type": "container_auto"}}]
input_msg = "List files in /mnt/data and show python --version."
try:
response = await litellm.aresponses(
input=input_msg,
max_output_tokens=256,
tools=tools,
tool_choice="auto",
**base_completion_call_args,
)
except litellm.InternalServerError:
pytest.skip("Skipping test due to litellm.InternalServerError")
except litellm.BadRequestError as e:
if "shell" in str(e).lower() and "not supported" in str(e).lower():
pytest.skip(
"Shell tool is not supported for this model (e.g. gpt-4o); use a model that supports shell"
)
raise
validate_responses_api_response(response, final_chunk=True)
assert response.get("id") is not None
assert response.get("status") is not None