diff --git a/tests/e2e/llm_translation/endpoints_client.py b/tests/e2e/llm_translation/endpoints_client.py index 54461405a23..f60e5e589ff 100644 --- a/tests/e2e/llm_translation/endpoints_client.py +++ b/tests/e2e/llm_translation/endpoints_client.py @@ -19,11 +19,30 @@ from e2e_http import StreamingResponse from models import ChatMessage, LiteLLMParamsBody +class FunctionParameterProperty(BaseModel): + type: str + description: str | None = None + + +class FunctionParameters(BaseModel): + type: Literal["object"] = "object" + properties: dict[str, FunctionParameterProperty] + required: list[str] = [] + + +class ResponsesFunctionTool(BaseModel): + type: Literal["function"] = "function" + name: str + description: str | None = None + parameters: FunctionParameters + + class ResponsesRequest(BaseModel): model: str input: str instructions: str | None = None stream: bool = False + tools: list[ResponsesFunctionTool] | None = None class MessagesRequest(BaseModel): @@ -87,6 +106,9 @@ class ResponsesOutputContent(BaseModel): class ResponsesOutputItem(BaseModel): type: str | None = None content: list[ResponsesOutputContent] = [] + name: str | None = None + arguments: str | None = None + call_id: str | None = None class ResponsesResult(BaseModel): @@ -101,6 +123,16 @@ class ResponsesResult(BaseModel): content.text or "" for item in self.output for content in item.content ) + @property + def function_calls(self) -> tuple[ResponsesOutputItem, ...]: + return tuple( + item + for item in self.output + if item.type == "function_call" + and item.name is not None + and item.arguments is not None + ) + class ResponsesStreamEvent(BaseModel): event_id: str | None = None @@ -204,6 +236,20 @@ class EndpointsClient: stream=stream, ) + def responses_with_tools( + self, key: str, model: str, text: str, tools: list[ResponsesFunctionTool] + ) -> StreamingResponse: + return self._send( + "/v1/responses", + key, + ResponsesRequest( + model=model, + input=text, + instructions="You are a helpful assistant", + tools=tools, + ), + ) + def messages( self, key: str, model: str, text: str, *, max_tokens: int = 64 ) -> StreamingResponse: diff --git a/tests/e2e/llm_translation/test_responses_e2e.py b/tests/e2e/llm_translation/test_responses_e2e.py index 91c0759f233..f02721f23b5 100644 --- a/tests/e2e/llm_translation/test_responses_e2e.py +++ b/tests/e2e/llm_translation/test_responses_e2e.py @@ -7,13 +7,19 @@ litellm-regression-tests/tests/test_inference_endpoints.py. from __future__ import annotations +import json +from typing import cast + import pytest -from pydantic import ValidationError +from pydantic import BaseModel, ValidationError from e2e_config import unique_marker from e2e_http import require_successful_call from endpoints_client import ( EndpointsClient, + FunctionParameterProperty, + FunctionParameters, + ResponsesFunctionTool, ResponsesOutputTextDeltaEvent, ResponsesResult, ResponsesStreamEventType, @@ -24,6 +30,10 @@ from models import LiteLLMParamsBody pytestmark = pytest.mark.e2e +class WeatherArguments(BaseModel): + location: str + + class TestResponses: @pytest.mark.covers("llm.responses.openai.basic.nonstream.works") def test_responses_returns_completion( @@ -69,6 +79,71 @@ class TestResponses: == "response.completed" ), "responses stream did not terminate with response.completed" + @pytest.mark.covers("llm.responses.openai.basic.nonstream.cost_logged") + def test_responses_logs_cost( + self, endpoints_client: EndpointsClient, resources: ResourceManager + ) -> None: + model = f"e2e-responses-{unique_marker()}" + model_id = endpoints_client.create_model( + model, + LiteLLMParamsBody(model="openai/gpt-4o-mini", api_key="os.environ/OPENAI_API_KEY"), + ) + resources.defer(lambda: endpoints_client.delete_model(model_id)) + key = resources.key() + + result = endpoints_client.responses(key, model, f"reply with one word {unique_marker()}") + require_successful_call(result) + parsed = ResponsesResult.model_validate_json(result.body) + assert parsed.text.strip(), f"/responses returned no output text: {result.body[:300]}" + assert result.call_id and parsed.id, f"missing response identifiers: {result.body[:300]}" + + rows = endpoints_client.proxy.poll_logs_for_request_id( + parsed.id, + predicate=lambda logged_rows: any((row.spend or 0) > 0 for row in logged_rows), + ) + row = next((logged_row for logged_row in rows if (logged_row.spend or 0) > 0), None) + assert row is not None, f"no costed spend row for response id {parsed.id}" + assert "gpt-4o-mini" in (row.model or ""), f"unexpected spend row model: {row.model}" + + @pytest.mark.covers("llm.responses.openai.tool_use.nonstream.works") + def test_responses_returns_function_call( + self, endpoints_client: EndpointsClient, resources: ResourceManager + ) -> None: + model = f"e2e-responses-{unique_marker()}" + model_id = endpoints_client.create_model( + model, + LiteLLMParamsBody(model="openai/gpt-4o-mini", api_key="os.environ/OPENAI_API_KEY"), + ) + resources.defer(lambda: endpoints_client.delete_model(model_id)) + key = resources.key() + + result = endpoints_client.responses_with_tools( + key, + model, + "What is the weather in San Francisco? Use the get_weather tool.", + [ + ResponsesFunctionTool( + name="get_weather", + description="Get the weather for a location", + parameters=FunctionParameters( + properties={"location": FunctionParameterProperty(type="string")}, + required=["location"], + ), + ) + ], + ) + require_successful_call(result) + parsed = ResponsesResult.model_validate_json(result.body) + function_call = next( + (call for call in parsed.function_calls if call.name == "get_weather"), + None, + ) + assert function_call is not None, f"no get_weather function call: {result.body[:500]}" + assert function_call.arguments is not None + raw_arguments = cast(object, json.loads(function_call.arguments)) + arguments = WeatherArguments.model_validate(raw_arguments) + assert arguments.location, f"function call arguments missing location: {function_call.arguments}" + def _parse_stream_event( event: str,