litellm/tests/unit/responses/test_text_format_conversion.py

from importlib import import_module
import json

import pytest
from pydantic import BaseModel


import litellm
from litellm.types.llms.openai import (
    IncompleteDetails,
    ResponseAPIUsage,
    ResponsesAPIResponse,
)


class TestTextFormatConversion:
    """Test text_format to text parameter conversion for responses API"""

    def get_base_completion_call_args(self):
        """Get base arguments for completion call"""
        return {
            "model": "gpt-4o",
            "api_key": "test-key",
            "api_base": "https://api.openai.com/v1",
        }

    @pytest.mark.asyncio
    async def test_text_format_to_text_conversion(self):
        """
        Test that when text_format parameter is passed to litellm.aresponses,
        it gets converted to text parameter in the raw API call to OpenAI.
        """
        from unittest.mock import AsyncMock, MagicMock, patch

        class TestResponse(BaseModel):
            """Test Pydantic model for structured output"""

            answer: str
            confidence: float

        # Mock response from OpenAI
        mock_response_data = {
            "id": "resp_123",
            "object": "response",
            "created_at": 1741476542,
            "status": "completed",
            "model": "gpt-4o",
            "output": [
                {
                    "type": "message",
                    "id": "msg_123",
                    "status": "completed",
                    "role": "assistant",
                    "content": [
                        {
                            "type": "output_text",
                            "text": '{"answer": "Paris", "confidence": 0.95}',
                            "annotations": [],
                        }
                    ],
                }
            ],
            "parallel_tool_calls": True,
            "usage": {
                "input_tokens": 10,
                "output_tokens": 20,
                "total_tokens": 30,
                "output_tokens_details": {"reasoning_tokens": 0},
            },
            "text": {"format": {"type": "json_object"}},
            "error": None,
            "incomplete_details": None,
            "instructions": None,
            "metadata": {},
            "temperature": 1.0,
            "tool_choice": "auto",
            "tools": [],
            "top_p": 1.0,
            "max_output_tokens": None,
            "previous_response_id": None,
            "reasoning": {"effort": None, "summary": None},
            "truncation": "disabled",
            "user": None,
        }

        base_completion_call_args = self.get_base_completion_call_args()

        # Mock the response_api_handler function to capture the request
        captured_request = {}

        def mock_handler(
            model,
            input,
            responses_api_provider_config,
            response_api_optional_request_params,
            custom_llm_provider,
            litellm_params,
            logging_obj,
            extra_headers=None,
            extra_body=None,
            timeout=None,
            client=None,
            fake_stream=False,
            litellm_metadata=None,
            shared_session=None,
            _is_async=False,
        ):
            # Capture the request parameters
            captured_request["model"] = model
            captured_request["input"] = input
            captured_request["params"] = response_api_optional_request_params

            # Return a mock ResponsesAPIResponse wrapped in a coroutine if async
            async def async_response():
                return ResponsesAPIResponse(
                    id="resp_123",
                    object="response",
                    created_at=1741476542,
                    status="completed",
                    model="gpt-4o",
                    output=mock_response_data["output"],
                    usage=ResponseAPIUsage(
                        input_tokens=10,
                        output_tokens=20,
                        total_tokens=30,
                    ),
                    text=mock_response_data.get("text"),
                    error=None,
                    incomplete_details=None,
                )

            if _is_async:
                return async_response()
            else:
                return ResponsesAPIResponse(
                    id="resp_123",
                    object="response",
                    created_at=1741476542,
                    status="completed",
                    model="gpt-4o",
                    output=mock_response_data["output"],
                    usage=ResponseAPIUsage(
                        input_tokens=10,
                        output_tokens=20,
                        total_tokens=30,
                    ),
                    text=mock_response_data.get("text"),
                    error=None,
                    incomplete_details=None,
                )

        with patch.object(
            import_module("litellm.responses.main").base_llm_http_handler, "response_api_handler",
            new=mock_handler,
        ):
            litellm.turn_on_debug()

            # Call aresponses with text_format parameter
            response = await litellm.aresponses(
                input="What is the capital of France?",
                text_format=TestResponse,
                **base_completion_call_args,
            )

            # Verify the captured request
            print(
                "Captured request:", json.dumps(captured_request, indent=4, default=str)
            )

            # Validate that text_format was converted to text parameter
            assert (
                "text" in captured_request["params"]
            ), "text parameter should be present in request params"
            assert (
                "text_format" not in captured_request["params"]
            ), "text_format should not be in request params"

            # Validate the text parameter structure
            text_param = captured_request["params"]["text"]
            assert "format" in text_param, "text parameter should have format field"
            assert (
                text_param["format"]["type"] == "json_schema"
            ), "format type should be json_schema"
            assert "name" in text_param["format"], "format should have name field"
            assert (
                text_param["format"]["name"] == "TestResponse"
            ), "format name should match Pydantic model name"
            assert "schema" in text_param["format"], "format should have schema field"
            assert "strict" in text_param["format"], "format should have strict field"

            # Validate the schema structure
            schema = text_param["format"]["schema"]
            assert schema["type"] == "object", "schema type should be object"
            assert "properties" in schema, "schema should have properties"
            assert (
                "answer" in schema["properties"]
            ), "schema should have answer property"
            assert (
                "confidence" in schema["properties"]
            ), "schema should have confidence property"

            # Validate other request parameters
            assert captured_request["input"] == "What is the capital of France?"

            # Validate the response
            print("Response:", json.dumps(response, indent=4, default=str))