diff --git a/tests/test_litellm/llms/openai/evals/test_openai_evals_transformation.py b/tests/test_litellm/llms/openai/evals/test_openai_evals_transformation.py index f39be511b97..9a30ca0ee60 100644 --- a/tests/test_litellm/llms/openai/evals/test_openai_evals_transformation.py +++ b/tests/test_litellm/llms/openai/evals/test_openai_evals_transformation.py @@ -252,9 +252,7 @@ def test_transform_cancel_eval_response(config: OpenAIEvalsConfig): "object": "eval", "status": "cancelled", }, - request=httpx.Request( - "POST", "https://api.openai.com/v1/evals/eval_123/cancel" - ), + request=httpx.Request("POST", "https://api.openai.com/v1/evals/eval_123/cancel"), ) result = config.transform_cancel_eval_response( @@ -276,8 +274,169 @@ def test_transform_run_requests_encode_eval_and_run_ids(config: OpenAIEvalsConfi headers={}, ) - assert ( - url - == "https://api.openai.com/v1/evals/..%2F..%2Fevals%3Fx%3D1%23frag/runs/..%2Fruns%23other/cancel" - ) + assert url == "https://api.openai.com/v1/evals/..%2F..%2Fevals%3Fx%3D1%23frag/runs/..%2Fruns%23other/cancel" assert request_body == {} + + +def _eval_json_response(url: str, method: str = "GET") -> httpx.Response: + return httpx.Response( + status_code=200, + json={ + "id": "eval_123", + "object": "eval", + "created_at": 1234567890, + "name": "Test Eval", + "data_source_config": {"type": "stored_completions"}, + "testing_criteria": [], + }, + request=httpx.Request(method, url), + ) + + +def _run_json(run_id: str = "evalrun_123", status: str = "queued") -> dict: + return { + "id": run_id, + "object": "eval.run", + "created_at": 1234567890, + "status": status, + "data_source": {"type": "completions"}, + "eval_id": "eval_123", + } + + +def test_transform_get_eval_response(config: OpenAIEvalsConfig): + result = config.transform_get_eval_response( + raw_response=_eval_json_response("https://api.openai.com/v1/evals/eval_123"), + logging_obj=None, + ) + + assert result.id == "eval_123" + assert result.object == "eval" + assert result.name == "Test Eval" + + +def test_transform_update_eval_response(config: OpenAIEvalsConfig): + result = config.transform_update_eval_response( + raw_response=_eval_json_response("https://api.openai.com/v1/evals/eval_123", method="POST"), + logging_obj=None, + ) + + assert result.id == "eval_123" + assert result.name == "Test Eval" + + +def test_transform_create_run_response(config: OpenAIEvalsConfig): + response = httpx.Response( + status_code=200, + json=_run_json(), + request=httpx.Request("POST", "https://api.openai.com/v1/evals/eval_123/runs"), + ) + + result = config.transform_create_run_response( + raw_response=response, + logging_obj=None, + ) + + assert result.id == "evalrun_123" + assert result.status == "queued" + assert result.eval_id == "eval_123" + + +def test_transform_list_runs_request(config: OpenAIEvalsConfig): + url, query_params = config.transform_list_runs_request( + eval_id="eval_123", + list_params={"limit": 5, "after": "evalrun_1", "order": "asc"}, + litellm_params=GenericLiteLLMParams(api_base="https://api.openai.com"), + headers={}, + ) + + assert url == "https://api.openai.com/v1/evals/eval_123/runs" + assert query_params == {"limit": 5, "after": "evalrun_1", "order": "asc"} + + +def test_transform_list_runs_response(config: OpenAIEvalsConfig): + response = httpx.Response( + status_code=200, + json={ + "object": "list", + "data": [_run_json()], + "first_id": "evalrun_123", + "last_id": "evalrun_123", + "has_more": False, + }, + request=httpx.Request("GET", "https://api.openai.com/v1/evals/eval_123/runs"), + ) + + result = config.transform_list_runs_response( + raw_response=response, + logging_obj=None, + ) + + assert result.object == "list" + assert len(result.data) == 1 + assert result.data[0].id == "evalrun_123" + assert result.has_more is False + + +def test_transform_get_run_response(config: OpenAIEvalsConfig): + response = httpx.Response( + status_code=200, + json=_run_json(status="completed"), + request=httpx.Request("GET", "https://api.openai.com/v1/evals/eval_123/runs/evalrun_123"), + ) + + result = config.transform_get_run_response( + raw_response=response, + logging_obj=None, + ) + + assert result.id == "evalrun_123" + assert result.status == "completed" + + +def test_transform_cancel_run_response(config: OpenAIEvalsConfig): + response = httpx.Response( + status_code=200, + json={"id": "evalrun_123", "object": "eval.run", "status": "cancelled"}, + request=httpx.Request( + "POST", + "https://api.openai.com/v1/evals/eval_123/runs/evalrun_123/cancel", + ), + ) + + result = config.transform_cancel_run_response( + raw_response=response, + logging_obj=None, + ) + + assert result.id == "evalrun_123" + assert result.status == "cancelled" + + +def test_transform_delete_run_request(config: OpenAIEvalsConfig): + url, headers, request_body = config.transform_delete_run_request( + eval_id="eval_123", + run_id="evalrun_123", + api_base="https://api.openai.com", + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + + assert url == "https://api.openai.com/v1/evals/eval_123/runs/evalrun_123" + assert request_body == {} + + +def test_transform_delete_run_response(config: OpenAIEvalsConfig): + response = httpx.Response( + status_code=200, + json={"run_id": "evalrun_123", "object": "eval.run.deleted", "deleted": True}, + request=httpx.Request("DELETE", "https://api.openai.com/v1/evals/eval_123/runs/evalrun_123"), + ) + + result = config.transform_delete_run_response( + raw_response=response, + logging_obj=None, + ) + + assert result.run_id == "evalrun_123" + assert result.deleted is True diff --git a/tests/test_litellm/llms/volcengine/responses/test_volcengine_responses_transformation.py b/tests/test_litellm/llms/volcengine/responses/test_volcengine_responses_transformation.py index 13571e63c7d..4581f4af7b6 100644 --- a/tests/test_litellm/llms/volcengine/responses/test_volcengine_responses_transformation.py +++ b/tests/test_litellm/llms/volcengine/responses/test_volcengine_responses_transformation.py @@ -4,9 +4,11 @@ Tests for Volcengine Responses API transformation. import os import sys +from typing import List, Literal, Optional, Union import httpx import pytest +from pydantic import BaseModel, Field sys.path.insert(0, os.path.abspath("../../../../..")) @@ -32,12 +34,10 @@ class TestVolcengineResponsesAPITransformation: ) assert config is not None, "Config should not be None for Volcengine provider" - assert isinstance( - config, VolcEngineResponsesAPIConfig - ), f"Expected VolcEngineResponsesAPIConfig, got {type(config)}" - assert ( - config.custom_llm_provider == LlmProviders.VOLCENGINE - ), "custom_llm_provider should be VOLCENGINE" + assert isinstance(config, VolcEngineResponsesAPIConfig), ( + f"Expected VolcEngineResponsesAPIConfig, got {type(config)}" + ) + assert config.custom_llm_provider == LlmProviders.VOLCENGINE, "custom_llm_provider should be VOLCENGINE" def test_parallel_tool_calls_dropped(self): """Volcengine does not list parallel_tool_calls; ensure it is removed.""" @@ -54,9 +54,7 @@ class TestVolcengineResponsesAPITransformation: drop_params=False, ) - assert ( - "parallel_tool_calls" not in mapped - ), "parallel_tool_calls must be dropped" + assert "parallel_tool_calls" not in mapped, "parallel_tool_calls must be dropped" assert mapped.get("temperature") == 0.5 assert "metadata" not in mapped, "Undocumented params should not be included" @@ -91,14 +89,10 @@ class TestVolcengineResponsesAPITransformation: default_url = config.get_complete_url(api_base=None, litellm_params={}) assert default_url == "https://ark.cn-beijing.volces.com/api/v3/responses" - api_base_with_api = config.get_complete_url( - api_base="https://custom.volc.com/api/v3", litellm_params={} - ) + api_base_with_api = config.get_complete_url(api_base="https://custom.volc.com/api/v3", litellm_params={}) assert api_base_with_api == "https://custom.volc.com/api/v3/responses" - api_base_full = config.get_complete_url( - api_base="https://custom.volc.com/api/v3/responses", litellm_params={} - ) + api_base_full = config.get_complete_url(api_base="https://custom.volc.com/api/v3/responses", litellm_params={}) assert api_base_full == "https://custom.volc.com/api/v3/responses" def test_response_id_path_requests_encode_response_id(self): @@ -112,10 +106,7 @@ class TestVolcengineResponsesAPITransformation: headers={}, ) - assert ( - url - == "https://custom.volc.com/api/v3/responses/..%2F..%2Fresponses%2Fother%3Fx%3D1%23frag/cancel" - ) + assert url == "https://custom.volc.com/api/v3/responses/..%2F..%2Fresponses%2Fother%3Fx%3D1%23frag/cancel" assert params == {} @pytest.mark.parametrize( @@ -125,9 +116,7 @@ class TestVolcengineResponsesAPITransformation: (GenericLiteLLMParams(api_key="attr-key"), "attr-key"), ], ) - def test_validate_environment_uses_api_key( - self, monkeypatch, litellm_params, expected_key - ): + def test_validate_environment_uses_api_key(self, monkeypatch, litellm_params, expected_key): """validate_environment should pull api key from params/env and attach headers.""" config = VolcEngineResponsesAPIConfig() @@ -135,9 +124,7 @@ class TestVolcengineResponsesAPITransformation: monkeypatch.delenv("ARK_API_KEY", raising=False) monkeypatch.delenv("VOLCENGINE_API_KEY", raising=False) - headers = config.validate_environment( - headers={}, model="volcengine/demo-model", litellm_params=litellm_params - ) + headers = config.validate_environment(headers={}, model="volcengine/demo-model", litellm_params=litellm_params) assert headers.get("Authorization") == f"Bearer {expected_key}" assert headers.get("Content-Type") == "application/json" @@ -151,9 +138,7 @@ class TestVolcengineResponsesAPITransformation: monkeypatch.delenv("VOLCENGINE_API_KEY", raising=False) with pytest.raises(ValueError): - config.validate_environment( - headers={}, model="volcengine/demo", litellm_params={} - ) + config.validate_environment(headers={}, model="volcengine/demo", litellm_params={}) def test_unsupported_params_are_dropped_with_extra_body(self): """Unknown fields (including extra_body) should be dropped before send.""" @@ -240,9 +225,7 @@ class TestVolcengineResponsesAPITransformation: # Use class name comparison instead of isinstance to avoid issues with # module reloading during parallel test execution (conftest reloads litellm) - assert ( - type(error).__name__ == "VolcEngineError" - ), f"Expected VolcEngineError, got {type(error).__name__}" + assert type(error).__name__ == "VolcEngineError", f"Expected VolcEngineError, got {type(error).__name__}" assert error.status_code == 400 assert error.message == "bad request" assert error.headers.get("x") == "y" @@ -296,3 +279,206 @@ class TestVolcengineResponsesAPITransformation: assert isinstance(result, DeleteResponseResult) assert result.deleted is True + + def test_transform_streaming_response_fills_missing_required_fields(self): + config = VolcEngineResponsesAPIConfig() + + event = config.transform_streaming_response( + model="volcengine/demo-model", + parsed_chunk={"type": "response.completed", "response": {"id": "resp_1"}}, + logging_obj=None, + ) + + assert type(event).__name__ == "ResponseCompletedEvent" + assert event.type == "response.completed" + assert event.response.id == "resp_1" + assert event.response.output == [] + assert event.response.created_at == 0 + + def test_transform_response_api_response_falls_back_to_model_construct(self): + config = VolcEngineResponsesAPIConfig() + http_response = httpx.Response( + status_code=200, + json={"id": "resp_fallback", "created_at": 123, "output": "not-a-list"}, + request=httpx.Request("POST", "https://example.com/responses"), + headers={"x-test": "1"}, + ) + + result = config.transform_response_api_response( + model="volcengine/demo-model", + raw_response=http_response, + logging_obj=type( + "Logger", + (), + {"post_call": staticmethod(lambda **kwargs: None)}, + ), + ) + + assert result.id == "resp_fallback" + assert result.output == "not-a-list" + assert result._hidden_params["headers"].get("x-test") == "1" + + def test_transform_delete_response_api_request_builds_url(self): + config = VolcEngineResponsesAPIConfig() + + url, data = config.transform_delete_response_api_request( + response_id="resp_123", + api_base="https://custom.volc.com/api/v3/responses", + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + + assert url == "https://custom.volc.com/api/v3/responses/resp_123" + assert data == {} + + def test_transform_get_response_api_request_and_response(self): + config = VolcEngineResponsesAPIConfig() + + url, data = config.transform_get_response_api_request( + response_id="resp 123", + api_base="https://custom.volc.com/api/v3/responses", + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + + assert url == "https://custom.volc.com/api/v3/responses/resp%20123" + assert data == {} + + http_response = httpx.Response( + status_code=200, + json={ + "id": "resp_123", + "object": "response", + "created_at": 123, + "status": "completed", + "output": [], + "model": "demo-model", + }, + request=httpx.Request("GET", url), + headers={"x-test": "1"}, + ) + + result = config.transform_get_response_api_response( + raw_response=http_response, + logging_obj=None, + ) + + assert result.id == "resp_123" + assert result._hidden_params["headers"].get("x-test") == "1" + + def test_transform_cancel_response_api_response_parses_json(self): + config = VolcEngineResponsesAPIConfig() + http_response = httpx.Response( + status_code=200, + json={ + "id": "resp_123", + "object": "response", + "created_at": 123, + "status": "cancelled", + "output": [], + "model": "demo-model", + }, + request=httpx.Request("POST", "https://example.com/responses/resp_123/cancel"), + headers={"x-test": "1"}, + ) + + result = config.transform_cancel_response_api_response( + raw_response=http_response, + logging_obj=None, + ) + + assert result.id == "resp_123" + assert result.status == "cancelled" + assert result._hidden_params["headers"].get("x-test") == "1" + + def test_transform_list_input_items_request_builds_query_params(self): + config = VolcEngineResponsesAPIConfig() + + url, params = config.transform_list_input_items_request( + response_id="resp_123", + api_base="https://custom.volc.com/api/v3/responses", + litellm_params=GenericLiteLLMParams(), + headers={}, + after="item_a", + before="item_b", + include=["metadata", "usage"], + limit=5, + order="asc", + ) + + assert url == "https://custom.volc.com/api/v3/responses/resp_123/input_items" + assert params == { + "after": "item_a", + "before": "item_b", + "include": "metadata,usage", + "limit": 5, + "order": "asc", + } + + def test_transform_list_input_items_response_returns_parsed_body(self): + config = VolcEngineResponsesAPIConfig() + payload = {"object": "list", "data": [{"id": "item_1"}]} + http_response = httpx.Response( + status_code=200, + json=payload, + request=httpx.Request("GET", "https://example.com/responses/resp_123/input_items"), + ) + + result = config.transform_list_input_items_response( + raw_response=http_response, + logging_obj=None, + ) + + assert result == payload + + +class _FillWidget(BaseModel): + type: Literal["widget"] + count: int + parts: List[str] + label: Optional[str] + + +class _FillGadget(BaseModel): + type: Literal["gadget"] + name: str + + +class _FillEnvelope(BaseModel): + kind: str = "envelope" + tags: List[str] = Field(default_factory=lambda: ["default-tag"]) + payload: Union[_FillWidget, _FillGadget] + entries: List[_FillWidget] + note: Optional[str] + values: Union[List[str], str] + + +class TestVolcengineStreamingFieldFill: + def test_fill_uses_defaults_factories_and_heuristics(self): + filled = VolcEngineResponsesAPIConfig._fill_missing_fields( + {"payload": {"type": "gadget", "name": "g"}, "entries": [{"type": "widget"}]}, + _FillEnvelope, + ) + + assert filled["kind"] == "envelope" + assert filled["tags"] == ["default-tag"] + assert filled["note"] is None + assert filled["values"] == [] + + validated = _FillEnvelope.model_validate(filled) + assert isinstance(validated.payload, _FillGadget) + assert validated.entries[0].count == 0 + assert validated.entries[0].parts == [] + assert validated.entries[0].label is None + + def test_fill_selects_union_member_by_type_literal(self): + filled = VolcEngineResponsesAPIConfig._fill_missing_fields( + {"payload": {"type": "widget"}, "entries": []}, + _FillEnvelope, + ) + + validated = _FillEnvelope.model_validate(filled) + assert isinstance(validated.payload, _FillWidget) + assert validated.payload.count == 0 + assert validated.payload.parts == [] + assert validated.payload.label is None