mirror of
https://github.com/BerriAI/litellm.git
synced 2026-08-28 05:25:59 +00:00
test: cover volcengine responses and openai evals transformations
Exercises the streaming field-fill heuristics, model_construct fallbacks, and the get/cancel/delete/list request and response transforms that had no tests.
This commit is contained in:
mateo-berri
parent
26ab846ebf
commit
8b08c31ebe
2 changed files with 383 additions and 38 deletions
|
|
@ -252,9 +252,7 @@ def test_transform_cancel_eval_response(config: OpenAIEvalsConfig):
|
|||
"object": "eval",
|
||||
"status": "cancelled",
|
||||
},
|
||||
request=httpx.Request(
|
||||
"POST", "https://api.openai.com/v1/evals/eval_123/cancel"
|
||||
),
|
||||
request=httpx.Request("POST", "https://api.openai.com/v1/evals/eval_123/cancel"),
|
||||
)
|
||||
|
||||
result = config.transform_cancel_eval_response(
|
||||
|
|
@ -276,8 +274,169 @@ def test_transform_run_requests_encode_eval_and_run_ids(config: OpenAIEvalsConfi
|
|||
headers={},
|
||||
)
|
||||
|
||||
assert (
|
||||
url
|
||||
== "https://api.openai.com/v1/evals/..%2F..%2Fevals%3Fx%3D1%23frag/runs/..%2Fruns%23other/cancel"
|
||||
)
|
||||
assert url == "https://api.openai.com/v1/evals/..%2F..%2Fevals%3Fx%3D1%23frag/runs/..%2Fruns%23other/cancel"
|
||||
assert request_body == {}
|
||||
|
||||
|
||||
def _eval_json_response(url: str, method: str = "GET") -> httpx.Response:
|
||||
return httpx.Response(
|
||||
status_code=200,
|
||||
json={
|
||||
"id": "eval_123",
|
||||
"object": "eval",
|
||||
"created_at": 1234567890,
|
||||
"name": "Test Eval",
|
||||
"data_source_config": {"type": "stored_completions"},
|
||||
"testing_criteria": [],
|
||||
},
|
||||
request=httpx.Request(method, url),
|
||||
)
|
||||
|
||||
|
||||
def _run_json(run_id: str = "evalrun_123", status: str = "queued") -> dict:
|
||||
return {
|
||||
"id": run_id,
|
||||
"object": "eval.run",
|
||||
"created_at": 1234567890,
|
||||
"status": status,
|
||||
"data_source": {"type": "completions"},
|
||||
"eval_id": "eval_123",
|
||||
}
|
||||
|
||||
|
||||
def test_transform_get_eval_response(config: OpenAIEvalsConfig):
|
||||
result = config.transform_get_eval_response(
|
||||
raw_response=_eval_json_response("https://api.openai.com/v1/evals/eval_123"),
|
||||
logging_obj=None,
|
||||
)
|
||||
|
||||
assert result.id == "eval_123"
|
||||
assert result.object == "eval"
|
||||
assert result.name == "Test Eval"
|
||||
|
||||
|
||||
def test_transform_update_eval_response(config: OpenAIEvalsConfig):
|
||||
result = config.transform_update_eval_response(
|
||||
raw_response=_eval_json_response("https://api.openai.com/v1/evals/eval_123", method="POST"),
|
||||
logging_obj=None,
|
||||
)
|
||||
|
||||
assert result.id == "eval_123"
|
||||
assert result.name == "Test Eval"
|
||||
|
||||
|
||||
def test_transform_create_run_response(config: OpenAIEvalsConfig):
|
||||
response = httpx.Response(
|
||||
status_code=200,
|
||||
json=_run_json(),
|
||||
request=httpx.Request("POST", "https://api.openai.com/v1/evals/eval_123/runs"),
|
||||
)
|
||||
|
||||
result = config.transform_create_run_response(
|
||||
raw_response=response,
|
||||
logging_obj=None,
|
||||
)
|
||||
|
||||
assert result.id == "evalrun_123"
|
||||
assert result.status == "queued"
|
||||
assert result.eval_id == "eval_123"
|
||||
|
||||
|
||||
def test_transform_list_runs_request(config: OpenAIEvalsConfig):
|
||||
url, query_params = config.transform_list_runs_request(
|
||||
eval_id="eval_123",
|
||||
list_params={"limit": 5, "after": "evalrun_1", "order": "asc"},
|
||||
litellm_params=GenericLiteLLMParams(api_base="https://api.openai.com"),
|
||||
headers={},
|
||||
)
|
||||
|
||||
assert url == "https://api.openai.com/v1/evals/eval_123/runs"
|
||||
assert query_params == {"limit": 5, "after": "evalrun_1", "order": "asc"}
|
||||
|
||||
|
||||
def test_transform_list_runs_response(config: OpenAIEvalsConfig):
|
||||
response = httpx.Response(
|
||||
status_code=200,
|
||||
json={
|
||||
"object": "list",
|
||||
"data": [_run_json()],
|
||||
"first_id": "evalrun_123",
|
||||
"last_id": "evalrun_123",
|
||||
"has_more": False,
|
||||
},
|
||||
request=httpx.Request("GET", "https://api.openai.com/v1/evals/eval_123/runs"),
|
||||
)
|
||||
|
||||
result = config.transform_list_runs_response(
|
||||
raw_response=response,
|
||||
logging_obj=None,
|
||||
)
|
||||
|
||||
assert result.object == "list"
|
||||
assert len(result.data) == 1
|
||||
assert result.data[0].id == "evalrun_123"
|
||||
assert result.has_more is False
|
||||
|
||||
|
||||
def test_transform_get_run_response(config: OpenAIEvalsConfig):
|
||||
response = httpx.Response(
|
||||
status_code=200,
|
||||
json=_run_json(status="completed"),
|
||||
request=httpx.Request("GET", "https://api.openai.com/v1/evals/eval_123/runs/evalrun_123"),
|
||||
)
|
||||
|
||||
result = config.transform_get_run_response(
|
||||
raw_response=response,
|
||||
logging_obj=None,
|
||||
)
|
||||
|
||||
assert result.id == "evalrun_123"
|
||||
assert result.status == "completed"
|
||||
|
||||
|
||||
def test_transform_cancel_run_response(config: OpenAIEvalsConfig):
|
||||
response = httpx.Response(
|
||||
status_code=200,
|
||||
json={"id": "evalrun_123", "object": "eval.run", "status": "cancelled"},
|
||||
request=httpx.Request(
|
||||
"POST",
|
||||
"https://api.openai.com/v1/evals/eval_123/runs/evalrun_123/cancel",
|
||||
),
|
||||
)
|
||||
|
||||
result = config.transform_cancel_run_response(
|
||||
raw_response=response,
|
||||
logging_obj=None,
|
||||
)
|
||||
|
||||
assert result.id == "evalrun_123"
|
||||
assert result.status == "cancelled"
|
||||
|
||||
|
||||
def test_transform_delete_run_request(config: OpenAIEvalsConfig):
|
||||
url, headers, request_body = config.transform_delete_run_request(
|
||||
eval_id="eval_123",
|
||||
run_id="evalrun_123",
|
||||
api_base="https://api.openai.com",
|
||||
litellm_params=GenericLiteLLMParams(),
|
||||
headers={},
|
||||
)
|
||||
|
||||
assert url == "https://api.openai.com/v1/evals/eval_123/runs/evalrun_123"
|
||||
assert request_body == {}
|
||||
|
||||
|
||||
def test_transform_delete_run_response(config: OpenAIEvalsConfig):
|
||||
response = httpx.Response(
|
||||
status_code=200,
|
||||
json={"run_id": "evalrun_123", "object": "eval.run.deleted", "deleted": True},
|
||||
request=httpx.Request("DELETE", "https://api.openai.com/v1/evals/eval_123/runs/evalrun_123"),
|
||||
)
|
||||
|
||||
result = config.transform_delete_run_response(
|
||||
raw_response=response,
|
||||
logging_obj=None,
|
||||
)
|
||||
|
||||
assert result.run_id == "evalrun_123"
|
||||
assert result.deleted is True
|
||||
|
|
|
|||
|
|
@ -4,9 +4,11 @@ Tests for Volcengine Responses API transformation.
|
|||
|
||||
import os
|
||||
import sys
|
||||
from typing import List, Literal, Optional, Union
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
sys.path.insert(0, os.path.abspath("../../../../.."))
|
||||
|
||||
|
|
@ -32,12 +34,10 @@ class TestVolcengineResponsesAPITransformation:
|
|||
)
|
||||
|
||||
assert config is not None, "Config should not be None for Volcengine provider"
|
||||
assert isinstance(
|
||||
config, VolcEngineResponsesAPIConfig
|
||||
), f"Expected VolcEngineResponsesAPIConfig, got {type(config)}"
|
||||
assert (
|
||||
config.custom_llm_provider == LlmProviders.VOLCENGINE
|
||||
), "custom_llm_provider should be VOLCENGINE"
|
||||
assert isinstance(config, VolcEngineResponsesAPIConfig), (
|
||||
f"Expected VolcEngineResponsesAPIConfig, got {type(config)}"
|
||||
)
|
||||
assert config.custom_llm_provider == LlmProviders.VOLCENGINE, "custom_llm_provider should be VOLCENGINE"
|
||||
|
||||
def test_parallel_tool_calls_dropped(self):
|
||||
"""Volcengine does not list parallel_tool_calls; ensure it is removed."""
|
||||
|
|
@ -54,9 +54,7 @@ class TestVolcengineResponsesAPITransformation:
|
|||
drop_params=False,
|
||||
)
|
||||
|
||||
assert (
|
||||
"parallel_tool_calls" not in mapped
|
||||
), "parallel_tool_calls must be dropped"
|
||||
assert "parallel_tool_calls" not in mapped, "parallel_tool_calls must be dropped"
|
||||
assert mapped.get("temperature") == 0.5
|
||||
assert "metadata" not in mapped, "Undocumented params should not be included"
|
||||
|
||||
|
|
@ -91,14 +89,10 @@ class TestVolcengineResponsesAPITransformation:
|
|||
default_url = config.get_complete_url(api_base=None, litellm_params={})
|
||||
assert default_url == "https://ark.cn-beijing.volces.com/api/v3/responses"
|
||||
|
||||
api_base_with_api = config.get_complete_url(
|
||||
api_base="https://custom.volc.com/api/v3", litellm_params={}
|
||||
)
|
||||
api_base_with_api = config.get_complete_url(api_base="https://custom.volc.com/api/v3", litellm_params={})
|
||||
assert api_base_with_api == "https://custom.volc.com/api/v3/responses"
|
||||
|
||||
api_base_full = config.get_complete_url(
|
||||
api_base="https://custom.volc.com/api/v3/responses", litellm_params={}
|
||||
)
|
||||
api_base_full = config.get_complete_url(api_base="https://custom.volc.com/api/v3/responses", litellm_params={})
|
||||
assert api_base_full == "https://custom.volc.com/api/v3/responses"
|
||||
|
||||
def test_response_id_path_requests_encode_response_id(self):
|
||||
|
|
@ -112,10 +106,7 @@ class TestVolcengineResponsesAPITransformation:
|
|||
headers={},
|
||||
)
|
||||
|
||||
assert (
|
||||
url
|
||||
== "https://custom.volc.com/api/v3/responses/..%2F..%2Fresponses%2Fother%3Fx%3D1%23frag/cancel"
|
||||
)
|
||||
assert url == "https://custom.volc.com/api/v3/responses/..%2F..%2Fresponses%2Fother%3Fx%3D1%23frag/cancel"
|
||||
assert params == {}
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
|
|
@ -125,9 +116,7 @@ class TestVolcengineResponsesAPITransformation:
|
|||
(GenericLiteLLMParams(api_key="attr-key"), "attr-key"),
|
||||
],
|
||||
)
|
||||
def test_validate_environment_uses_api_key(
|
||||
self, monkeypatch, litellm_params, expected_key
|
||||
):
|
||||
def test_validate_environment_uses_api_key(self, monkeypatch, litellm_params, expected_key):
|
||||
"""validate_environment should pull api key from params/env and attach headers."""
|
||||
config = VolcEngineResponsesAPIConfig()
|
||||
|
||||
|
|
@ -135,9 +124,7 @@ class TestVolcengineResponsesAPITransformation:
|
|||
monkeypatch.delenv("ARK_API_KEY", raising=False)
|
||||
monkeypatch.delenv("VOLCENGINE_API_KEY", raising=False)
|
||||
|
||||
headers = config.validate_environment(
|
||||
headers={}, model="volcengine/demo-model", litellm_params=litellm_params
|
||||
)
|
||||
headers = config.validate_environment(headers={}, model="volcengine/demo-model", litellm_params=litellm_params)
|
||||
|
||||
assert headers.get("Authorization") == f"Bearer {expected_key}"
|
||||
assert headers.get("Content-Type") == "application/json"
|
||||
|
|
@ -151,9 +138,7 @@ class TestVolcengineResponsesAPITransformation:
|
|||
monkeypatch.delenv("VOLCENGINE_API_KEY", raising=False)
|
||||
|
||||
with pytest.raises(ValueError):
|
||||
config.validate_environment(
|
||||
headers={}, model="volcengine/demo", litellm_params={}
|
||||
)
|
||||
config.validate_environment(headers={}, model="volcengine/demo", litellm_params={})
|
||||
|
||||
def test_unsupported_params_are_dropped_with_extra_body(self):
|
||||
"""Unknown fields (including extra_body) should be dropped before send."""
|
||||
|
|
@ -240,9 +225,7 @@ class TestVolcengineResponsesAPITransformation:
|
|||
|
||||
# Use class name comparison instead of isinstance to avoid issues with
|
||||
# module reloading during parallel test execution (conftest reloads litellm)
|
||||
assert (
|
||||
type(error).__name__ == "VolcEngineError"
|
||||
), f"Expected VolcEngineError, got {type(error).__name__}"
|
||||
assert type(error).__name__ == "VolcEngineError", f"Expected VolcEngineError, got {type(error).__name__}"
|
||||
assert error.status_code == 400
|
||||
assert error.message == "bad request"
|
||||
assert error.headers.get("x") == "y"
|
||||
|
|
@ -296,3 +279,206 @@ class TestVolcengineResponsesAPITransformation:
|
|||
|
||||
assert isinstance(result, DeleteResponseResult)
|
||||
assert result.deleted is True
|
||||
|
||||
def test_transform_streaming_response_fills_missing_required_fields(self):
|
||||
config = VolcEngineResponsesAPIConfig()
|
||||
|
||||
event = config.transform_streaming_response(
|
||||
model="volcengine/demo-model",
|
||||
parsed_chunk={"type": "response.completed", "response": {"id": "resp_1"}},
|
||||
logging_obj=None,
|
||||
)
|
||||
|
||||
assert type(event).__name__ == "ResponseCompletedEvent"
|
||||
assert event.type == "response.completed"
|
||||
assert event.response.id == "resp_1"
|
||||
assert event.response.output == []
|
||||
assert event.response.created_at == 0
|
||||
|
||||
def test_transform_response_api_response_falls_back_to_model_construct(self):
|
||||
config = VolcEngineResponsesAPIConfig()
|
||||
http_response = httpx.Response(
|
||||
status_code=200,
|
||||
json={"id": "resp_fallback", "created_at": 123, "output": "not-a-list"},
|
||||
request=httpx.Request("POST", "https://example.com/responses"),
|
||||
headers={"x-test": "1"},
|
||||
)
|
||||
|
||||
result = config.transform_response_api_response(
|
||||
model="volcengine/demo-model",
|
||||
raw_response=http_response,
|
||||
logging_obj=type(
|
||||
"Logger",
|
||||
(),
|
||||
{"post_call": staticmethod(lambda **kwargs: None)},
|
||||
),
|
||||
)
|
||||
|
||||
assert result.id == "resp_fallback"
|
||||
assert result.output == "not-a-list"
|
||||
assert result._hidden_params["headers"].get("x-test") == "1"
|
||||
|
||||
def test_transform_delete_response_api_request_builds_url(self):
|
||||
config = VolcEngineResponsesAPIConfig()
|
||||
|
||||
url, data = config.transform_delete_response_api_request(
|
||||
response_id="resp_123",
|
||||
api_base="https://custom.volc.com/api/v3/responses",
|
||||
litellm_params=GenericLiteLLMParams(),
|
||||
headers={},
|
||||
)
|
||||
|
||||
assert url == "https://custom.volc.com/api/v3/responses/resp_123"
|
||||
assert data == {}
|
||||
|
||||
def test_transform_get_response_api_request_and_response(self):
|
||||
config = VolcEngineResponsesAPIConfig()
|
||||
|
||||
url, data = config.transform_get_response_api_request(
|
||||
response_id="resp 123",
|
||||
api_base="https://custom.volc.com/api/v3/responses",
|
||||
litellm_params=GenericLiteLLMParams(),
|
||||
headers={},
|
||||
)
|
||||
|
||||
assert url == "https://custom.volc.com/api/v3/responses/resp%20123"
|
||||
assert data == {}
|
||||
|
||||
http_response = httpx.Response(
|
||||
status_code=200,
|
||||
json={
|
||||
"id": "resp_123",
|
||||
"object": "response",
|
||||
"created_at": 123,
|
||||
"status": "completed",
|
||||
"output": [],
|
||||
"model": "demo-model",
|
||||
},
|
||||
request=httpx.Request("GET", url),
|
||||
headers={"x-test": "1"},
|
||||
)
|
||||
|
||||
result = config.transform_get_response_api_response(
|
||||
raw_response=http_response,
|
||||
logging_obj=None,
|
||||
)
|
||||
|
||||
assert result.id == "resp_123"
|
||||
assert result._hidden_params["headers"].get("x-test") == "1"
|
||||
|
||||
def test_transform_cancel_response_api_response_parses_json(self):
|
||||
config = VolcEngineResponsesAPIConfig()
|
||||
http_response = httpx.Response(
|
||||
status_code=200,
|
||||
json={
|
||||
"id": "resp_123",
|
||||
"object": "response",
|
||||
"created_at": 123,
|
||||
"status": "cancelled",
|
||||
"output": [],
|
||||
"model": "demo-model",
|
||||
},
|
||||
request=httpx.Request("POST", "https://example.com/responses/resp_123/cancel"),
|
||||
headers={"x-test": "1"},
|
||||
)
|
||||
|
||||
result = config.transform_cancel_response_api_response(
|
||||
raw_response=http_response,
|
||||
logging_obj=None,
|
||||
)
|
||||
|
||||
assert result.id == "resp_123"
|
||||
assert result.status == "cancelled"
|
||||
assert result._hidden_params["headers"].get("x-test") == "1"
|
||||
|
||||
def test_transform_list_input_items_request_builds_query_params(self):
|
||||
config = VolcEngineResponsesAPIConfig()
|
||||
|
||||
url, params = config.transform_list_input_items_request(
|
||||
response_id="resp_123",
|
||||
api_base="https://custom.volc.com/api/v3/responses",
|
||||
litellm_params=GenericLiteLLMParams(),
|
||||
headers={},
|
||||
after="item_a",
|
||||
before="item_b",
|
||||
include=["metadata", "usage"],
|
||||
limit=5,
|
||||
order="asc",
|
||||
)
|
||||
|
||||
assert url == "https://custom.volc.com/api/v3/responses/resp_123/input_items"
|
||||
assert params == {
|
||||
"after": "item_a",
|
||||
"before": "item_b",
|
||||
"include": "metadata,usage",
|
||||
"limit": 5,
|
||||
"order": "asc",
|
||||
}
|
||||
|
||||
def test_transform_list_input_items_response_returns_parsed_body(self):
|
||||
config = VolcEngineResponsesAPIConfig()
|
||||
payload = {"object": "list", "data": [{"id": "item_1"}]}
|
||||
http_response = httpx.Response(
|
||||
status_code=200,
|
||||
json=payload,
|
||||
request=httpx.Request("GET", "https://example.com/responses/resp_123/input_items"),
|
||||
)
|
||||
|
||||
result = config.transform_list_input_items_response(
|
||||
raw_response=http_response,
|
||||
logging_obj=None,
|
||||
)
|
||||
|
||||
assert result == payload
|
||||
|
||||
|
||||
class _FillWidget(BaseModel):
|
||||
type: Literal["widget"]
|
||||
count: int
|
||||
parts: List[str]
|
||||
label: Optional[str]
|
||||
|
||||
|
||||
class _FillGadget(BaseModel):
|
||||
type: Literal["gadget"]
|
||||
name: str
|
||||
|
||||
|
||||
class _FillEnvelope(BaseModel):
|
||||
kind: str = "envelope"
|
||||
tags: List[str] = Field(default_factory=lambda: ["default-tag"])
|
||||
payload: Union[_FillWidget, _FillGadget]
|
||||
entries: List[_FillWidget]
|
||||
note: Optional[str]
|
||||
values: Union[List[str], str]
|
||||
|
||||
|
||||
class TestVolcengineStreamingFieldFill:
|
||||
def test_fill_uses_defaults_factories_and_heuristics(self):
|
||||
filled = VolcEngineResponsesAPIConfig._fill_missing_fields(
|
||||
{"payload": {"type": "gadget", "name": "g"}, "entries": [{"type": "widget"}]},
|
||||
_FillEnvelope,
|
||||
)
|
||||
|
||||
assert filled["kind"] == "envelope"
|
||||
assert filled["tags"] == ["default-tag"]
|
||||
assert filled["note"] is None
|
||||
assert filled["values"] == []
|
||||
|
||||
validated = _FillEnvelope.model_validate(filled)
|
||||
assert isinstance(validated.payload, _FillGadget)
|
||||
assert validated.entries[0].count == 0
|
||||
assert validated.entries[0].parts == []
|
||||
assert validated.entries[0].label is None
|
||||
|
||||
def test_fill_selects_union_member_by_type_literal(self):
|
||||
filled = VolcEngineResponsesAPIConfig._fill_missing_fields(
|
||||
{"payload": {"type": "widget"}, "entries": []},
|
||||
_FillEnvelope,
|
||||
)
|
||||
|
||||
validated = _FillEnvelope.model_validate(filled)
|
||||
assert isinstance(validated.payload, _FillWidget)
|
||||
assert validated.payload.count == 0
|
||||
assert validated.payload.parts == []
|
||||
assert validated.payload.label is None
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue