mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-13 23:11:40 +00:00
test(harness): move live OpenAI chat tests to chat_live_openai suite
Per audit 5/8c: - test_openai.py: live prediction-param pair, TestOpenAIChatCompletion, o1 parallel tool calls, web search (+streaming) with the validate_web_search_annotations helper, codex (+stream), gemini streaming bridge, deepresearch bridge, tool calling, gpt5/gpt-5-codex reasoning, n>1 streaming pair, gpt-5 web search; the streaming-handler reasoning golden, pdf-url and xhigh-reasoning keepers stay; TestOpenAIGPT4OAudioTranscription stays (non-chat, Sameer) - test_openai_o1.py: TestOpenAIO1, TestOpenAIO3, test_o3_reasoning_effort, test_streaming_response moved; the o1 transform goldens (system-role/max_completion_tokens mapping, vision support, completion_tokens_details) stay - test_gpt4o_audio.py, test_prompt_caching.py, test_router_llm_translation_tests.py moved whole (live, nothing local)
This commit is contained in:
parent
b9a4d9bd41
commit
ed8ce74ef3
5 changed files with 1127 additions and 1102 deletions
|
|
@ -50,7 +50,6 @@ async def check_streaming_response(completion):
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
# @pytest.mark.flaky(retries=3, delay=1)
|
||||
@pytest.mark.parametrize("stream", [True, False])
|
||||
async def test_audio_output_from_model(stream):
|
||||
audio_format = "pcm16"
|
||||
|
|
|
|||
1049
tests/harness_suites/chat_live_openai/test_openai_chat_live.py
Normal file
1049
tests/harness_suites/chat_live_openai/test_openai_chat_live.py
Normal file
File diff suppressed because one or more lines are too long
|
|
@ -0,0 +1,78 @@
|
|||
import os
|
||||
import sys
|
||||
from unittest.mock import patch
|
||||
|
||||
sys.path.insert(
|
||||
0, os.path.abspath("../..")
|
||||
) # Adds the parent directory to the system path
|
||||
|
||||
|
||||
import pytest
|
||||
|
||||
import litellm
|
||||
from litellm import ModelResponse
|
||||
from base_llm_unit_tests import BaseLLMChatTest
|
||||
|
||||
class TestOpenAIO1(BaseLLMChatTest):
|
||||
def get_base_completion_call_args(self):
|
||||
return {
|
||||
"model": "o1",
|
||||
}
|
||||
|
||||
def get_client(self):
|
||||
from openai import OpenAI
|
||||
|
||||
return OpenAI(api_key="fake-api-key")
|
||||
|
||||
def test_prompt_caching(self):
|
||||
"""Temporary override. o1 prompt caching is not working."""
|
||||
pass
|
||||
|
||||
|
||||
class TestOpenAIO3(BaseLLMChatTest):
|
||||
def get_base_completion_call_args(self):
|
||||
return {
|
||||
"model": "o3-mini",
|
||||
}
|
||||
|
||||
def get_client(self):
|
||||
from openai import OpenAI
|
||||
|
||||
return OpenAI(api_key="fake-api-key")
|
||||
|
||||
def test_prompt_caching(self):
|
||||
"""Override, as o3 prompt caching is flaky"""
|
||||
pass
|
||||
|
||||
|
||||
def test_o3_reasoning_effort():
|
||||
resp = litellm.completion(
|
||||
model="o3-mini",
|
||||
messages=[{"role": "user", "content": "Hello!"}],
|
||||
reasoning_effort="high",
|
||||
)
|
||||
assert resp.choices[0].message.content is not None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", ["o1", "o3-mini"])
|
||||
def test_streaming_response(model):
|
||||
"""Test that streaming response is returned correctly"""
|
||||
from litellm import completion
|
||||
|
||||
response = completion(
|
||||
model=model,
|
||||
messages=[
|
||||
{"role": "system", "content": "Be a good bot!"},
|
||||
{"role": "user", "content": "Hello!"},
|
||||
],
|
||||
stream=True,
|
||||
)
|
||||
|
||||
assert response is not None
|
||||
|
||||
chunks = []
|
||||
for chunk in response:
|
||||
chunks.append(chunk)
|
||||
|
||||
resp = litellm.stream_chunk_builder(chunks=chunks)
|
||||
print(resp)
|
||||
File diff suppressed because one or more lines are too long
|
|
@ -11,7 +11,6 @@ import pytest
|
|||
|
||||
import litellm
|
||||
from litellm import ModelResponse
|
||||
from base_llm_unit_tests import BaseLLMChatTest
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", ["o1"])
|
||||
|
|
@ -110,38 +109,6 @@ def test_litellm_responses():
|
|||
assert isinstance(response.usage.completion_tokens_details, CompletionTokensDetails)
|
||||
|
||||
|
||||
class TestOpenAIO1(BaseLLMChatTest):
|
||||
def get_base_completion_call_args(self):
|
||||
return {
|
||||
"model": "o1",
|
||||
}
|
||||
|
||||
def get_client(self):
|
||||
from openai import OpenAI
|
||||
|
||||
return OpenAI(api_key="fake-api-key")
|
||||
|
||||
def test_prompt_caching(self):
|
||||
"""Temporary override. o1 prompt caching is not working."""
|
||||
pass
|
||||
|
||||
|
||||
class TestOpenAIO3(BaseLLMChatTest):
|
||||
def get_base_completion_call_args(self):
|
||||
return {
|
||||
"model": "o3-mini",
|
||||
}
|
||||
|
||||
def get_client(self):
|
||||
from openai import OpenAI
|
||||
|
||||
return OpenAI(api_key="fake-api-key")
|
||||
|
||||
def test_prompt_caching(self):
|
||||
"""Override, as o3 prompt caching is flaky"""
|
||||
pass
|
||||
|
||||
|
||||
def test_o1_supports_vision():
|
||||
"""Test that o1 supports vision"""
|
||||
os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True"
|
||||
|
|
@ -151,34 +118,3 @@ def test_o1_supports_vision():
|
|||
assert v.get("supports_vision") is True, f"{k} does not support vision"
|
||||
|
||||
|
||||
def test_o3_reasoning_effort():
|
||||
resp = litellm.completion(
|
||||
model="o3-mini",
|
||||
messages=[{"role": "user", "content": "Hello!"}],
|
||||
reasoning_effort="high",
|
||||
)
|
||||
assert resp.choices[0].message.content is not None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", ["o1", "o3-mini"])
|
||||
def test_streaming_response(model):
|
||||
"""Test that streaming response is returned correctly"""
|
||||
from litellm import completion
|
||||
|
||||
response = completion(
|
||||
model=model,
|
||||
messages=[
|
||||
{"role": "system", "content": "Be a good bot!"},
|
||||
{"role": "user", "content": "Hello!"},
|
||||
],
|
||||
stream=True,
|
||||
)
|
||||
|
||||
assert response is not None
|
||||
|
||||
chunks = []
|
||||
for chunk in response:
|
||||
chunks.append(chunk)
|
||||
|
||||
resp = litellm.stream_chunk_builder(chunks=chunks)
|
||||
print(resp)
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue