mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-06 08:16:43 +00:00
e8b9f3675b
batch_completion collects per-request failures into its result list rather than raising them; its own source says "return exceptions if any". So the test's `except Timeout` and `except litellm.InternalServerError` arms could never fire for the case they were written for. An upstream 500 instead reached `response.choices`, raised AttributeError on the exception object, and fell through to the bare `except Exception` that calls pytest.fail. That is what CircleCI hit. The tolerance now reads the returned values, which is where the failures actually are. The same two exception types are tolerated as before, nothing broader. Checked against four injected outcomes: three InternalServerErrors pass, three Timeouts pass, an AuthenticationError fails, and a response whose content is None fails. So it is not tolerating its way to a vacuous green.
86 lines
2.1 KiB
Python
86 lines
2.1 KiB
Python
#### What this tests ####
|
|
# This tests calling batch_completions by running 100 messages together
|
|
|
|
import sys, os
|
|
import traceback
|
|
import pytest
|
|
|
|
from openai import APITimeoutError as Timeout
|
|
import litellm
|
|
|
|
litellm.num_retries = 0
|
|
from litellm import (
|
|
batch_completion,
|
|
batch_completion_models,
|
|
completion,
|
|
batch_completion_models_all_responses,
|
|
)
|
|
|
|
# litellm.set_verbose=True
|
|
|
|
|
|
TOLERATED_UPSTREAM_FAILURES = (Timeout, litellm.InternalServerError)
|
|
|
|
|
|
def test_batch_completions():
|
|
messages = [[{"role": "user", "content": "write a short poem"}] for _ in range(3)]
|
|
model = "gpt-3.5-turbo"
|
|
litellm.set_verbose = True
|
|
|
|
result = batch_completion(
|
|
model=model,
|
|
messages=messages,
|
|
max_tokens=10,
|
|
temperature=0.2,
|
|
request_timeout=1,
|
|
)
|
|
print(result)
|
|
|
|
assert len(result) == 3
|
|
|
|
for response in result:
|
|
if isinstance(response, TOLERATED_UPSTREAM_FAILURES):
|
|
continue
|
|
assert not isinstance(
|
|
response, Exception
|
|
), f"batch_completion returned {type(response).__name__}: {response}"
|
|
assert response.choices[0].message.content is not None
|
|
|
|
|
|
# test_batch_completions()
|
|
|
|
|
|
def test_batch_completions_models():
|
|
try:
|
|
result = batch_completion_models(
|
|
models=["gpt-3.5-turbo", "gpt-3.5-turbo", "gpt-3.5-turbo"],
|
|
messages=[{"role": "user", "content": "Hey, how's it going"}],
|
|
)
|
|
print(result)
|
|
except Timeout as e:
|
|
pass
|
|
except Exception as e:
|
|
pytest.fail(f"An error occurred: {e}")
|
|
|
|
|
|
# test_batch_completions_models()
|
|
|
|
|
|
def test_batch_completion_models_all_responses():
|
|
try:
|
|
responses = batch_completion_models_all_responses(
|
|
models=["gemini/gemini-2.5-flash-lite", "claude-haiku-4-5-20251001"],
|
|
messages=[{"role": "user", "content": "write a poem"}],
|
|
max_tokens=10,
|
|
)
|
|
print(responses)
|
|
assert len(responses) == 2
|
|
except Timeout as e:
|
|
pass
|
|
except litellm.APIError as e:
|
|
pass
|
|
except Exception as e:
|
|
pytest.fail(f"An error occurred: {e}")
|
|
|
|
|
|
# test_batch_completion_models_all_responses()
|