mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-15 09:12:18 +00:00
test(e2e): use gpt-image-2.6-flare and check every n=2 choice reaches Langfuse
Some checks are pending
Cost map pins recheck / rerun-cost-map-guard (push) Waiting to run
LiteLLM Rust DB / db-queries (push) Waiting to run
LiteLLM Rust / rust-lint (push) Waiting to run
LiteLLM Rust / rust-test (push) Waiting to run
LiteLLM Rust / rust-wheel (push) Waiting to run
Terraform Modules / fmt, validate, test (aws) (push) Waiting to run
Terraform Modules / fmt, validate, test (gcp) (push) Waiting to run
Terraform Provider / gofmt, vet, build, test (push) Waiting to run
Terraform Provider / Provider endpoints vs proxy OpenAPI schema (push) Waiting to run
Some checks are pending
Cost map pins recheck / rerun-cost-map-guard (push) Waiting to run
LiteLLM Rust DB / db-queries (push) Waiting to run
LiteLLM Rust / rust-lint (push) Waiting to run
LiteLLM Rust / rust-test (push) Waiting to run
LiteLLM Rust / rust-wheel (push) Waiting to run
Terraform Modules / fmt, validate, test (aws) (push) Waiting to run
Terraform Modules / fmt, validate, test (gcp) (push) Waiting to run
Terraform Provider / gofmt, vet, build, test (push) Waiting to run
Terraform Provider / Provider endpoints vs proxy OpenAPI schema (push) Waiting to run
OpenAI names gpt-image-2.6-flare as the successor of gpt-image-1-mini, so the image tests use it. The n=2 completions test now needs one logged message per choice, since both choices repeat the same text and a single logged message used to pass
This commit is contained in:
parent
d421c968ed
commit
09cfd7a0f8
2 changed files with 11 additions and 7 deletions
|
|
@ -23,7 +23,7 @@ from sdk_clients import SdkClients
|
|||
|
||||
pytestmark = pytest.mark.e2e
|
||||
|
||||
OPENAI_IMAGE_BACKEND: Final = "openai/gpt-image-2.5-flare"
|
||||
OPENAI_IMAGE_BACKEND: Final = "openai/gpt-image-2.6-flare"
|
||||
BEDROCK_IMAGE_BACKEND: Final = "bedrock/stability.stable-image-core-v1:1"
|
||||
# Nova Canvas reached end of life on 2026-09-30. Stable Image Core is active on Bedrock only in us-west-2:
|
||||
# https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-stability-ai-stable-image-core-1-0.html (2026-10-16)
|
||||
|
|
|
|||
|
|
@ -71,7 +71,7 @@ RED_SQUARE_PNG: Final = base64.b64decode(
|
|||
BOUNDED_OUTPUT_CHARS: Final = 1024
|
||||
PLACEHOLDER_INPUT: Final = "default-message-value"
|
||||
COMPLETION_BACKEND: Final = "openai/gpt-5.4-mini"
|
||||
IMAGE_BACKEND: Final = "openai/gpt-image-2.5-flare"
|
||||
IMAGE_BACKEND: Final = "openai/gpt-image-2.6-flare"
|
||||
SPEECH_BACKEND: Final = "openai/tts-1-1106"
|
||||
TRANSCRIPTION_BACKEND: Final = "openai/gpt-transcribe"
|
||||
MODERATION_BACKEND: Final = "openai/omni-moderation-latest"
|
||||
|
|
@ -123,14 +123,18 @@ def _generation(client: LoggingClient, creds: LangfuseCreds, *, alias: str, star
|
|||
return observation
|
||||
|
||||
|
||||
def _output_text(observation: LangfuseObservation) -> str:
|
||||
def _output_contents(observation: LangfuseObservation) -> tuple[str, ...]:
|
||||
assert observation.output not in (None, "", [], {}), f"generation output is empty: {observation!r}"
|
||||
try:
|
||||
messages: Final = _OUTPUT_MESSAGES.validate_python(observation.output)
|
||||
except ValidationError:
|
||||
pytest.fail(f"generation output is not a list of assistant messages: {observation!r}")
|
||||
assert messages, f"generation output is empty: {observation!r}"
|
||||
return "\n".join(message.content for message in messages)
|
||||
return tuple(message.content for message in messages)
|
||||
|
||||
|
||||
def _output_text(observation: LangfuseObservation) -> str:
|
||||
return "\n".join(_output_contents(observation))
|
||||
|
||||
|
||||
def _input_text(observation: LangfuseObservation) -> str:
|
||||
|
|
@ -204,9 +208,9 @@ class TestOtelV2LangfuseGenerationOutput:
|
|||
texts: Final = tuple(choice.text.strip() for choice in response.choices)
|
||||
assert len(texts) == 2 and all(texts), f"/v1/completions returned no text: {response!r}"
|
||||
|
||||
output: Final = _output_text(_generation(client, langfuse_creds, alias=alias, started=started))
|
||||
assert all(text in output for text in texts), (
|
||||
f"generation output lacks the completion texts {texts!r}: {output!r}"
|
||||
contents: Final = _output_contents(_generation(client, langfuse_creds, alias=alias, started=started))
|
||||
assert sorted(content.strip() for content in contents) == sorted(texts), (
|
||||
f"generation output does not hold one message per completion choice {texts!r}: {contents!r}"
|
||||
)
|
||||
|
||||
@pytest.mark.covers("logging.langfuse.success.logs_spend", exercised_on=["images_generations"])
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue