test(e2e): use gpt-image-2.6-flare and check every n=2 choice reaches Langfuse
Some checks are pending
Cost map pins recheck / rerun-cost-map-guard (push) Waiting to run
LiteLLM Rust DB / db-queries (push) Waiting to run
LiteLLM Rust / rust-lint (push) Waiting to run
LiteLLM Rust / rust-test (push) Waiting to run
LiteLLM Rust / rust-wheel (push) Waiting to run
Terraform Modules / fmt, validate, test (aws) (push) Waiting to run
Terraform Modules / fmt, validate, test (gcp) (push) Waiting to run
Terraform Provider / gofmt, vet, build, test (push) Waiting to run
Terraform Provider / Provider endpoints vs proxy OpenAPI schema (push) Waiting to run

OpenAI names gpt-image-2.6-flare as the successor of gpt-image-1-mini, so the image tests use it. The n=2 completions test now needs one logged message per choice, since both choices repeat the same text and a single logged message used to pass
This commit is contained in:
mateo-berri 2026-11-14 05:50:41 -08:00
parent d421c968ed
commit 09cfd7a0f8
2 changed files with 11 additions and 7 deletions

View file

@ -23,7 +23,7 @@ from sdk_clients import SdkClients
pytestmark = pytest.mark.e2e
OPENAI_IMAGE_BACKEND: Final = "openai/gpt-image-2.5-flare"
OPENAI_IMAGE_BACKEND: Final = "openai/gpt-image-2.6-flare"
BEDROCK_IMAGE_BACKEND: Final = "bedrock/stability.stable-image-core-v1:1"
# Nova Canvas reached end of life on 2026-09-30. Stable Image Core is active on Bedrock only in us-west-2:
# https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-stability-ai-stable-image-core-1-0.html (2026-10-16)

View file

@ -71,7 +71,7 @@ RED_SQUARE_PNG: Final = base64.b64decode(
BOUNDED_OUTPUT_CHARS: Final = 1024
PLACEHOLDER_INPUT: Final = "default-message-value"
COMPLETION_BACKEND: Final = "openai/gpt-5.4-mini"
IMAGE_BACKEND: Final = "openai/gpt-image-2.5-flare"
IMAGE_BACKEND: Final = "openai/gpt-image-2.6-flare"
SPEECH_BACKEND: Final = "openai/tts-1-1106"
TRANSCRIPTION_BACKEND: Final = "openai/gpt-transcribe"
MODERATION_BACKEND: Final = "openai/omni-moderation-latest"
@ -123,14 +123,18 @@ def _generation(client: LoggingClient, creds: LangfuseCreds, *, alias: str, star
return observation
def _output_text(observation: LangfuseObservation) -> str:
def _output_contents(observation: LangfuseObservation) -> tuple[str, ...]:
assert observation.output not in (None, "", [], {}), f"generation output is empty: {observation!r}"
try:
messages: Final = _OUTPUT_MESSAGES.validate_python(observation.output)
except ValidationError:
pytest.fail(f"generation output is not a list of assistant messages: {observation!r}")
assert messages, f"generation output is empty: {observation!r}"
return "\n".join(message.content for message in messages)
return tuple(message.content for message in messages)
def _output_text(observation: LangfuseObservation) -> str:
return "\n".join(_output_contents(observation))
def _input_text(observation: LangfuseObservation) -> str:
@ -204,9 +208,9 @@ class TestOtelV2LangfuseGenerationOutput:
texts: Final = tuple(choice.text.strip() for choice in response.choices)
assert len(texts) == 2 and all(texts), f"/v1/completions returned no text: {response!r}"
output: Final = _output_text(_generation(client, langfuse_creds, alias=alias, started=started))
assert all(text in output for text in texts), (
f"generation output lacks the completion texts {texts!r}: {output!r}"
contents: Final = _output_contents(_generation(client, langfuse_creds, alias=alias, started=started))
assert sorted(content.strip() for content in contents) == sorted(texts), (
f"generation output does not hold one message per completion choice {texts!r}: {contents!r}"
)
@pytest.mark.covers("logging.langfuse.success.logs_spend", exercised_on=["images_generations"])