mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-15 09:12:18 +00:00
test(e2e): include fixture and tracing repairs for qualification
This commit is contained in:
commit
247efdd50d
17 changed files with 222 additions and 66 deletions
|
|
@ -38,7 +38,7 @@ from models import (
|
|||
from proxy_client import ProxyClient
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
GUARDRAIL_BACKEND: Final = "gemini/gemini-2.5-flash"
|
||||
GUARDRAIL_BACKEND: Final = "gemini/gemini-3.8-flash"
|
||||
|
||||
GuardrailMode = Literal["pre_call", "post_call", "during_call", "logging_only"]
|
||||
BlockedWordAction = Literal["BLOCK", "MASK"]
|
||||
|
|
|
|||
|
|
@ -45,7 +45,7 @@ from passthrough_client import PassthroughClient
|
|||
pytestmark = pytest.mark.e2e
|
||||
|
||||
COHERE_BACKEND = "cohere/command-r-08-2024"
|
||||
GEMINI_BACKEND = "gemini/gemini-2.5-flash"
|
||||
GEMINI_BACKEND = "gemini/gemini-3.8-flash"
|
||||
VERTEX_BACKEND: Final = "vertex_ai/gemini-2.5-flash"
|
||||
AZURE_OPENAI_BACKEND: Final = "azure/gpt-5.4-nano"
|
||||
AZURE_OPENAI_API_VERSION: Final = "v1"
|
||||
|
|
@ -243,7 +243,7 @@ _PERSON_SCHEMA: dict[str, object] = {
|
|||
|
||||
OPENAI_CHAT_MODEL: Final = "gpt-5.5"
|
||||
ANTHROPIC_CHAT_MODEL: Final = "claude-haiku-4-5"
|
||||
GEMINI_FLASH_MODEL: Final = "gemini-2.5-flash"
|
||||
GEMINI_FLASH_MODEL: Final = "gemini-3.8-flash"
|
||||
|
||||
CHAT_MODELS: Final = (
|
||||
pytest.param(
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ teardown) instead of relying on a statically configured model, so the check is
|
|||
self-contained and never inherits pricing another suite or a stale config left on
|
||||
the shared proxy. custom-priced-flash sets input/output rates deliberately far
|
||||
above the canonical gemini price; the isolation sibling shares the same
|
||||
gemini/gemini-2.5-flash backend but sets no override. Three behaviors are checked
|
||||
gemini/gemini-3.8-flash backend but sets no override. Three behaviors are checked
|
||||
independently:
|
||||
|
||||
- billing: a real call's logged cost breakdown charges input and output tokens at
|
||||
|
|
@ -37,10 +37,8 @@ from models import (
|
|||
|
||||
pytestmark = pytest.mark.e2e
|
||||
|
||||
BACKEND_MODEL = "gemini/gemini-2.5-flash"
|
||||
BACKEND_MODEL = "gemini/gemini-3.8-flash"
|
||||
GEMINI_API_KEY = "os.environ/GEMINI_API_KEY"
|
||||
# Deliberately ~100x above canonical gemini-2.5-flash (input 3e-7 / output 2.5e-6)
|
||||
# so an override that is ignored or under-applied bills at the base rate and fails.
|
||||
CUSTOM_INPUT_RATE = 5e-05
|
||||
CUSTOM_OUTPUT_RATE = 1e-04
|
||||
|
||||
|
|
@ -78,7 +76,7 @@ def _provision(
|
|||
input_cost_per_token: float | None,
|
||||
output_cost_per_token: float | None,
|
||||
) -> str:
|
||||
"""Register a fresh gemini/gemini-2.5-flash deployment (deleted on teardown) and
|
||||
"""Register a fresh gemini/gemini-3.8-flash deployment (deleted on teardown) and
|
||||
return its model name. With the cost fields set the deployment carries a custom
|
||||
pricing override; with them None it is a plain sibling on the same backend. The
|
||||
marker keeps the name unique so concurrent runs on the shared proxy never
|
||||
|
|
@ -251,7 +249,7 @@ class TestCustomPricing:
|
|||
assert sibling_entry is not None, f"{sibling} absent from /model/info"
|
||||
|
||||
# custom-priced-flash overrides pricing; the sibling shares the same
|
||||
# gemini/gemini-2.5-flash backend but sets no override, so it must keep its
|
||||
# gemini/gemini-3.8-flash backend but sets no override, so it must keep its
|
||||
# own price. Equal rates mean the override leaked into the shared cost map.
|
||||
assert (
|
||||
sibling_entry.model_info.input_cost_per_token
|
||||
|
|
|
|||
|
|
@ -21,7 +21,7 @@ from pydantic import BaseModel
|
|||
|
||||
pytestmark = pytest.mark.e2e
|
||||
|
||||
UPSTREAM_MODEL = "gemini/gemini-2.5-flash"
|
||||
UPSTREAM_MODEL = "gemini/gemini-3.8-flash"
|
||||
|
||||
|
||||
class _GenerateContentPart(BaseModel):
|
||||
|
|
|
|||
|
|
@ -24,7 +24,7 @@ from sdk_clients import SdkClients
|
|||
|
||||
pytestmark = pytest.mark.e2e
|
||||
|
||||
IMAGE_EDIT_BACKEND: Final = "openai/gpt-image-1"
|
||||
IMAGE_EDIT_BACKEND: Final = "openai/gpt-image-2.5-sunburst"
|
||||
|
||||
_TEST_PNG = base64.b64decode(
|
||||
"iVBORw0KGgoAAAANSUhEUgAAAEAAAABACAIAAAAlC+aJAAAAS0lEQVR42u3PMQ0AAAwDoPo3"
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ and verifies the proxy still logged a costed SpendLogs row
|
|||
(call_type="pass_through_endpoint"), correlated by the id the caller was served:
|
||||
the x-litellm-call-id header on gemini, the `msg_...` message id on anthropic.
|
||||
|
||||
Covered: gemini ("gemini-2.5-flash") + anthropic ("claude-haiku-4-5"), streaming +
|
||||
Covered: gemini ("gemini-3.8-flash") + anthropic ("claude-haiku-4-5"), streaming +
|
||||
non-streaming, plus native tool calls. See LLM_TRANSLATION_COVERAGE_MATRIX.md.
|
||||
|
||||
A passthrough call returning non-2xx fails hard (never a skip); once it returns
|
||||
|
|
@ -32,7 +32,7 @@ from passthrough_client import (
|
|||
completed_responses_object,
|
||||
)
|
||||
|
||||
GEMINI_MODEL: Final = "gemini-2.5-flash"
|
||||
GEMINI_MODEL: Final = "gemini-3.8-flash"
|
||||
ANTHROPIC_PASSTHROUGH_MODEL: Final = "claude-haiku-4-5"
|
||||
EMBEDDING_MODEL = "text-embedding-3-small"
|
||||
REALTIME_MODEL = "gpt-realtime-2"
|
||||
|
|
@ -291,7 +291,7 @@ class TestPassthroughModelAllowlist:
|
|||
|
||||
result = client.anthropic_message(key, ANTHROPIC_PASSTHROUGH_MODEL, f"say hi {unique_marker()}")
|
||||
assert result.status_code == 403, (
|
||||
"a key restricted to gemini-2.5-flash must be denied a claude passthrough call, "
|
||||
"a key restricted to gemini-3.8-flash must be denied a claude passthrough call, "
|
||||
f"got {result.status_code}: {result.body[:300]}"
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -57,7 +57,7 @@ OPENAI_VISION_BACKEND: Final = "openai/gpt-4o"
|
|||
ANTHROPIC_BACKEND: Final = "anthropic/claude-haiku-4-5"
|
||||
BEDROCK_CONVERSE_BACKEND: Final = "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0"
|
||||
VERTEX_BACKEND: Final = "vertex_ai/gemini-2.5-flash"
|
||||
GEMINI_BACKEND: Final = "gemini/gemini-2.5-flash"
|
||||
GEMINI_BACKEND: Final = "gemini/gemini-3.8-flash"
|
||||
OPENAI_RESPONSES_BACKEND: Final = "openai/gpt-5.5"
|
||||
INSTRUCTIONS = "You are a helpful assistant"
|
||||
CAT_IMAGE_URL = "https://upload.wikimedia.org/wikipedia/commons/3/3a/Cat03.jpg"
|
||||
|
|
|
|||
|
|
@ -18,10 +18,11 @@ import json
|
|||
import os
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from typing import Callable, Literal
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Callable, Final, Literal
|
||||
|
||||
import pytest
|
||||
from pydantic import BaseModel, ConfigDict, Field, JsonValue, TypeAdapter, ValidationError
|
||||
from pydantic import AliasChoices, BaseModel, ConfigDict, Field, JsonValue, TypeAdapter, ValidationError, field_validator
|
||||
|
||||
from e2e_config import POLL_INTERVAL, POLL_TIMEOUT, settle_propagation
|
||||
from e2e_metadata import step
|
||||
|
|
@ -124,23 +125,39 @@ class CreateGuardrailResponse(BaseModel):
|
|||
guardrail_name: str | None = None
|
||||
|
||||
|
||||
_LANGFUSE_JSON: Final[TypeAdapter[JsonValue]] = TypeAdapter(JsonValue)
|
||||
|
||||
|
||||
class LangfuseObservation(BaseModel):
|
||||
model_config = ConfigDict(extra="ignore", populate_by_name=True)
|
||||
model_config = ConfigDict(extra="ignore", populate_by_name=True, frozen=True)
|
||||
|
||||
id: str
|
||||
trace_id: str | None = Field(default=None, alias="traceId")
|
||||
name: str | None = None
|
||||
type: str | None = None
|
||||
calculated_total_cost: float | None = Field(default=None, alias="calculatedTotalCost")
|
||||
calculated_total_cost: float | None = Field(
|
||||
default=None, validation_alias=AliasChoices("totalCost", "calculatedTotalCost")
|
||||
)
|
||||
level: str | None = None
|
||||
input: object | None = None
|
||||
output: object | None = None
|
||||
metadata: object | None = None
|
||||
input: JsonValue = None
|
||||
output: JsonValue = None
|
||||
metadata: JsonValue = None
|
||||
usage: object | None = None
|
||||
usage_details: object | None = Field(default=None, alias="usageDetails")
|
||||
model: str | None = None
|
||||
|
||||
|
||||
@field_validator("input", "output", "metadata", mode="before")
|
||||
@classmethod
|
||||
def decode_json_fields(cls, value: object) -> JsonValue:
|
||||
if not isinstance(value, str):
|
||||
return _LANGFUSE_JSON.validate_python(value)
|
||||
try:
|
||||
return _LANGFUSE_JSON.validate_json(value)
|
||||
except ValidationError:
|
||||
return value
|
||||
|
||||
|
||||
class LangfuseObservationList(BaseModel):
|
||||
model_config = ConfigDict(extra="ignore")
|
||||
|
||||
|
|
@ -171,12 +188,15 @@ def is_otel_v2_generation(obs: LangfuseObservation, *, key_alias: str) -> bool:
|
|||
|
||||
|
||||
class LangfuseListParams(BaseModel):
|
||||
model_config = ConfigDict(populate_by_name=True)
|
||||
model_config = ConfigDict(populate_by_name=True, frozen=True)
|
||||
|
||||
limit: int = 100
|
||||
trace_id: str | None = Field(default=None, alias="traceId")
|
||||
name: str | None = None
|
||||
from_start_time: str | None = Field(default=None, alias="fromStartTime")
|
||||
from_start_time: str = Field(alias="fromStartTime")
|
||||
to_start_time: str = Field(alias="toStartTime")
|
||||
fields: str = "core,basic,io,metadata,model,usage"
|
||||
expand_metadata: str = Field(default="attributes", alias="expandMetadata")
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
|
|
@ -580,14 +600,16 @@ class LoggingClient:
|
|||
name: str | None = None,
|
||||
from_start_time: str | None = None,
|
||||
) -> list[LangfuseObservation]:
|
||||
result = get(
|
||||
URL(f"{creds.host}/api/public/observations"),
|
||||
now: Final = datetime.now(timezone.utc)
|
||||
result: Final = get(
|
||||
URL(f"{creds.host}/api/public/v2/observations"),
|
||||
headers=creds.auth_headers,
|
||||
params=LangfuseListParams(
|
||||
limit=100,
|
||||
traceId=trace_id,
|
||||
name=name,
|
||||
fromStartTime=from_start_time,
|
||||
fromStartTime=from_start_time or (now - timedelta(minutes=15)).isoformat(),
|
||||
toStartTime=now.isoformat(),
|
||||
),
|
||||
response_type=LangfuseObservationList,
|
||||
timeout=30.0,
|
||||
|
|
|
|||
|
|
@ -79,7 +79,7 @@ def _chain_reaches(span_id: str, root_id: str, trace: JaegerTrace) -> bool:
|
|||
return False
|
||||
|
||||
|
||||
def _assert_complete_trace(traces: CallTraces, *, route: str, genai_span: str) -> None:
|
||||
def assert_complete_trace(traces: CallTraces, *, route: str, genai_span: str) -> None:
|
||||
"""The enforced behavior: the destination holds exactly one call-id-tagged
|
||||
trace for the call, rooted at the SERVER span, with auth/db children and
|
||||
the gen-AI span all connected into that one tree - no dangling parent
|
||||
|
|
@ -112,10 +112,9 @@ def _assert_complete_trace(traces: CallTraces, *, route: str, genai_span: str) -
|
|||
roots = [span for span in trace.spans if not span.references]
|
||||
assert len(roots) == 1, f"expected exactly one root span, got {[s.operation_name for s in roots]}; spans: {names}"
|
||||
root = roots[0]
|
||||
assert root.operation_name == f"POST {route}", (
|
||||
f"the root must be the SERVER span 'POST {route}', got {root.operation_name!r}"
|
||||
)
|
||||
assert root.kind == "server", f"the root span must have kind=server, got {root.kind!r}"
|
||||
request_route: Final = root.tag("http.route") or root.tag("url.path") or root.tag("http.target")
|
||||
assert request_route == route, f"the root must identify route {route!r}, got {request_route!r}"
|
||||
|
||||
assert f"auth {route}" in names, f"auth phase span 'auth {route}' missing; spans: {names}"
|
||||
assert any(name.startswith(DB_SPAN_PREFIX) for name in names), (
|
||||
|
|
@ -133,12 +132,12 @@ def _assert_complete_trace(traces: CallTraces, *, route: str, genai_span: str) -
|
|||
def _poll(otel_reader: OtelReader, *, call_id: str, route: str, genai_span: str) -> CallTraces:
|
||||
return otel_reader.poll_traces_for_call(
|
||||
call_id=call_id,
|
||||
settled_names={f"POST {route}", f"auth {route}", genai_span},
|
||||
settled_names={f"auth {route}", genai_span},
|
||||
settled_prefixes={DB_SPAN_PREFIX},
|
||||
)
|
||||
|
||||
|
||||
def _assert_real_ttft(hits: tuple[JaegerTrace, ...], *, genai_span: str) -> None:
|
||||
def assert_real_ttft(hits: tuple[JaegerTrace, ...], *, genai_span: str) -> None:
|
||||
"""The enforced behavior: the gen-AI span for the attempt that served the
|
||||
stream records a TTFT that is a real measurement - present, numeric,
|
||||
positive, and strictly less than that span's own total duration. A TTFT of
|
||||
|
|
@ -282,7 +281,7 @@ class TestOtelTraceCompleteness:
|
|||
assert outcome.call_id is not None, "success response must carry x-litellm-call-id"
|
||||
|
||||
traces = _poll(otel_reader, call_id=outcome.call_id, route=route, genai_span=f"chat {MODEL}")
|
||||
_assert_complete_trace(traces, route=route, genai_span=f"chat {MODEL}")
|
||||
assert_complete_trace(traces, route=route, genai_span=f"chat {MODEL}")
|
||||
|
||||
@pytest.mark.covers("logging.otel.success.exports_metric", exercised_on=["chat_completions"])
|
||||
@meta(
|
||||
|
|
@ -315,7 +314,7 @@ class TestOtelTraceCompleteness:
|
|||
assert outcome.call_id is not None, "success response must carry x-litellm-call-id"
|
||||
|
||||
hits: Final = _poll(otel_reader, call_id=outcome.call_id, route=route, genai_span=f"chat {MODEL}")
|
||||
_assert_complete_trace(hits, route=route, genai_span=f"chat {MODEL}")
|
||||
assert_complete_trace(hits, route=route, genai_span=f"chat {MODEL}")
|
||||
|
||||
@pytest.mark.covers("logging.otel.success.exports_metric", exercised_on=["messages"])
|
||||
@meta(
|
||||
|
|
@ -333,7 +332,7 @@ class TestOtelTraceCompleteness:
|
|||
"""This test verifies that one successful non-streaming /v1/messages request
|
||||
produces exactly one complete OTEL trace.
|
||||
|
||||
The trace must have a single root span named "POST /v1/messages". The
|
||||
The trace must have a single root SERVER span identifying /v1/messages. The
|
||||
authentication, database, and model-call spans must all belong to
|
||||
the same trace and have valid parent relationships leading back to that root.
|
||||
|
||||
|
|
@ -353,7 +352,7 @@ class TestOtelTraceCompleteness:
|
|||
assert outcome.call_id is not None, "success response must carry x-litellm-call-id"
|
||||
|
||||
traces = _poll(otel_reader, call_id=outcome.call_id, route=route, genai_span=f"chat {MODEL}")
|
||||
_assert_complete_trace(traces, route=route, genai_span=f"chat {MODEL}")
|
||||
assert_complete_trace(traces, route=route, genai_span=f"chat {MODEL}")
|
||||
|
||||
@pytest.mark.covers("logging.otel.success.exports_metric", exercised_on=["responses"])
|
||||
@meta(
|
||||
|
|
@ -392,7 +391,7 @@ class TestOtelTraceCompleteness:
|
|||
|
||||
genai_span = f"chat {CHEAP_OPENAI_MODEL}"
|
||||
traces = _poll(otel_reader, call_id=outcome.call_id, route=route, genai_span=genai_span)
|
||||
_assert_complete_trace(traces, route=route, genai_span=genai_span)
|
||||
assert_complete_trace(traces, route=route, genai_span=genai_span)
|
||||
|
||||
@pytest.mark.covers("logging.otel.stream.exports_metric", exercised_on=["chat_completions"])
|
||||
@meta(
|
||||
|
|
@ -443,7 +442,7 @@ class TestOtelTraceCompleteness:
|
|||
|
||||
genai_span = f"chat {MODEL}"
|
||||
traces = _poll(otel_reader, call_id=outcome.call_id, route=route, genai_span=genai_span)
|
||||
_assert_complete_trace(traces, route=route, genai_span=genai_span)
|
||||
assert_complete_trace(traces, route=route, genai_span=genai_span)
|
||||
|
||||
served = one_served_genai_span(traces.hits[0], genai_span)
|
||||
assert served.tag("litellm.request.streaming") is True, (
|
||||
|
|
@ -500,7 +499,7 @@ class TestOtelTraceCompleteness:
|
|||
|
||||
genai_span = f"chat {MODEL}"
|
||||
traces = _poll(otel_reader, call_id=outcome.call_id, route=route, genai_span=genai_span)
|
||||
_assert_complete_trace(traces, route=route, genai_span=genai_span)
|
||||
assert_complete_trace(traces, route=route, genai_span=genai_span)
|
||||
|
||||
served = one_served_genai_span(traces.hits[0], genai_span)
|
||||
assert served.tag("litellm.request.streaming") is True, (
|
||||
|
|
@ -557,7 +556,7 @@ class TestOtelTraceCompleteness:
|
|||
|
||||
genai_span = f"chat {CHEAP_OPENAI_MODEL}"
|
||||
traces = _poll(otel_reader, call_id=outcome.call_id, route=route, genai_span=genai_span)
|
||||
_assert_complete_trace(traces, route=route, genai_span=genai_span)
|
||||
assert_complete_trace(traces, route=route, genai_span=genai_span)
|
||||
|
||||
one_served_genai_span(traces.hits[0], genai_span)
|
||||
|
||||
|
|
@ -614,7 +613,7 @@ class TestOtelTraceCompleteness:
|
|||
|
||||
genai_span = f"chat {MODEL}"
|
||||
traces = _poll(otel_reader, call_id=outcome.call_id, route=route, genai_span=genai_span)
|
||||
_assert_real_ttft(traces.hits, genai_span=genai_span)
|
||||
assert_real_ttft(traces.hits, genai_span=genai_span)
|
||||
|
||||
@pytest.mark.covers("logging.otel.stream.records_ttft", exercised_on=["messages"])
|
||||
@meta(
|
||||
|
|
@ -661,7 +660,7 @@ class TestOtelTraceCompleteness:
|
|||
|
||||
genai_span = f"chat {MODEL}"
|
||||
traces = _poll(otel_reader, call_id=outcome.call_id, route=route, genai_span=genai_span)
|
||||
_assert_real_ttft(traces.hits, genai_span=genai_span)
|
||||
assert_real_ttft(traces.hits, genai_span=genai_span)
|
||||
|
||||
@pytest.mark.covers("logging.otel.stream.records_ttft", exercised_on=["responses"])
|
||||
@meta(
|
||||
|
|
@ -708,7 +707,7 @@ class TestOtelTraceCompleteness:
|
|||
|
||||
genai_span = f"chat {CHEAP_OPENAI_MODEL}"
|
||||
traces = _poll(otel_reader, call_id=outcome.call_id, route=route, genai_span=genai_span)
|
||||
_assert_real_ttft(traces.hits, genai_span=genai_span)
|
||||
assert_real_ttft(traces.hits, genai_span=genai_span)
|
||||
|
||||
@pytest.mark.covers("logging.otel.failure.exports_metric", exercised_on=["chat_completions"])
|
||||
@meta(
|
||||
|
|
@ -763,7 +762,7 @@ class TestOtelTraceCompleteness:
|
|||
|
||||
genai_span = f"chat {model_name}"
|
||||
traces = _poll(otel_reader, call_id=outcome.call_id, route=route, genai_span=genai_span)
|
||||
_assert_complete_trace(traces, route=route, genai_span=genai_span)
|
||||
assert_complete_trace(traces, route=route, genai_span=genai_span)
|
||||
|
||||
root = next(span for span in traces.hits[0].spans if not span.references)
|
||||
assert str(root.tag("http.status_code")) == "401", (
|
||||
|
|
@ -825,7 +824,7 @@ class TestOtelTraceCompleteness:
|
|||
|
||||
genai_span = f"chat {model_name}"
|
||||
traces = _poll(otel_reader, call_id=outcome.call_id, route=route, genai_span=genai_span)
|
||||
_assert_complete_trace(traces, route=route, genai_span=genai_span)
|
||||
assert_complete_trace(traces, route=route, genai_span=genai_span)
|
||||
|
||||
root = next(span for span in traces.hits[0].spans if not span.references)
|
||||
assert str(root.tag("http.status_code")) == "401", (
|
||||
|
|
|
|||
|
|
@ -418,7 +418,7 @@ class TestBudgetListV1:
|
|||
@pytest.mark.covers("mgmt.budget.list_v1.happy_path")
|
||||
@meta(Subject(domain=Domain.SPEND_BUDGETS, route=Route.BUDGET_MANAGEMENT))
|
||||
def test_refuses_a_sort_field_and_a_parameter_it_does_not_support(self, client: ManagementClient) -> None:
|
||||
assert _list_status(client, BudgetPageParams(sort="budget_duration")) == 400
|
||||
assert _list_status(client, BudgetPageParams(sort="not_a_sort_field")) == 400
|
||||
assert _list_status(client, BudgetPageParams(not_a_parameter="b-1")) == 400
|
||||
|
||||
@pytest.mark.covers("mgmt.budget.list_v1.admin_only")
|
||||
|
|
|
|||
|
|
@ -28,11 +28,10 @@ from dataclasses import dataclass
|
|||
from typing import Final
|
||||
|
||||
import pytest
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
|
||||
from e2e_config import OTEL_QUERY_URL, POLL_INTERVAL, POLL_TIMEOUT
|
||||
from e2e_http import URL, NetworkError, NoBody, Result, Success, get
|
||||
from e2e_metadata import step
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
|
||||
#: OTEL resource service.name the proxy exports under (OTEL_SERVICE_NAME default).
|
||||
JAEGER_SERVICE = "litellm"
|
||||
|
|
@ -118,9 +117,15 @@ def _ticks() -> Iterator[None]:
|
|||
time.sleep(POLL_INTERVAL)
|
||||
|
||||
|
||||
def _settled(trace: JaegerTrace, names: set[str], prefixes: set[str]) -> bool:
|
||||
present = set(trace.span_names())
|
||||
return names.issubset(present) and all(any(name.startswith(prefix) for name in present) for prefix in prefixes)
|
||||
def trace_settled(trace: JaegerTrace, names: set[str], prefixes: set[str]) -> bool:
|
||||
root: Final = root_span(trace)
|
||||
present: Final = frozenset(trace.span_names())
|
||||
return (
|
||||
root is not None
|
||||
and root.kind == "server"
|
||||
and names.issubset(present)
|
||||
and all(any(name.startswith(prefix) for name in present) for prefix in prefixes)
|
||||
)
|
||||
|
||||
|
||||
def root_span(trace: JaegerTrace) -> JaegerSpan | None:
|
||||
|
|
@ -172,7 +177,7 @@ class _Observation:
|
|||
|
||||
def settled(self, names: set[str], prefixes: set[str]) -> bool:
|
||||
hits: Final = self.traces.hits
|
||||
return self.unreachable is None and len(hits) == 1 and not self.missing and _settled(hits[0], names, prefixes)
|
||||
return self.unreachable is None and len(hits) == 1 and not self.missing and trace_settled(hits[0], names, prefixes)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
|
|
|
|||
|
|
@ -55,7 +55,7 @@ from spend_e2e_client import (
|
|||
|
||||
pytestmark = pytest.mark.e2e
|
||||
|
||||
CHAT_MODEL: Final = "gemini-2.5-flash"
|
||||
CHAT_MODEL: Final = "gemini-3.8-flash"
|
||||
MESSAGES_MODEL: Final = "claude-haiku-4-5"
|
||||
RESPONSES_MODEL: Final = "openai-responses-codex"
|
||||
EMBED_MODEL: Final = "openai-text-embedding-3-small"
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@
|
|||
Run against a proxy started with the gateway config. Coverage rationale:
|
||||
SPEND_TRACKING_COVERAGE_MATRIX.md.
|
||||
|
||||
Model names are literals from that config: chat tests hit "gemini-2.5-flash",
|
||||
Model names are literals from that config: chat tests hit "gemini-3.8-flash",
|
||||
embedding tests hit "openai-text-embedding-3-small".
|
||||
|
||||
Every test: fresh scoped key (isolation) -> real provider call -> unwrap (hard
|
||||
|
|
@ -37,7 +37,7 @@ from spend_reconciliation import BACKEND as TRAFFIC_BACKEND
|
|||
|
||||
pytestmark = pytest.mark.e2e
|
||||
|
||||
GEMINI_MODEL = "gemini-2.5-flash"
|
||||
GEMINI_MODEL = "gemini-3.8-flash"
|
||||
CLAUDE_MODEL = "claude-haiku-4-5"
|
||||
CODEX_MODEL = "openai-responses-codex"
|
||||
EMBEDDING_MODEL = "openai-text-embedding-3-small"
|
||||
|
|
@ -780,7 +780,7 @@ def test_spend_calculate_returns_nonzero_cost(client: SpendClient) -> None:
|
|||
GEMINI_MODEL, "estimate the cost of this request"
|
||||
)
|
||||
assert cost > 0, (
|
||||
"/spend/calculate returned 0 for gemini-2.5-flash; "
|
||||
"/spend/calculate returned 0 for gemini-3.8-flash; "
|
||||
"cost map may be missing this model"
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -57,11 +57,9 @@ CONTENT_POLICY_PROMPT = (
|
|||
COOLDOWN_SECONDS = 30.0
|
||||
REPLICA_PROPAGATION_SECONDS = 15.0
|
||||
|
||||
# The smallest-context chat model OpenAI still serves (16385 tokens). A prompt
|
||||
# past that limit comes back as a real `context_length_exceeded` 400, which is
|
||||
# what litellm maps to ContextWindowExceededError.
|
||||
SMALL_CONTEXT_MODEL = "openai/gpt-3.5-turbo"
|
||||
SMALL_CONTEXT_LIMIT_TOKENS = 16385
|
||||
SMALL_CONTEXT_MODEL: Final = "openai/gpt-5.4-nano"
|
||||
# model_prices_and_context_window.json, gpt-5.4-nano, verified 2026-11-02
|
||||
SMALL_CONTEXT_LIMIT_TOKENS: Final = 272000
|
||||
|
||||
|
||||
def oversized_prompt(marker: str) -> str:
|
||||
|
|
@ -107,7 +105,7 @@ def create_timeout_deployment(proxy: ProxyClient, name: str) -> str:
|
|||
|
||||
@step(f"Add a deployment named {{name}} that calls the small-context model {SMALL_CONTEXT_MODEL}")
|
||||
def create_small_context_deployment(proxy: ProxyClient, name: str) -> str:
|
||||
"""Register a deployment on the smallest-context model OpenAI still serves, so an
|
||||
"""Register a deployment with a smaller context window than the backup, so an
|
||||
oversized prompt earns a real context-window refusal from the provider."""
|
||||
return proxy.create_model(name, LiteLLMParamsBody(model=SMALL_CONTEXT_MODEL, api_key=REAL_KEY))
|
||||
|
||||
|
|
@ -263,8 +261,7 @@ def spend_only_request_of(proxy: ProxyClient, spent_key: str) -> None:
|
|||
|
||||
@step(f"Add a deployment named {{name}} that calls {SMALL_CONTEXT_MODEL} and takes all of its group's traffic")
|
||||
def create_always_picked_small_context_deployment(proxy: ProxyClient, name: str) -> str:
|
||||
"""The always-picked half of a retry pair on the smallest-context model OpenAI
|
||||
still serves: it holds all of the model group's shuffle weight, so an oversized
|
||||
"""The always-picked half of a retry pair on the smaller-context model: it holds all of the model group's shuffle weight, so an oversized
|
||||
prompt opens on it and earns a real context-window refusal, which never benches
|
||||
a deployment, so only the retry itself can steer the request off it."""
|
||||
return proxy.register_model(
|
||||
|
|
|
|||
49
tests/e2e_harness/logging/test_logging_client.py
Normal file
49
tests/e2e_harness/logging/test_logging_client.py
Normal file
|
|
@ -0,0 +1,49 @@
|
|||
import json
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
from pydantic import JsonValue
|
||||
|
||||
from logging_client import LangfuseObservation, is_otel_v2_generation, observation_spend
|
||||
|
||||
|
||||
@pytest.mark.parametrize("encoded", [False, True])
|
||||
@pytest.mark.parametrize("cost", [0.0, 0.000023])
|
||||
def test_observation_preserves_v2_content_usage_and_cost(encoded: bool, cost: float) -> None:
|
||||
content: Final[JsonValue] = [{"role": "user", "content": "unique-request-marker"}]
|
||||
output: Final[JsonValue] = [{"role": "assistant", "content": "response"}]
|
||||
metadata: Final[JsonValue] = {
|
||||
"attributes": {
|
||||
"langfuse.observation.type": "generation",
|
||||
"litellm.metadata.user_api_key_alias": "test-key-alias",
|
||||
}
|
||||
}
|
||||
usage: Final[JsonValue] = {"input": 17, "output": 3}
|
||||
observation: Final = LangfuseObservation.model_validate(
|
||||
{
|
||||
"id": "observation",
|
||||
"input": json.dumps(content) if encoded else content,
|
||||
"output": json.dumps(output) if encoded else output,
|
||||
"metadata": json.dumps(metadata) if encoded else metadata,
|
||||
"usageDetails": usage,
|
||||
"totalCost": cost,
|
||||
}
|
||||
)
|
||||
|
||||
assert observation.input == content
|
||||
assert observation.output == output
|
||||
assert observation.usage_details == usage
|
||||
assert observation_spend(observation) == cost
|
||||
assert is_otel_v2_generation(observation, key_alias="test-key-alias")
|
||||
assert not is_otel_v2_generation(observation, key_alias="another-key")
|
||||
|
||||
|
||||
def test_observation_preserves_plain_text_and_legacy_cost() -> None:
|
||||
cost: Final = 0.000031
|
||||
observation: Final = LangfuseObservation.model_validate(
|
||||
{"id": "observation", "input": "plain prompt", "output": "plain reply", "calculatedTotalCost": cost}
|
||||
)
|
||||
|
||||
assert observation.input == "plain prompt"
|
||||
assert observation.output == "plain reply"
|
||||
assert observation_spend(observation) == cost
|
||||
|
|
@ -8,8 +8,21 @@ helper is worth pinning here.
|
|||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
from otel_client import TTFT_TAG, JaegerTrace, one_served_genai_span, served_genai_spans
|
||||
from otel_client import (
|
||||
TTFT_TAG,
|
||||
CallTraces,
|
||||
JaegerReference,
|
||||
JaegerSpan,
|
||||
JaegerTag,
|
||||
JaegerTrace,
|
||||
one_served_genai_span,
|
||||
served_genai_spans,
|
||||
trace_settled,
|
||||
)
|
||||
from test_otel_trace_e2e import assert_complete_trace, assert_real_ttft
|
||||
|
||||
GENAI_SPAN = "chat claude-haiku-4-5"
|
||||
|
||||
|
|
@ -79,3 +92,73 @@ def test_other_operations_are_not_counted() -> None:
|
|||
trace = _trace(_span("chat gpt-5.5", ttft=0.3), _span(GENAI_SPAN, ttft=0.3))
|
||||
|
||||
assert len(served_genai_spans(trace, GENAI_SPAN)) == 1
|
||||
|
||||
|
||||
def _request_trace(
|
||||
*,
|
||||
root_name: str = "POST",
|
||||
root_kind: str = "server",
|
||||
route: str = "/v1/messages",
|
||||
genai_parent: str = "root",
|
||||
ttft: float = 0.2,
|
||||
) -> JaegerTrace:
|
||||
parent: Final = JaegerReference(refType="CHILD_OF", traceID="request", spanID="root")
|
||||
return JaegerTrace(
|
||||
traceID="request",
|
||||
spans=[
|
||||
JaegerSpan(
|
||||
spanID="root",
|
||||
operationName=root_name,
|
||||
tags=[JaegerTag(key="span.kind", value=root_kind), JaegerTag(key="http.target", value=route)],
|
||||
),
|
||||
JaegerSpan(spanID="auth", operationName="auth /v1/messages", references=[parent]),
|
||||
JaegerSpan(spanID="db", operationName="postgres.select tokens", references=[parent]),
|
||||
JaegerSpan(
|
||||
spanID="genai",
|
||||
operationName=GENAI_SPAN,
|
||||
duration=1_000_000,
|
||||
references=[JaegerReference(refType="CHILD_OF", traceID="request", spanID=genai_parent)],
|
||||
tags=[JaegerTag(key="span.kind", value="client"), JaegerTag(key=TTFT_TAG, value=ttft)],
|
||||
),
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("root_name", ("POST", "POST /v1/messages"))
|
||||
def test_connected_request_trace_is_independent_of_server_display_name(root_name: str) -> None:
|
||||
trace: Final = _request_trace(root_name=root_name)
|
||||
assert_complete_trace(CallTraces((trace,), ()), route="/v1/messages", genai_span=GENAI_SPAN)
|
||||
assert_real_ttft((trace,), genai_span=GENAI_SPAN)
|
||||
assert trace_settled(trace, {"auth /v1/messages", GENAI_SPAN}, {"postgres."})
|
||||
|
||||
|
||||
def test_incomplete_trace_cannot_settle_before_the_server_span_arrives() -> None:
|
||||
trace: Final = _request_trace()
|
||||
incomplete: Final = trace.model_copy(update={"spans": trace.spans[1:]})
|
||||
assert not trace_settled(incomplete, {"auth /v1/messages", GENAI_SPAN}, {"postgres."})
|
||||
|
||||
|
||||
@pytest.mark.parametrize("route", ("/v1/responses", ""))
|
||||
def test_request_trace_must_still_identify_the_requested_route(route: str) -> None:
|
||||
trace: Final = _request_trace(root_name="POST /v1/messages", route=route)
|
||||
with pytest.raises(AssertionError, match="route"):
|
||||
assert_complete_trace(CallTraces((trace,), ()), route="/v1/messages", genai_span=GENAI_SPAN)
|
||||
|
||||
|
||||
def test_request_trace_must_still_be_rooted_at_a_server_span() -> None:
|
||||
trace: Final = _request_trace(root_name="POST /v1/messages", root_kind="client")
|
||||
with pytest.raises(AssertionError, match="kind=server"):
|
||||
assert_complete_trace(CallTraces((trace,), ()), route="/v1/messages", genai_span=GENAI_SPAN)
|
||||
|
||||
|
||||
def test_request_trace_must_still_connect_the_model_call_to_the_request() -> None:
|
||||
trace: Final = _request_trace(genai_parent="absent")
|
||||
with pytest.raises(AssertionError, match="orphaned trace"):
|
||||
assert_complete_trace(CallTraces((trace,), ()), route="/v1/messages", genai_span=GENAI_SPAN)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("ttft", (0.0, 1.0))
|
||||
def test_server_name_tolerance_does_not_accept_invalid_first_token_timing(ttft: float) -> None:
|
||||
trace: Final = _request_trace(ttft=ttft)
|
||||
with pytest.raises(AssertionError, match="TTFT"):
|
||||
assert_real_ttft((trace,), genai_span=GENAI_SPAN)
|
||||
|
|
|
|||
|
|
@ -5,5 +5,8 @@
|
|||
addopts = --strict-markers --strict-config -p no:cacheprovider
|
||||
pythonpath = ../e2e ../e2e/batches ../e2e/guardrails ../e2e/logging ../e2e/perf
|
||||
markers =
|
||||
e2e: live assertion modules imported by harness regression tests
|
||||
meta(subject): metadata on live assertion modules imported by harness regression tests
|
||||
otel_tls: TLS coverage in live assertion modules imported by harness regression tests
|
||||
covers(cell_id, *, exercised_on=()): coverage-registry cell(s) a test covers; exercised here only to prove the collector and the JUnit properties read it
|
||||
cli_determinism: drives the real claude CLI for several seconds
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue