mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-12 23:01:41 +00:00
4725cb4661
* test(e2e): cover google-native generateContent framing and prometheus queue time
Adds live coverage for three shipped regressions that had none, all reached
through surfaces a customer drives from Google SDKs and operator dashboards.
The managed google-native route (`/v1beta/models/{model}:generateContent`) had
no harness support at all, so EndpointsClient gains generate_content and
stream_generate_content plus the request body models, and a new suite asserts
the two contracts that broke there: the response carries
x-litellm-response-cost so SDK traffic reconciles against spend (LIT-4076), and
the stream relays single-prefixed SSE frames with no OpenAI [DONE] terminator.
A doubled `data:` prefix, a leaked bytes literal, or the [DONE] sentinel each
fail the stream test; [DONE] absence is only asserted once real content has
arrived, because a first-chunk upstream error legitimately falls back to the
OpenAI error shape and does emit it.
The prometheus test pins litellm_request_queue_time_seconds to an actual
observation on our own key's series rather than to the family merely existing,
which is the distinction the original regression turned on: the histogram stayed
registered while nothing was ever written to it (LIT-2034).
Each assertion was mutation-checked against the live proxy; inverting the
[DONE] expectation, the cost-header expectation, or the metric name fails the
corresponding test.
* refactor(e2e): simplify google native coverage
214 lines
4.5 KiB
Python
214 lines
4.5 KiB
Python
"""Registry row schema: the contract every denominator cell validates against.
|
|
|
|
A cell is one customer-noticeable behavior a single e2e test can assert pass/fail
|
|
on. `module` is the id's segment-1 prefix (eight of them); dashboard rollups can
|
|
split or merge those prefixes. The union is discriminated on `module`, so an LLM
|
|
row cannot carry a guardrail field and vice versa.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from enum import Enum
|
|
from typing import Annotated, Literal
|
|
|
|
from pydantic import BaseModel, ConfigDict, Field, TypeAdapter
|
|
|
|
|
|
class Tier(str, Enum):
|
|
P0 = "P0"
|
|
P1 = "P1"
|
|
P2 = "P2"
|
|
|
|
|
|
class FailBeforeFix(str, Enum):
|
|
proven = "proven"
|
|
unproven = "unproven"
|
|
|
|
|
|
LlmEndpoint = Literal[
|
|
"chat_completions",
|
|
"completions",
|
|
"messages",
|
|
"responses",
|
|
"embeddings",
|
|
"batches",
|
|
"files",
|
|
"rerank",
|
|
"images_generations",
|
|
"images_edits",
|
|
"audio_speech",
|
|
"audio_transcriptions",
|
|
"moderations",
|
|
"realtime",
|
|
"google_native",
|
|
"vector_stores",
|
|
"ocr",
|
|
"bedrock_native",
|
|
]
|
|
|
|
LlmRoute = Literal[
|
|
"anthropic",
|
|
"azure_foundry",
|
|
"azure_openai",
|
|
"bedrock_converse",
|
|
"bedrock_invoke",
|
|
"cohere",
|
|
"gemini",
|
|
"hosted_vllm",
|
|
"openai",
|
|
"together_ai",
|
|
"vertex",
|
|
]
|
|
|
|
LlmCapability = Literal[
|
|
"assume_role",
|
|
"basic",
|
|
"count_tokens",
|
|
"input_validation",
|
|
"long_context_1m",
|
|
"mid_conversation_system",
|
|
"multi_turn",
|
|
"pdf_input",
|
|
"prompt_cache_1h",
|
|
"prompt_cache_5m",
|
|
"service_tier",
|
|
"structured_output",
|
|
"thinking",
|
|
"thinking_with_tool_use",
|
|
"tool_search",
|
|
"tool_use",
|
|
"vision",
|
|
"web_search",
|
|
"web_search_server_tool",
|
|
]
|
|
|
|
|
|
class _Base(BaseModel):
|
|
model_config = ConfigDict(frozen=True, extra="forbid")
|
|
|
|
id: str
|
|
tier: Tier
|
|
assertions: tuple[str, ...]
|
|
source: str
|
|
rationale: str = ""
|
|
fail_before_fix: FailBeforeFix = FailBeforeFix.unproven
|
|
supported: bool = True
|
|
|
|
|
|
class LlmCell(_Base):
|
|
module: Literal["llm"]
|
|
subject_endpoint: LlmEndpoint
|
|
route: LlmRoute
|
|
capability: LlmCapability
|
|
streaming: Literal["stream", "nonstream", "na"]
|
|
|
|
|
|
class MgmtCell(_Base):
|
|
module: Literal["mgmt"]
|
|
surface: Literal["api", "ui"]
|
|
|
|
|
|
class McpCell(_Base):
|
|
module: Literal["mcp"]
|
|
operation: str
|
|
auth_family: Literal["none", "api_key", "bearer", "oauth"]
|
|
|
|
|
|
class ReliabilityCell(_Base):
|
|
module: Literal["reliability"]
|
|
behavior: str
|
|
variant: str
|
|
exercised_on: tuple[str, ...]
|
|
|
|
|
|
class QuotaCell(_Base):
|
|
module: Literal["quota_management"]
|
|
behavior: Literal["ratelimit", "budget", "spend_tracking"]
|
|
variant: str
|
|
exercised_on: tuple[str, ...]
|
|
|
|
|
|
class LoggingCell(_Base):
|
|
module: Literal["logging"]
|
|
event: str
|
|
exercised_on: tuple[str, ...]
|
|
|
|
|
|
class GuardrailCell(_Base):
|
|
module: Literal["guardrail"]
|
|
hook_point: str
|
|
exercised_on: tuple[str, ...]
|
|
|
|
|
|
class OtherCell(_Base):
|
|
module: Literal["other"]
|
|
area: str
|
|
|
|
|
|
Cell = Annotated[
|
|
LlmCell
|
|
| MgmtCell
|
|
| McpCell
|
|
| ReliabilityCell
|
|
| QuotaCell
|
|
| LoggingCell
|
|
| GuardrailCell
|
|
| OtherCell,
|
|
Field(discriminator="module"),
|
|
]
|
|
|
|
CELL_ADAPTER: TypeAdapter[Cell] = TypeAdapter(Cell)
|
|
|
|
CORE_LLM_ENDPOINTS: frozenset[str] = frozenset(
|
|
{
|
|
"chat_completions",
|
|
"messages",
|
|
"responses",
|
|
}
|
|
)
|
|
|
|
PREFIX_ROLLUP: dict[str, str] = {
|
|
"mcp": "MCPs",
|
|
"mgmt": "Management/UI",
|
|
"reliability": "Reliability & Performance",
|
|
"quota_management": "Quota Management",
|
|
"logging": "Logging & Guardrails",
|
|
"guardrail": "Logging & Guardrails",
|
|
"other": "Other",
|
|
}
|
|
|
|
MODULE_ORDER: tuple[str, ...] = (
|
|
"Core LLMs",
|
|
"Non-Core LLMs",
|
|
"MCPs",
|
|
"Management/UI",
|
|
"Reliability & Performance",
|
|
"Quota Management",
|
|
"Logging & Guardrails",
|
|
"Other",
|
|
)
|
|
|
|
LOKI_MODULE_LABELS: dict[str, str] = {
|
|
"Core LLMs": "core_llms",
|
|
"Non-Core LLMs": "non_core_llms",
|
|
"MCPs": "mcp",
|
|
"Management/UI": "management_ui",
|
|
"Reliability & Performance": "reliability_performance",
|
|
"Quota Management": "quota_management",
|
|
"Logging & Guardrails": "logging_guardrails",
|
|
"Other": "other",
|
|
}
|
|
|
|
|
|
def dashboard_module(cell: Cell) -> str:
|
|
"""Return the Grafana/reporting module for a registry cell."""
|
|
if isinstance(cell, LlmCell):
|
|
if cell.subject_endpoint in CORE_LLM_ENDPOINTS:
|
|
return "Core LLMs"
|
|
return "Non-Core LLMs"
|
|
return PREFIX_ROLLUP[cell.module]
|
|
|
|
|
|
def loki_module_label(module: str) -> str:
|
|
"""Return the log-safe Loki label for a dashboard module."""
|
|
return LOKI_MODULE_LABELS[module]
|