mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-26 01:12:21 +00:00
c9e8a04139
* feat(vertex): native batch JSONL passthrough with cost tracking Add a per-request `passthrough=true` multipart field on `POST /v1/files` (and the same kwarg on `litellm.create_file`) that uploads a native Vertex AI batch JSONL to the deployment's GCS bucket unchanged, so rows using `googleSearch` and other Gemini-only features run as written and the output, `groundingMetadata` included, comes back untouched. Passthrough is sticky through the GCS object path (`litellm-vertex-files/passthrough/...`), so batch create and output retrieval inherit it without new state. Native output rows are costed from their `usageMetadata` with the deployment's model and model_info, in the polling and retrieve paths and for the existing global `disable_vertex_batch_output_transformation` flag, which billed $0 before. The proxy requires the target to resolve to vertex_ai deployments only, refuses `passthrough` with a non-batch purpose, a non-default `target_storage`, or pre-call guardrails, and validates native rows on `request` instead of the OpenAI batch keys. * refactor(vertex): keep native batch row pricing inside the Vertex adapter Moves native Vertex batch row detection, response parsing, and per-row pricing from litellm/batches/batch_utils.py into litellm/llms/vertex_ai/batches/transformation.py, so batch_utils only aggregates the rows it gets back. Adds tests/test_litellm/files to the misc unit shard so the new test directory is claimed by a shard. * fix(files): say what a passthrough batch upload takes when a row is not native The missing-key 400 listed bare key names, so an OpenAI-shaped row under passthrough=true read "Each line must be a JSON object with keys request". The batch line shape now carries its own hint, and the passthrough one says a passthrough upload takes native Vertex batch rows with a request key * fix(batches): bill native Vertex embedding batch rows on the native cost path A native Vertex output row whose response holds an embedding was validated as a generateContent response, so the documented tokenCount-only shape counted as a failed row. Price embedding rows from their own usage (promptTokenCount, else tokenCount) with the helper the transformed embeddings path already used, and drop the prompt-details helper nothing calls anymore. * fix(batches): keep modality batch rates on native Vertex embedding rows An embedding row that carries usageMetadata was billed from promptTokenCount alone, so its promptTokensDetails no longer reached the audio, image, and video batch rates the way it did before the native cost path. Run every row with usageMetadata through the Gemini usage parser and keep the flat tokenCount fallback for embedding rows without it. * fix(batches): price native Vertex batch rows by modelVersion under a wildcard deployment A `vertex_ai/*` deployment hands the batch cost path `*` as the deployment model, which no cost map resolves, so every native (passthrough or flag-on) row was billed at $0. A wildcard deployment model now defers to the row's own `modelVersion`, the way the transformed path already prices by the row's `model`. Also moves the native passthrough tests under tests/test_litellm, the tree codecov reads, and covers the raw upload chunking, the embedding output translation, the unpriceable-row path, and the flag-on dispatch. * fix(batches): keep explicit deployment prices for native Vertex rows without a modelVersion Under a wildcard deployment a native batch row that carries no modelVersion (an embedding row, or a generateContent row Vertex returned without one) was billed at $0 even when the deployment's model_info sets explicit batch prices, because the cost calculator was never called. The row now falls back to the wildcard name, which the cost calculator prices from the explicit model_info, and only a row with neither a modelVersion nor a deployment model is billed at $0 with the warning --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
222 lines
4.7 KiB
Python
222 lines
4.7 KiB
Python
"""Registry row schema: the contract every denominator cell validates against.
|
|
|
|
A cell is one customer-noticeable behavior a single e2e test can assert pass/fail
|
|
on. `module` is the id's segment-1 prefix (eight of them); dashboard rollups can
|
|
split or merge those prefixes. The union is discriminated on `module`, so an LLM
|
|
row cannot carry a guardrail field and vice versa.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from enum import Enum
|
|
from typing import Annotated, Literal
|
|
|
|
from pydantic import BaseModel, ConfigDict, Field, TypeAdapter
|
|
|
|
|
|
class Tier(str, Enum):
|
|
P0 = "P0"
|
|
P1 = "P1"
|
|
P2 = "P2"
|
|
|
|
|
|
class FailBeforeFix(str, Enum):
|
|
proven = "proven"
|
|
unproven = "unproven"
|
|
|
|
|
|
LlmEndpoint = Literal[
|
|
"chat_completions",
|
|
"completions",
|
|
"messages",
|
|
"responses",
|
|
"embeddings",
|
|
"batches",
|
|
"files",
|
|
"rerank",
|
|
"images_generations",
|
|
"images_edits",
|
|
"audio_speech",
|
|
"audio_transcriptions",
|
|
"moderations",
|
|
"realtime",
|
|
"google_native",
|
|
"vector_stores",
|
|
"ocr",
|
|
"bedrock_native",
|
|
]
|
|
|
|
LlmRoute = Literal[
|
|
"anthropic",
|
|
"azure_foundry",
|
|
"azure_openai",
|
|
"bedrock_converse",
|
|
"bedrock_invoke",
|
|
"cohere",
|
|
"gemini",
|
|
"hosted_vllm",
|
|
"openai",
|
|
"together_ai",
|
|
"vertex",
|
|
"xiaomi_mimo",
|
|
]
|
|
|
|
LlmCapability = Literal[
|
|
"assume_role",
|
|
"basic",
|
|
"batch_deployment",
|
|
"blank_s3_env",
|
|
"count_tokens",
|
|
"govcloud_partition",
|
|
"split_s3_credentials",
|
|
"input_validation",
|
|
"long_context_1m",
|
|
"mid_conversation_system",
|
|
"multi_turn",
|
|
"native_passthrough",
|
|
"pdf_input",
|
|
"prompt_cache_1h",
|
|
"prompt_cache_5m",
|
|
"response_headers",
|
|
"service_tier",
|
|
"structured_output",
|
|
"thinking",
|
|
"thinking_with_tool_use",
|
|
"tool_search",
|
|
"tool_search_history",
|
|
"tool_use",
|
|
"vision",
|
|
"web_search",
|
|
"web_search_server_tool",
|
|
]
|
|
|
|
|
|
class _Base(BaseModel):
|
|
model_config = ConfigDict(frozen=True, extra="forbid")
|
|
|
|
id: str
|
|
tier: Tier
|
|
assertions: tuple[str, ...]
|
|
source: str
|
|
rationale: str = ""
|
|
fail_before_fix: FailBeforeFix = FailBeforeFix.unproven
|
|
supported: bool = True
|
|
|
|
|
|
class LlmCell(_Base):
|
|
module: Literal["llm"]
|
|
subject_endpoint: LlmEndpoint
|
|
route: LlmRoute
|
|
capability: LlmCapability
|
|
streaming: Literal["stream", "nonstream", "na"]
|
|
|
|
|
|
class MgmtCell(_Base):
|
|
module: Literal["mgmt"]
|
|
surface: Literal["api", "ui"]
|
|
|
|
|
|
class McpCell(_Base):
|
|
module: Literal["mcp"]
|
|
operation: str
|
|
auth_family: Literal["none", "api_key", "bearer", "oauth"]
|
|
|
|
|
|
class ReliabilityCell(_Base):
|
|
module: Literal["reliability"]
|
|
behavior: str
|
|
variant: str
|
|
exercised_on: tuple[str, ...]
|
|
|
|
|
|
class QuotaCell(_Base):
|
|
module: Literal["quota_management"]
|
|
behavior: Literal["ratelimit", "budget", "spend_tracking"]
|
|
variant: str
|
|
exercised_on: tuple[str, ...]
|
|
|
|
|
|
class LoggingCell(_Base):
|
|
module: Literal["logging"]
|
|
event: str
|
|
exercised_on: tuple[str, ...]
|
|
|
|
|
|
class GuardrailCell(_Base):
|
|
module: Literal["guardrail"]
|
|
hook_point: str
|
|
exercised_on: tuple[str, ...]
|
|
|
|
|
|
class OtherCell(_Base):
|
|
module: Literal["other"]
|
|
area: str
|
|
|
|
|
|
Cell = Annotated[
|
|
LlmCell
|
|
| MgmtCell
|
|
| McpCell
|
|
| ReliabilityCell
|
|
| QuotaCell
|
|
| LoggingCell
|
|
| GuardrailCell
|
|
| OtherCell,
|
|
Field(discriminator="module"),
|
|
]
|
|
|
|
CELL_ADAPTER: TypeAdapter[Cell] = TypeAdapter(Cell)
|
|
|
|
CORE_LLM_ENDPOINTS: frozenset[str] = frozenset(
|
|
{
|
|
"chat_completions",
|
|
"messages",
|
|
"responses",
|
|
}
|
|
)
|
|
|
|
PREFIX_ROLLUP: dict[str, str] = {
|
|
"mcp": "MCPs",
|
|
"mgmt": "Management/UI",
|
|
"reliability": "Reliability & Performance",
|
|
"quota_management": "Quota Management",
|
|
"logging": "Logging & Guardrails",
|
|
"guardrail": "Logging & Guardrails",
|
|
"other": "Other",
|
|
}
|
|
|
|
MODULE_ORDER: tuple[str, ...] = (
|
|
"Core LLMs",
|
|
"Non-Core LLMs",
|
|
"MCPs",
|
|
"Management/UI",
|
|
"Reliability & Performance",
|
|
"Quota Management",
|
|
"Logging & Guardrails",
|
|
"Other",
|
|
)
|
|
|
|
LOKI_MODULE_LABELS: dict[str, str] = {
|
|
"Core LLMs": "core_llms",
|
|
"Non-Core LLMs": "non_core_llms",
|
|
"MCPs": "mcp",
|
|
"Management/UI": "management_ui",
|
|
"Reliability & Performance": "reliability_performance",
|
|
"Quota Management": "quota_management",
|
|
"Logging & Guardrails": "logging_guardrails",
|
|
"Other": "other",
|
|
}
|
|
|
|
|
|
def dashboard_module(cell: Cell) -> str:
|
|
"""Return the Grafana/reporting module for a registry cell."""
|
|
if isinstance(cell, LlmCell):
|
|
if cell.subject_endpoint in CORE_LLM_ENDPOINTS:
|
|
return "Core LLMs"
|
|
return "Non-Core LLMs"
|
|
return PREFIX_ROLLUP[cell.module]
|
|
|
|
|
|
def loki_module_label(module: str) -> str:
|
|
"""Return the log-safe Loki label for a dashboard module."""
|
|
return LOKI_MODULE_LABELS[module]
|