scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-05 08:07:05 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 137

Merge remote-tracking branch 'origin/litellm_internal_staging' into litellm_bedrock_guardrail_stream_audit

Browse source
This commit is contained in:
mateo-berri 2026-08-29 21:49:13 -07:00
commit 60296cb540
825 changed files with 10254 additions and 5684 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

26
basedpyright-code-budget.json
View file

@ -1,9 +1,9 @@
{
"reportAny": {
"limit": 17270
"limit": 16389
},
"reportArgumentType": {
"limit": 2538
"limit": 2229
},
"reportAssignmentType": {
"limit": 319
@ -24,7 +24,7 @@
"limit": 19
},
"reportExplicitAny": {
"limit": 5485
"limit": 5242
},
"reportFunctionMemberAccess": {
"limit": 7
@ -54,10 +54,10 @@
"limit": 0
},
"reportMissingParameterType": {
"limit": 5658
"limit": 5614
},
"reportMissingTypeArgument": {
"limit": 15425
"limit": 15356
},
"reportMissingTypeStubs": {
"limit": 40
@ -72,7 +72,7 @@
"limit": 0
},
"reportOptionalMemberAccess": {
"limit": 1055
"limit": 0
},
"reportOptionalOperand": {
"limit": 0
@ -90,7 +90,7 @@
"limit": 8
},
"reportReturnType": {
"limit": 213
"limit": 181
},
"reportTypedDictNotRequiredAccess": {
"limit": 25
@ -99,25 +99,25 @@
"limit": 0
},
"reportUnknownArgumentType": {
"limit": 44526
"limit": 44389
},
"reportUnknownLambdaType": {
"limit": 109
},
"reportUnknownMemberType": {
"limit": 38721
"limit": 38500
},
"reportUnknownParameterType": {
"limit": 19778
"limit": 19673
},
"reportUnknownVariableType": {
"limit": 30290
"limit": 30092
},
"reportUnnecessaryCast": {
"limit": 117
"limit": 111
},
"reportUnnecessaryComparison": {
"limit": 697
"limit": 695
},
"reportUnnecessaryContains": {
"limit": 5

22
enterprise/litellm_enterprise/proxy/audit_logging_endpoints.py
View file

@ -7,7 +7,7 @@ GET - /audit/{id} - Get audit log by id
GET - /audit - Get all audit logs
"""
from typing import TYPE_CHECKING, Final, Optional
from typing import TYPE_CHECKING, Final
#### AUDIT LOGGING ####
from fastapi import APIRouter, Depends, HTTPException, Query
@ -58,33 +58,33 @@ async def get_audit_logs(
page: int = Query(1, ge=1),
page_size: int = Query(10, ge=1, le=100),
# Filter parameters
changed_by: Optional[str] = Query(
changed_by: str | None = Query(
None, description="Filter by user or system that performed the action"
),
changed_by_api_key: Optional[str] = Query(
changed_by_api_key: str | None = Query(
None, description="Filter by API key hash that performed the action"
),
action: Optional[str] = Query(
action: str | None = Query(
None, description="Filter by action type (create, update, delete)"
),
table_name: Optional[str] = Query(
table_name: str | None = Query(
None, description="Filter by table name that was modified"
),
object_id: Optional[str] = Query(
object_id: str | None = Query(
None, description="Filter by ID of the object that was modified"
),
start_date: Optional[str] = Query(None, description="Filter logs after this date"),
end_date: Optional[str] = Query(None, description="Filter logs before this date"),
object_team_id: Optional[str] = Query(
start_date: str | None = Query(None, description="Filter logs after this date"),
end_date: str | None = Query(None, description="Filter logs before this date"),
object_team_id: str | None = Query(
None,
description="Filter by team_id present in before_value or updated_values JSON (PostgreSQL only)",
),
object_key_hash: Optional[str] = Query(
object_key_hash: str | None = Query(
None,
description="Filter by token (key hash) present in before_value or updated_values JSON (PostgreSQL only)",
),
# Sorting parameters
sort_by: Optional[str] = Query(
sort_by: str | None = Query(
None,
description="Column to sort by (e.g. 'updated_at', 'action', 'table_name')",
),

148
helm/litellm/templates/ingress.yaml
View file

@ -5,6 +5,41 @@
{{- $gatewayPort := .Values.gateway.service.port -}}
{{- $backendPort := .Values.backend.service.port -}}
{{- $uiPort := .Values.ui.service.port -}}
{{/*
Backends addressable from ingress.extraPaths, keyed by the `service` field.
*/}}
{{- $extraPathBackends := dict
"gateway" (dict "name" $gatewayName "port" $gatewayPort)
"backend" (dict "name" $backendName "port" $backendPort)
"ui" (dict "name" $uiName "port" $uiPort)
-}}
{{/*
UI paths (Next.js static export).
/ui/* is where the SPA serves its login + dashboard routes (e.g. /ui/login).
Without it, /ui/* falls into the catch-all → backend → 404.
The App Router (output: "export", basePath: "") emits the RSC/flight payload
for every route as a ROOT-level <route>.txt (/index.txt, /teams.txt,
/__next._tree.txt, ...). The client router fetches these on every soft
navigation / prefetch as <route>.txt?_rsc=<hash> (the query string is
irrelevant to path matching). They are not under /ui, /_next, or
/litellm-asset-prefix, so without /*.txt they fall to the backend catch-all
→ 404 → client-side navigation never settles and the login flow spins in an
infinite redirect loop (/ ⇄ /ui/login). ui/nginx.conf already serves *.txt
from the export; the rule only routes the request to it. Needs an ingress
controller whose ImplementationSpecific path is a wildcard pattern
(AWS ALB: `*` = 0+ chars); this chart targets the AWS Load Balancer
Controller.
*/}}
{{- $uiPaths := list
(dict "path" "/" "pathType" "Exact")
(dict "path" "/favicon.ico" "pathType" "Exact")
(dict "path" "/litellm-asset-prefix" "pathType" "Prefix")
(dict "path" "/_next" "pathType" "Prefix")
(dict "path" "/ui" "pathType" "Prefix")
(dict "path" "/*.txt" "pathType" "ImplementationSpecific")
-}}
{{/*
Gateway data-plane prefixes — must mirror gateway/routes/allowlist.py.
Versioned paths are listed explicitly to avoid routing management routes
@ -39,6 +74,21 @@
routes at startup -> 404. So /test is rendered as a standalone Exact path
and /test/* falls through to the backend catch-all.
*/}}
{{/*
Every "<path>|<pathType>" this template renders on its own. An
ingress.extraPaths entry that repeats one of these is rejected: duplicates
in a single rule are resolved by position or by controller-specific tie
breaking, so the operator entry could take over a built-in route (an entry
at "/" Prefix would swallow the whole backend management API) instead of
adding to it.
*/}}
{{- $builtinPathKeys := list "/test|Exact" "/|Prefix" -}}
{{- range $uiPaths }}
{{- $builtinPathKeys = append $builtinPathKeys (printf "%s|%s" .path .pathType) }}
{{- end }}
{{- range $gatewayPrefixes }}
{{- $builtinPathKeys = append $builtinPathKeys (printf "%s|Prefix" .) }}
{{- end }}
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
@ -64,65 +114,15 @@ spec:
http:
paths:
# --- UI (Next.js static export) ---
- path: /
pathType: Exact
backend:
service:
name: {{ $uiName }}
port:
number: {{ $uiPort }}
- path: /favicon.ico
pathType: Exact
backend:
service:
name: {{ $uiName }}
port:
number: {{ $uiPort }}
- path: /litellm-asset-prefix
pathType: Prefix
backend:
service:
name: {{ $uiName }}
port:
number: {{ $uiPort }}
- path: /_next
pathType: Prefix
backend:
service:
name: {{ $uiName }}
port:
number: {{ $uiPort }}
# /ui/* is where the Next.js SPA serves its login + dashboard
# routes (e.g. /ui/login). Without this, /ui/* falls into the
# catch-all → backend → 404.
- path: /ui
pathType: Prefix
backend:
service:
name: {{ $uiName }}
port:
number: {{ $uiPort }}
# Next.js App Router (output: "export", basePath: "") emits the
# RSC/flight payload for every route as a ROOT-level <route>.txt
# (/index.txt, /teams.txt, /__next._tree.txt, ...). The client
# router fetches these on every soft navigation / prefetch as
# <route>.txt?_rsc=<hash> (the query string is irrelevant to path
# matching). They are not under /ui, /_next, or
# /litellm-asset-prefix, so without this rule they fall to the
# backend catch-all → 404 → client-side navigation never settles
# and the login flow spins in an infinite redirect loop
# (/ ⇄ /ui/login). ui/nginx.conf already serves *.txt from the
# export; this rule only routes the request to it. Needs an
# ingress controller whose ImplementationSpecific path is a
# wildcard pattern (AWS ALB: `*` = 0+ chars); this chart targets
# the AWS Load Balancer Controller.
- path: /*.txt
pathType: ImplementationSpecific
{{- range $uiPaths }}
- path: {{ .path }}
pathType: {{ .pathType }}
backend:
service:
name: {{ $uiName }}
port:
number: {{ $uiPort }}
{{- end }}
# --- Gateway data plane ---
# Exact /test only (see the $gatewayPrefixes comment above);
# /test/* MCP management endpoints fall to the backend catch-all.
@ -142,6 +142,46 @@ spec:
port:
number: {{ $gatewayPort }}
{{- end }}
{{- /*
--- Operator-supplied extra paths (ingress.extraPaths) ---
Rendered after every built-in path so an entry can never take
precedence over a default, and before the backend catch-all.
Position only decides the match on controllers that honour manifest
order: the AWS Load Balancer Controller this chart targets sorts
Exact paths first and Prefix paths longest-first, but keeps
ImplementationSpecific paths in manifest order, which is what the
/*.txt rule above already depends on.
*/}}
{{- range $idx, $extra := .Values.ingress.extraPaths }}
{{- if not (kindIs "map" $extra) }}
{{- fail (printf "ingress.extraPaths[%d]: each entry must be a mapping with a 'path' key" $idx) }}
{{- end }}
{{- if not $extra.path }}
{{- fail (printf "ingress.extraPaths[%d]: 'path' is required" $idx) }}
{{- end }}
{{- $service := $extra.service | default "gateway" }}
{{- $target := get $extraPathBackends $service }}
{{- if not $target }}
{{- fail (printf "ingress.extraPaths[%d] (path %s): unknown service %q, expected one of backend, gateway, ui" $idx $extra.path $service) }}
{{- end }}
{{- $pathType := $extra.pathType | default "Prefix" }}
{{- if not (has $pathType (list "Prefix" "Exact" "ImplementationSpecific")) }}
{{- fail (printf "ingress.extraPaths[%d] (path %s): unknown pathType %q, expected one of Exact, ImplementationSpecific, Prefix" $idx $extra.path $pathType) }}
{{- end }}
{{- if eq $extra.path "/" }}
{{- fail (printf "ingress.extraPaths[%d]: path / is already routed in both directions, Exact to ui and Prefix to backend, so no pathType leaves a request for an entry here to capture" $idx) }}
{{- end }}
{{- if has (printf "%s|%s" $extra.path $pathType) $builtinPathKeys }}
{{- fail (printf "ingress.extraPaths[%d]: path %s with pathType %s is already routed by this chart, and a duplicate would take it over rather than add to it" $idx $extra.path $pathType) }}
{{- end }}
- path: {{ $extra.path | quote }}
pathType: {{ $pathType }}
backend:
service:
name: {{ $target.name }}
port:
number: {{ $target.port }}
{{- end }}
# --- Catch-all → backend (management API: /key/*, /user/*, /team/*, ...) ---
- path: /
pathType: Prefix

317
helm/litellm/tests/ingress_extra_paths_tests.yaml Normal file
View file

@ -0,0 +1,317 @@
suite: test ingress.extraPaths
templates:
- ingress.yaml
values:
- ./values/required.yaml
tests:
- it: renders nothing extra between the built-in gateway prefixes and the backend catch-all when unset
set:
ingress.enabled: true
asserts:
- equal:
path: spec.rules[0].http.paths[-1]
value:
path: /
pathType: Prefix
backend:
service:
name: RELEASE-NAME-litellm-backend
port:
number: 4001
- equal:
path: spec.rules[0].http.paths[-2]
value:
path: /metrics
pathType: Prefix
backend:
service:
name: RELEASE-NAME-litellm-gateway
port:
number: 4000
- it: routes an extra path to the gateway by default, immediately before the backend catch-all
set:
ingress.enabled: true
ingress.extraPaths:
- path: /watsonx
asserts:
- equal:
path: spec.rules[0].http.paths[-2]
value:
path: /watsonx
pathType: Prefix
backend:
service:
name: RELEASE-NAME-litellm-gateway
port:
number: 4000
- equal:
path: spec.rules[0].http.paths[-1]
value:
path: /
pathType: Prefix
backend:
service:
name: RELEASE-NAME-litellm-backend
port:
number: 4001
- it: keeps every built-in path when extra paths are supplied
set:
ingress.enabled: true
ingress.extraPaths:
- path: /watsonx
asserts:
- contains:
path: spec.rules[0].http.paths
content:
path: /
pathType: Exact
backend:
service:
name: RELEASE-NAME-litellm-ui
port:
number: 3000
- contains:
path: spec.rules[0].http.paths
content:
path: /ui
pathType: Prefix
backend:
service:
name: RELEASE-NAME-litellm-ui
port:
number: 3000
- contains:
path: spec.rules[0].http.paths
content:
path: /test
pathType: Exact
backend:
service:
name: RELEASE-NAME-litellm-gateway
port:
number: 4000
- contains:
path: spec.rules[0].http.paths
content:
path: /v1/chat
pathType: Prefix
backend:
service:
name: RELEASE-NAME-litellm-gateway
port:
number: 4000
- contains:
path: spec.rules[0].http.paths
content:
path: /vertex_ai
pathType: Prefix
backend:
service:
name: RELEASE-NAME-litellm-gateway
port:
number: 4000
- it: renders every entry in order and honours the service and pathType selectors
set:
ingress.enabled: true
ingress.extraPaths:
- path: /watsonx
service: gateway
- path: /my-passthrough
pathType: Exact
service: backend
- path: /brand.txt
pathType: ImplementationSpecific
service: ui
asserts:
- equal:
path: spec.rules[0].http.paths[-4]
value:
path: /watsonx
pathType: Prefix
backend:
service:
name: RELEASE-NAME-litellm-gateway
port:
number: 4000
- equal:
path: spec.rules[0].http.paths[-3]
value:
path: /my-passthrough
pathType: Exact
backend:
service:
name: RELEASE-NAME-litellm-backend
port:
number: 4001
- equal:
path: spec.rules[0].http.paths[-2]
value:
path: /brand.txt
pathType: ImplementationSpecific
backend:
service:
name: RELEASE-NAME-litellm-ui
port:
number: 3000
- it: addresses the component services by their configured ports
set:
ingress.enabled: true
gateway.service.port: 8000
backend.service.port: 8001
ui.service.port: 8080
ingress.extraPaths:
- path: /watsonx
- path: /my-passthrough
service: backend
- path: /brand.txt
service: ui
asserts:
- equal:
path: spec.rules[0].http.paths[-4].backend.service.port.number
value: 8000
- equal:
path: spec.rules[0].http.paths[-3].backend.service.port.number
value: 8001
- equal:
path: spec.rules[0].http.paths[-2].backend.service.port.number
value: 8080
- it: rejects an entry naming a service the chart does not deploy
set:
ingress.enabled: true
ingress.extraPaths:
- path: /watsonx
service: proxy
asserts:
- failedTemplate:
errorMessage: 'ingress.extraPaths[0] (path /watsonx): unknown service "proxy", expected one of backend, gateway, ui'
- it: rejects an entry whose pathType is not a kubernetes pathType
set:
ingress.enabled: true
ingress.extraPaths:
- path: /watsonx
pathType: prefix
asserts:
- failedTemplate:
errorMessage: 'ingress.extraPaths[0] (path /watsonx): unknown pathType "prefix", expected one of Exact, ImplementationSpecific, Prefix'
- it: rejects an entry with no path
set:
ingress.enabled: true
ingress.extraPaths:
- service: gateway
asserts:
- failedTemplate:
errorMessage: "ingress.extraPaths[0]: 'path' is required"
- it: rejects a root entry that would take over the backend catch-all
set:
ingress.enabled: true
ingress.extraPaths:
- path: /
service: gateway
asserts:
- failedTemplate:
errorMessage: "ingress.extraPaths[0]: path / is already routed in both directions, Exact to ui and Prefix to backend, so no pathType leaves a request for an entry here to capture"
- it: rejects a root entry that would take over the UI root
set:
ingress.enabled: true
ingress.extraPaths:
- path: /
pathType: Exact
service: gateway
asserts:
- failedTemplate:
errorMessage: "ingress.extraPaths[0]: path / is already routed in both directions, Exact to ui and Prefix to backend, so no pathType leaves a request for an entry here to capture"
# A root ImplementationSpecific entry duplicates no built-in pair, so the
# duplicate check alone would admit it. It is still dead: the built-in
# Exact / sorts ahead of it on the AWS Load Balancer Controller and claims
# the only request its pattern matches, so it renders and never routes.
- it: rejects a root entry that would render but never match
set:
ingress.enabled: true
ingress.extraPaths:
- path: /
pathType: ImplementationSpecific
service: gateway
asserts:
- failedTemplate:
errorMessage: "ingress.extraPaths[0]: path / is already routed in both directions, Exact to ui and Prefix to backend, so no pathType leaves a request for an entry here to capture"
- it: rejects an entry that would take over a UI prefix
set:
ingress.enabled: true
ingress.extraPaths:
- path: /ui
service: gateway
asserts:
- failedTemplate:
errorMessage: "ingress.extraPaths[0]: path /ui with pathType Prefix is already routed by this chart, and a duplicate would take it over rather than add to it"
- it: rejects an entry that would take over the UI RSC payload rule
set:
ingress.enabled: true
ingress.extraPaths:
- path: /*.txt
pathType: ImplementationSpecific
service: backend
asserts:
- failedTemplate:
errorMessage: "ingress.extraPaths[0]: path /*.txt with pathType ImplementationSpecific is already routed by this chart, and a duplicate would take it over rather than add to it"
- it: rejects an entry that would take over a gateway data-plane prefix
set:
ingress.enabled: true
ingress.extraPaths:
- path: /v1/chat
service: backend
asserts:
- failedTemplate:
errorMessage: "ingress.extraPaths[0]: path /v1/chat with pathType Prefix is already routed by this chart, and a duplicate would take it over rather than add to it"
- it: rejects an entry that would take over the exact /test route
set:
ingress.enabled: true
ingress.extraPaths:
- path: /test
pathType: Exact
service: backend
asserts:
- failedTemplate:
errorMessage: "ingress.extraPaths[0]: path /test with pathType Exact is already routed by this chart, and a duplicate would take it over rather than add to it"
- it: allows a built-in path under a different pathType, which is a distinct rule
set:
ingress.enabled: true
ingress.extraPaths:
- path: /ui
pathType: Exact
service: ui
asserts:
- equal:
path: spec.rules[0].http.paths[-2]
value:
path: /ui
pathType: Exact
backend:
service:
name: RELEASE-NAME-litellm-ui
port:
number: 3000
- it: rejects a bare string entry instead of failing on template internals
set:
ingress.enabled: true
ingress.extraPaths:
- /watsonx
asserts:
- failedTemplate:
errorMessage: "ingress.extraPaths[0]: each entry must be a mapping with a 'path' key"

21
helm/litellm/values.yaml
View file

@ -13,6 +13,27 @@ ingress:
annotations: {}
host: "" # optional; if set, becomes the rule's host
tls: []
# Extra HTTP paths appended to the ingress rule. Additive: every built-in
# UI / gateway / backend path is still rendered, these entries are placed
# after them and before the backend catch-all, and an entry that repeats a
# path the chart already routes is rejected at render time rather than
# silently taking it over.
#
# The chart's built-in gateway prefix list is a snapshot of the data-plane
# surface at release time. Use extraPaths for passthrough routes it does not
# cover: a provider prefix added upstream after this chart version, or a
# custom general_settings.pass_through_endpoints route.
#
# path required; the HTTP path to route
# service which component serves it: gateway (default), backend, or ui
# pathType Prefix (default), Exact, or ImplementationSpecific
#
# The target component only answers paths its own route allowlist keeps, so
# a path here still has to be one that component serves.
extraPaths: []
# - path: /watsonx
# pathType: Prefix
# service: gateway
# Per-component ServiceAccounts for gateway, backend, and ui.
#

20
litellm/_lazy_imports.py
View file

@ -17,7 +17,7 @@ until they're actually needed.
import importlib
import sys
from collections.abc import Callable
from collections.abc import Callable, Mapping
from types import ModuleType
from typing import TYPE_CHECKING, Any, Final, cast
@ -57,10 +57,11 @@ from ._lazy_imports_registry import (
)
if TYPE_CHECKING:
import httpx
from tiktoken import Encoding
def get_litellm_globals() -> dict:
def get_litellm_globals() -> dict[str, object]:
"""
Get the globals dictionary of the litellm module.
@ -70,7 +71,7 @@ def get_litellm_globals() -> dict:
return sys.modules["litellm"].__dict__
def _get_utils_globals() -> dict:
def _get_utils_globals() -> dict[str, object]:
"""
Get the globals dictionary of the utils module.
@ -80,6 +81,11 @@ def _get_utils_globals() -> dict:
return sys.modules["litellm.utils"].__dict__
def _get_module_level_client_timeout(litellm_globals: Mapping[str, Any]) -> "float | httpx.Timeout | None":
"""Read the configured `litellm.request_timeout` used for the module level http clients."""
return litellm_globals.get("request_timeout")
# These are special lazy loaders for things that are used internally
# They're separate from the main lazy import system because they have specific use cases
@ -435,8 +441,8 @@ def _lazy_import_http_handlers(name: str) -> object:
from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
# Get timeout from module config (if set)
timeout = _globals.get("request_timeout")
params: Final = {"timeout": timeout, "client_alias": "module level aclient"}
async_timeout: Final = _get_module_level_client_timeout(_globals)
params: Final = {"timeout": async_timeout, "client_alias": "module level aclient"}
# Create the client instance
provider_id: Final = cast(Any, "litellm_module_level_client")
@ -453,8 +459,8 @@ def _lazy_import_http_handlers(name: str) -> object:
# Create a sync HTTP client
from litellm.llms.custom_httpx.http_handler import HTTPHandler
timeout = _globals.get("request_timeout")
sync_client: Final = HTTPHandler(timeout=timeout)
sync_timeout: Final = _get_module_level_client_timeout(_globals)
sync_client: Final = HTTPHandler(timeout=sync_timeout)
# Cache it
_globals["module_level_client"] = sync_client

85
litellm/caching/valkey_semantic_cache.py
View file

@ -17,6 +17,7 @@ RedisSemanticCache since those are backend agnostic.
import asyncio
import hashlib
import os
from collections.abc import Mapping, Sequence
from dataclasses import dataclass
from typing import Any, Final
@ -64,7 +65,7 @@ class ValkeySemanticCache(RedisSemanticCache):
async_client: AsyncRedis | None = None,
embedding_max_input_tokens: int | None = None,
embedding_timeout: float | None = None,
**kwargs: Any,
**kwargs: object,
):
if similarity_threshold is None:
raise ValueError("similarity_threshold must be provided, passed None")
@ -87,11 +88,13 @@ class ValkeySemanticCache(RedisSemanticCache):
self.key_prefix = f"{self.index_name}:"
self._index_dim: int | None = None
resolved_url = None
if sync_client is None or async_client is None:
resolved_url = redis_url or self._build_valkey_url(host, port, password, ssl)
self.sync_client = sync_client if sync_client is not None else Redis.from_url(resolved_url)
self.async_client = async_client if async_client is not None else AsyncRedis.from_url(resolved_url)
if sync_client is not None and async_client is not None:
self.sync_client = sync_client
self.async_client = async_client
else:
resolved_url: Final = redis_url or self._build_valkey_url(host, port, password, ssl)
self.sync_client = sync_client if sync_client is not None else Redis.from_url(resolved_url)
self.async_client = async_client if async_client is not None else AsyncRedis.from_url(resolved_url)
print_verbose(f"Valkey semantic-cache initializing index - {self.index_name}")
@ -118,7 +121,7 @@ class ValkeySemanticCache(RedisSemanticCache):
return hashlib.sha256(str(key).encode("utf-8")).hexdigest()
@staticmethod
def _embedding_to_bytes(embedding: list[float]) -> bytes:
def _embedding_to_bytes(embedding: Sequence[float]) -> bytes:
return pack_vector(embedding)
def _index_schema(self, dim: int) -> tuple[TagField, VectorField]:
@ -192,7 +195,9 @@ class ValkeySemanticCache(RedisSemanticCache):
def _doc_key(self, key: str) -> str:
return f"{self.key_prefix}{self._scope_tag(key)}:{uuid.uuid4()}"
def _doc_mapping(self, key: str, prompt: str, value_str: str, embedding: list[float]) -> dict:
def _doc_mapping(
self, key: str, prompt: str, value_str: str, embedding: Sequence[float]
) -> Mapping[str | bytes, str | bytes]:
return {
self.CACHE_KEY_FIELD_NAME: self._scope_tag(key),
self.PROMPT_FIELD_NAME: prompt,
@ -208,30 +213,49 @@ class ValkeySemanticCache(RedisSemanticCache):
)
return Query(query_string).return_fields(self.RESPONSE_FIELD_NAME, self.DISTANCE_FIELD_NAME).dialect(2)
async def _async_search(self, key: str, embedding: Sequence[float]) -> object:
"""Run the KNN query on the async client, stopping the untyped search surface here."""
return await self.async_client.ft(self.index_name).search(
self._knn_query(key),
query_params={"vec": self._embedding_to_bytes(embedding)}, # pyright: ignore[reportArgumentType] # redis stubs omit bytes; KNN vectors are raw bytes at runtime
)
@classmethod
def _first_hit(cls, search_result: Any) -> _ValkeyCacheHit | None:
docs: Final = getattr(search_result, "docs", [])
def _first_hit(cls, search_result: object) -> _ValkeyCacheHit | None:
docs: Final[Sequence[object]] = getattr(search_result, "docs", [])
if not docs:
return None
doc: Final = docs[0]
response_field: Final[object] = getattr(doc, cls.RESPONSE_FIELD_NAME)
distance_field: Final[str | bytes | float] = getattr(doc, cls.DISTANCE_FIELD_NAME)
return _ValkeyCacheHit(
response=str(getattr(doc, cls.RESPONSE_FIELD_NAME)),
distance=float(getattr(doc, cls.DISTANCE_FIELD_NAME)),
response=str(response_field),
distance=float(distance_field),
)
def _resolve_hit(self, hit: _ValkeyCacheHit | None, key: str, **kwargs: Any) -> Any:
@staticmethod
def _record_similarity(kwargs: dict[str, Any], similarity: float) -> None:
"""Stamp the semantic-similarity score onto the request metadata carried in ``kwargs``."""
kwargs.setdefault("metadata", {})["semantic-similarity"] = similarity
@staticmethod
def _embedding_metadata(kwargs: dict[str, Any]) -> dict[str, Any] | None:
"""The request metadata forwarded to the embedding call."""
return kwargs.get("metadata")
def _resolve_hit(self, hit: _ValkeyCacheHit | None, key: str, **kwargs: object) -> object:
if hit is None:
kwargs.setdefault("metadata", {})["semantic-similarity"] = 0.0
self._record_similarity(kwargs, 0.0)
return None
similarity: Final = 1 - hit.distance
kwargs.setdefault("metadata", {})["semantic-similarity"] = similarity
self._record_similarity(kwargs, similarity)
if similarity < self.similarity_threshold:
return None
return self._get_cache_logic(cached_response=hit.response)
def set_cache(self, key: str, value: Any, **kwargs: Any) -> None:
def set_cache(self, key: str, value: object, **kwargs: object) -> None:
print_verbose(f"Valkey semantic-cache set_cache, kwargs: {kwargs}")
try:
prompt: Final = self._get_prompt_from_kwargs(**kwargs)
@ -250,12 +274,12 @@ class ValkeySemanticCache(RedisSemanticCache):
except Exception as e:
print_verbose(f"Error in Valkey semantic-cache set_cache: {e}")
def get_cache(self, key: str, **kwargs: Any) -> Any:
def get_cache(self, key: str, **kwargs: object) -> object:
print_verbose(f"Valkey semantic-cache get_cache, kwargs: {kwargs}")
try:
prompt: Final = self._get_prompt_from_kwargs(**kwargs)
if prompt is None:
kwargs.setdefault("metadata", {})["semantic-similarity"] = 0.0
self._record_similarity(kwargs, 0.0)
return None
embedding: Final = self._get_embedding(prompt)
@ -263,14 +287,14 @@ class ValkeySemanticCache(RedisSemanticCache):
search_result: Final = self.sync_client.ft(self.index_name).search(
self._knn_query(key),
query_params={"vec": self._embedding_to_bytes(embedding)},
query_params={"vec": self._embedding_to_bytes(embedding)}, # pyright: ignore[reportArgumentType] # redis stubs omit bytes; KNN vectors are raw bytes at runtime
)
return self._resolve_hit(self._first_hit(search_result), key, **kwargs)
except Exception as e:
print_verbose(f"Error in Valkey semantic-cache get_cache: {e}")
kwargs.setdefault("metadata", {})["semantic-similarity"] = 0.0
self._record_similarity(kwargs, 0.0)
async def async_set_cache(self, key: str, value: Any, **kwargs: Any) -> None:
async def async_set_cache(self, key: str, value: object, **kwargs: object) -> None:
print_verbose(f"Async Valkey semantic-cache set_cache, kwargs: {kwargs}")
try:
prompt: Final = self._get_prompt_from_kwargs(**kwargs)
@ -278,7 +302,7 @@ class ValkeySemanticCache(RedisSemanticCache):
print_verbose("No prompt provided for semantic caching")
return
embedding: Final = await self._get_async_embedding(prompt, metadata=kwargs.get("metadata"))
embedding: Final = await self._get_async_embedding(prompt, metadata=self._embedding_metadata(kwargs))
await self._ensure_index_async(len(embedding))
doc_key: Final = self._doc_key(key)
@ -289,31 +313,28 @@ class ValkeySemanticCache(RedisSemanticCache):
except Exception as e:
print_verbose(f"Error in async Valkey semantic-cache set_cache: {e}")
async def async_get_cache(self, key: str, **kwargs: Any) -> Any:
async def async_get_cache(self, key: str, **kwargs: object) -> object:
print_verbose(f"Async Valkey semantic-cache get_cache, kwargs: {kwargs}")
try:
prompt: Final = self._get_prompt_from_kwargs(**kwargs)
if prompt is None:
kwargs.setdefault("metadata", {})["semantic-similarity"] = 0.0
self._record_similarity(kwargs, 0.0)
return None
embedding: Final = await self._get_async_embedding(prompt, metadata=kwargs.get("metadata"))
embedding: Final = await self._get_async_embedding(prompt, metadata=self._embedding_metadata(kwargs))
await self._ensure_index_async(len(embedding))
search_result: Final = await self.async_client.ft(self.index_name).search(
self._knn_query(key),
query_params={"vec": self._embedding_to_bytes(embedding)},
)
search_result: Final[object] = await self._async_search(key, embedding)
return self._resolve_hit(self._first_hit(search_result), key, **kwargs)
except Exception as e:
print_verbose(f"Error in async Valkey semantic-cache get_cache: {e}")
kwargs.setdefault("metadata", {})["semantic-similarity"] = 0.0
self._record_similarity(kwargs, 0.0)
async def async_set_cache_pipeline(self, cache_list: list[tuple[str, Any]], **kwargs: Any) -> None:
async def async_set_cache_pipeline(self, cache_list: list[tuple[str, object]], **kwargs: object) -> None:
try:
await asyncio.gather(*[self.async_set_cache(key, value, **kwargs) for key, value in cache_list])
except Exception as e:
print_verbose(f"Error in Valkey semantic-cache async_set_cache_pipeline: {e}")
async def _index_info(self) -> dict:
async def _index_info(self) -> Mapping[str, object]:
return await self.async_client.ft(self.index_name).info()

14
litellm/integrations/bitbucket/bitbucket_client.py
View file

@ -163,15 +163,11 @@ class BitBucketClient:
response.raise_for_status()
data: Final[BitBucketSrcListing] = response.json()
files: Final[list[str]] = []
for item in data.get("values", []):
if item.get("type") == "commit_file":
file_path = item.get("path", "")
if file_path.endswith(file_extension):
files.append(file_path)
return files
return [
file_path
for item in data.get("values", [])
if item.get("type") == "commit_file" and (file_path := item.get("path", "")).endswith(file_extension)
]
except Exception as e:
# Check if it's an HTTP error

87
litellm/integrations/compression_interception/handler.py
View file

@ -7,7 +7,10 @@ litellm_content_retrieve tool calls server-side via the typed agentic loop plan.
import time
import uuid
from typing import TYPE_CHECKING, Any, ClassVar, Final, cast
from collections.abc import Mapping, Sequence
from typing import Any, ClassVar, Final, Protocol, cast
from typing_extensions import ReadOnly, TypedDict
from litellm._logging import verbose_logger
from litellm.compression import compress
@ -22,13 +25,23 @@ from litellm.types.integrations.custom_logger import (
)
from litellm.types.utils import CallTypes
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
LITELLM_CONTENT_RETRIEVE_TOOL_NAME: Final = "litellm_content_retrieve"
_CACHE_TTL_SECONDS: Final = 15 * 60
class _AgenticLoopParams(TypedDict, total=False):
"""The ``agentic_loop_params`` entry the agentic loop driver records on the logging object."""
model: ReadOnly[str]
class _AgenticLoopLoggingObj(Protocol):
"""Logging object view exposing the untyped call details this handler reads."""
@property
def model_call_details(self) -> Mapping[str, _AgenticLoopParams]: ...
def _compression_savings_from_counts(
original_tokens: object, compressed_tokens: object
) -> CompressionSavingsMetadata | None:
@ -83,7 +96,7 @@ class CompressionInterceptionLogger(CustomLogger):
compression_trigger: int = 200_000,
compression_target: int | None = None,
embedding_model: str | None = None,
embedding_model_params: dict[str, Any] | None = None,
embedding_model_params: dict[str, object] | None = None,
):
super().__init__()
self.enabled = enabled
@ -106,7 +119,7 @@ class CompressionInterceptionLogger(CustomLogger):
@staticmethod
def initialize_from_proxy_config(
litellm_settings: dict[str, Any],
callback_specific_params: dict[str, Any],
callback_specific_params: Mapping[str, object],
) -> "CompressionInterceptionLogger":
compression_params: CompressionInterceptionConfig = {}
if "compression_interception_params" in litellm_settings:
@ -120,7 +133,9 @@ class CompressionInterceptionLogger(CustomLogger):
)
return CompressionInterceptionLogger.from_config_yaml(compression_params)
async def async_pre_call_deployment_hook(self, kwargs: dict[str, Any], call_type: CallTypes | None) -> dict | None:
async def async_pre_call_deployment_hook(
self, kwargs: dict[str, Any], call_type: CallTypes | None
) -> dict[str, object] | None:
if not self.enabled:
return None
if call_type is not None and call_type != CallTypes.anthropic_messages:
@ -150,7 +165,7 @@ class CompressionInterceptionLogger(CustomLogger):
cache: Final = cast(dict[str, str], compressed.get("cache", {}))
skip_reason: Final = cast(str | None, compressed.get("compression_skipped_reason"))
compressed_tools: Final = cast(list[dict[str, Any]], compressed.get("tools", []))
compressed_tools: Final = cast(list[dict[str, object]], compressed.get("tools", []))
# Only mutate kwargs when compression actually produced a result.
# If compression was a no-op (below trigger, invalid tool sequence, etc.),
@ -161,7 +176,7 @@ class CompressionInterceptionLogger(CustomLogger):
kwargs["messages"] = compressed["messages"]
if compressed_tools:
kwargs["tools"] = self._merge_tools(
existing_tools=cast(list[dict[str, Any]] | None, kwargs.get("tools")),
existing_tools=cast(list[dict[str, object]] | None, kwargs.get("tools")),
compressed_tools=compressed_tools,
)
call_id = cast(str | None, kwargs.get("litellm_call_id"))
@ -194,14 +209,14 @@ class CompressionInterceptionLogger(CustomLogger):
async def async_should_run_agentic_loop(
self,
response: Any,
response: object,
model: str,
messages: list[dict],
tools: list[dict] | None,
messages: Sequence[Mapping[str, object]],
tools: Sequence[Mapping[str, object]] | None,
stream: bool,
custom_llm_provider: str,
kwargs: dict,
) -> tuple[bool, dict]:
kwargs: Mapping[str, object],
) -> tuple[bool, dict[str, object]]:
if not self.enabled:
return False, {}
if not self._has_retrieval_tool(tools):
@ -219,19 +234,19 @@ class CompressionInterceptionLogger(CustomLogger):
async def async_build_agentic_loop_plan(
self,
tools: dict,
tools: Mapping[str, object],
model: str,
messages: list[dict],
response: Any,
anthropic_messages_provider_config: Any,
anthropic_messages_optional_request_params: dict,
logging_obj: "LiteLLMLoggingObj | None",
messages: list[dict[str, object]],
response: object,
anthropic_messages_provider_config: object,
anthropic_messages_optional_request_params: Mapping[str, object],
logging_obj: _AgenticLoopLoggingObj | None,
stream: bool,
kwargs: dict,
kwargs: Mapping[str, object],
) -> AgenticLoopPlan:
self._prune_expired_cache()
tool_calls: Final = cast(list[dict[str, Any]], tools.get("tool_calls", []))
thinking_blocks: Final = cast(list[dict[str, Any]], tools.get("thinking_blocks", []))
tool_calls: Final = cast(list[dict[str, object]], tools.get("tool_calls", []))
thinking_blocks: Final = cast(list[dict[str, object]], tools.get("thinking_blocks", []))
call_id: Final = self._resolve_call_id(logging_obj=logging_obj, kwargs=kwargs)
cache: Final = self._get_cache(call_id=call_id)
@ -274,7 +289,7 @@ class CompressionInterceptionLogger(CustomLogger):
full_model_name = model
if logging_obj is not None:
agentic_params: Final = logging_obj.model_call_details.get("agentic_loop_params", {})
full_model_name = cast(str, agentic_params.get("model", model))
full_model_name = agentic_params.get("model", model)
request_patch: Final = AgenticLoopRequestPatch(
model=full_model_name,
@ -309,15 +324,15 @@ class CompressionInterceptionLogger(CustomLogger):
return {}
return cache_entry[0]
def _resolve_call_id(self, logging_obj: Any, kwargs: dict[str, Any]) -> str | None:
def _resolve_call_id(self, logging_obj: _AgenticLoopLoggingObj | None, kwargs: Mapping[str, object]) -> str | None:
if logging_obj is not None:
logging_call_id: Final = getattr(logging_obj, "litellm_call_id", None)
if isinstance(logging_call_id, str) and logging_call_id:
return logging_call_id
kwargs_call_id: Final = kwargs.get("litellm_call_id")
return cast(str | None, kwargs_call_id if isinstance(kwargs_call_id, str) else None)
return kwargs_call_id if isinstance(kwargs_call_id, str) else None
def _resolve_retrieval_content(self, tool_call: dict[str, Any], cache: dict[str, str]) -> str:
def _resolve_retrieval_content(self, tool_call: Mapping[str, object], cache: Mapping[str, str]) -> str:
raw_input: Final = tool_call.get("input", {})
key = ""
if isinstance(raw_input, dict):
@ -328,7 +343,9 @@ class CompressionInterceptionLogger(CustomLogger):
return cache[key]
return f"[compressed content key '{key}' not found]"
def _extract_retrieval_tool_calls(self, response: Any) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
def _extract_retrieval_tool_calls(
self, response: object
) -> tuple[list[dict[str, object]], list[dict[str, object]]]:
if isinstance(response, dict):
content = response.get("content", [])
else:
@ -337,8 +354,8 @@ class CompressionInterceptionLogger(CustomLogger):
if not isinstance(content, list):
return [], []
tool_calls: Final[list[dict[str, Any]]] = []
thinking_blocks: Final[list[dict[str, Any]]] = []
tool_calls: Final[list[dict[str, object]]] = []
thinking_blocks: Final[list[dict[str, object]]] = []
for block in content:
if isinstance(block, dict):
@ -385,13 +402,13 @@ class CompressionInterceptionLogger(CustomLogger):
return tool_calls, thinking_blocks
def _prepare_followup_kwargs(self, kwargs: dict[str, Any]) -> dict[str, Any]:
def _prepare_followup_kwargs(self, kwargs: Mapping[str, object]) -> dict[str, object]:
internal_keys: Final = {"litellm_logging_obj"}
return {
k: v for k, v in kwargs.items() if not k.startswith("_compression_interception") and k not in internal_keys
}
def _has_retrieval_tool(self, tools: Any) -> bool:
def _has_retrieval_tool(self, tools: object) -> bool:
if not isinstance(tools, list):
return False
for tool in tools:
@ -407,9 +424,9 @@ class CompressionInterceptionLogger(CustomLogger):
def _merge_tools(
self,
existing_tools: list[dict[str, Any]] | None,
compressed_tools: list[dict[str, Any]],
) -> list[dict[str, Any]]:
existing_tools: Sequence[Mapping[str, object]] | None,
compressed_tools: Sequence[Mapping[str, object]],
) -> list[Mapping[str, object]]:
merged: Final = list(existing_tools or [])
if self._has_retrieval_tool(merged):
return merged

28
litellm/integrations/custom_logger.py
View file

@ -2,7 +2,7 @@
# On success, logs events to Promptlayer
import re
import traceback
from collections.abc import AsyncGenerator, Mapping
from collections.abc import AsyncGenerator, Mapping, Sequence
from typing import TYPE_CHECKING, Any, ClassVar, Final, Optional
from pydantic import BaseModel
@ -123,11 +123,11 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac
return []
callbacks: Final = AllCallbacks()
callback_info: Final = getattr(callbacks, lookup_name, None)
callback_info: Final[object] = getattr(callbacks, lookup_name, None)
if callback_info is None:
return []
params: Final = getattr(callback_info, "litellm_callback_params", None)
params: Final[Sequence[str] | None] = getattr(callback_info, "litellm_callback_params", None)
if not params:
return []
@ -851,7 +851,7 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac
- Converting to string and then truncating the logged content catches this
2. We want to avoid modifying the original `messages`, `response`, and `error_str` in the logging payload since these are in kwargs and could be returned to the user
"""
field_value: Final = standard_logging_object.get(field_name)
field_value: Final[object] = standard_logging_object.get(field_name)
if field_value:
str_value: Final = str(field_value)
if len(str_value) > max_length:
@ -1005,8 +1005,8 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac
• Keep untyped or text content.
• Recursively redact inline base64 blobs in *any* string field, at any depth.
"""
raw_messages: Final[Any] = payload.get("messages", [])
messages: Final[list[Any]] = raw_messages if isinstance(raw_messages, list) else []
raw_messages: Final[object] = payload.get("messages", [])
messages: Final[list[object]] = raw_messages if isinstance(raw_messages, list) else []
verbose_logger.debug("[CustomLogger] Stripping base64 from %s messages", len(messages))
if messages:
@ -1037,8 +1037,8 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac
• Keep untyped or text content.
• Recursively redact inline base64 blobs in *any* string field, at any depth.
"""
raw_messages: Final[Any] = payload.get("messages", [])
messages: Final[list[Any]] = raw_messages if isinstance(raw_messages, list) else []
raw_messages: Final[object] = payload.get("messages", [])
messages: Final[list[object]] = raw_messages if isinstance(raw_messages, list) else []
verbose_logger.debug("[CustomLogger] Stripping base64 from %s messages", len(messages))
if messages:
@ -1059,7 +1059,7 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac
value: Any,
depth: int = 0,
max_depth: int = DEFAULT_MAX_RECURSE_DEPTH_SENSITIVE_DATA_MASKER,
) -> Any:
) -> object:
"""Recursively redact inline base64 from any nested structure with a max recursion depth limit."""
if depth > max_depth:
verbose_logger.warning("[CustomLogger] Max recursion depth %s reached while redacting base64", max_depth)
@ -1090,16 +1090,16 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac
def _process_messages(
self,
messages: list[Any],
messages: list[object],
max_depth: int = DEFAULT_MAX_RECURSE_DEPTH_SENSITIVE_DATA_MASKER,
) -> list[dict[str, Any]]:
filtered_messages: Final[list[dict[str, Any]]] = []
) -> list[dict[str, object]]:
filtered_messages: Final[list[dict[str, object]]] = []
for msg in messages:
if not isinstance(msg, dict):
continue
contents: Any = msg.get("content")
contents: object = msg.get("content")
if isinstance(contents, list):
cleaned: list[Any] = []
cleaned: list[object] = []
for c in contents:
if self._should_keep_content(content=c):
cleaned.append(self._redact_base64(value=c, max_depth=max_depth))

12
litellm/integrations/opik/opik_payload_builder/payload_builders.py
View file

@ -17,12 +17,12 @@ def build_trace_payload(
end_time: datetime,
input_data: Any,
output_data: Any,
metadata: dict[str, Any],
metadata: dict[str, object],
tags: list[str],
thread_id: str | None,
) -> types.TracePayload:
"""Build a complete trace payload."""
trace_name: Final = response_obj.get("object", "unknown type")
trace_name: Final[str] = response_obj.get("object", "unknown type")
return types.TracePayload(
project_name=project_name,
@ -47,7 +47,7 @@ def build_span_payload(
end_time: datetime,
input_data: Any,
output_data: Any,
metadata: dict[str, Any],
metadata: dict[str, object],
tags: list[str],
usage: dict[str, int],
provider: str | None = None,
@ -56,9 +56,9 @@ def build_span_payload(
"""Build a complete span payload."""
span_id: Final = utils.create_uuid7()
model: Final = response_obj.get("model", "unknown-model")
obj_type: Final = response_obj.get("object", "unknown-object")
created: Final = response_obj.get("created", 0)
model: Final[str] = response_obj.get("model", "unknown-model")
obj_type: Final[str] = response_obj.get("object", "unknown-object")
created: Final[int] = response_obj.get("created", 0)
span_name: Final = f"{model}_{obj_type}_{created}"
_logging.verbose_logger.debug("OpikLogger creating span with id %s for trace %s", span_id, trace_id)

30
litellm/integrations/otel/plumbing/providers.py
View file

@ -1,7 +1,7 @@
"""Provider / exporter factory + the Baggage span processor."""
from collections.abc import Callable, Iterable
from typing import TYPE_CHECKING, Any, Final
from typing import TYPE_CHECKING, Any, Final, Literal
from opentelemetry import _logs, baggage, metrics
from opentelemetry._events import EventLogger
@ -135,14 +135,36 @@ def parse_headers(raw: str | None) -> dict[str, str]:
return dict(parse_env_headers(raw, liberal=True))
_IN_MEMORY_KINDS: Final = ("in_memory", "inmemory", "memory")
_OTLP_HTTP_KINDS: Final = ("otlp_http", "http", "http/protobuf", "http/json")
_OTLP_GRPC_KINDS: Final = ("otlp_grpc", "grpc")
def exporter_transport(kind: str) -> Literal["http", "grpc", "headerless"]:
"""How an exporter of this ``kind`` carries credentials, per ``_exporter_from_spec``.
``http``/``grpc`` exporters (and any registered factory, which builds an
OTLP exporter) stamp ``spec.headers``; ``console``, ``in_memory``, and any
unrecognized kind (which falls back to a header-ignoring console exporter)
are ``headerless``. Routability decisions must read this rather than a
denylist, so a typo'd or unavailable kind is not mistaken for OTLP.
"""
resolved: Final = kind.lower()
if resolved in _OTLP_HTTP_KINDS or resolved in _EXPORTER_FACTORIES:
return "http"
if resolved in _OTLP_GRPC_KINDS:
return "grpc"
return "headerless"
def _exporter_from_spec(spec: ExporterSpec) -> SpanExporter:
kind: Final = (spec.kind or "console").lower()
factory: Final = _EXPORTER_FACTORIES.get(kind)
if factory is not None:
return factory(spec)
if kind in ("in_memory", "inmemory", "memory"):
if kind in _IN_MEMORY_KINDS:
return InMemorySpanExporter()
if kind in ("otlp_http", "http", "http/protobuf", "http/json"):
if kind in _OTLP_HTTP_KINDS:
from opentelemetry.exporter.otlp.proto.http.trace_exporter import (
OTLPSpanExporter as HTTPExporter,
)
@ -151,7 +173,7 @@ def _exporter_from_spec(spec: ExporterSpec) -> SpanExporter:
endpoint=_otlp_traces_endpoint(spec.endpoint),
headers=parse_headers(spec.headers),
)
if kind in ("otlp_grpc", "grpc"):
if kind in _OTLP_GRPC_KINDS:
from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import (
OTLPSpanExporter as GRPCExporter,
)

62
litellm/integrations/otel/plumbing/routing.py
View file

@ -27,6 +27,7 @@ from litellm.constants import OTEL_SERVICE_NAME_METADATA_KEYS
from litellm.integrations.otel.model.config import ExporterSpec, OpenTelemetryV2Config
from litellm.integrations.otel.plumbing.providers import (
build_tracer_provider,
exporter_transport,
get_tracer,
)
from litellm.integrations.otel.presets import (
@ -121,13 +122,27 @@ def _encoded_header_string(headers: Mapping[str, str]) -> str:
class TenantRoute:
"""The tracer to create a span on, plus whether it must root its own trace.
``detached`` is True when project routing engaged. Phoenix assigns a whole
``detached`` is True when the routed span exports to a DIFFERENT backend
than the request's root span, which always exports through the default
tracer. A detached span roots a fresh trace with a link back to the request
trace for correlation, so the destination account is not left holding a
child whose parent it never received. It is driven by whether routing
headers were actually applied to an owned exporter, not merely requested:
a credential or project route whose callback owns no exporter those headers
can reach exports through the default backend unchanged, so it stays
parented like an unrouted span.
Credential routing (a team/key's own vendor account) is one detaching case:
the root, auth, and db spans stay on the operator's default backend while
the LLM-call span exports to the tenant's account, so parenting it into the
request trace makes the tenant account show a fragmented span with a missing
parent. Project routing (Phoenix) is the other: Phoenix assigns a whole
trace to one project by whichever of its spans arrives first, so a
project-routed span parented into the request trace gets dragged into the
project of the default-exported request spans and the header does nothing.
The span must therefore start a fresh trace (with a link back to the
request trace for correlation) — which is also how the v1 Phoenix logger
behaved, exporting each request under its own Phoenix-local parent span.
Both mirror the v1 loggers, which exported each request under its own
backend-local root. Service-name routing does NOT detach: it relabels
``service.name`` on the SAME operator backend, where the parent is present.
"""
tracer: Tracer
@ -161,11 +176,20 @@ class TenantTracerCache:
self._open_span_counts: dict[TracerProvider, int] = {} # mutable-ok: live refcount state
# Oldest-first so an overflow of draining providers sheds the stalest.
self._retired: OrderedDict[TracerProvider, None] = OrderedDict() # mutable-ok: draining evicted providers
self._project_routable = any(
spec.owner == callback_name and spec.kind.lower() not in (*_NON_OTLP_KINDS, *_GRPC_KINDS)
for spec in config.exporters
# An owned exporter is routable only when its kind actually resolves to a
# header-carrying OTLP exporter. A denylist would accept a typo'd or
# unavailable kind, which ``_exporter_from_spec`` falls back to a
# header-ignoring console exporter: detaching such a span would strand it
# on the operator's console, never reaching the tenant backend. Project
# headers are HTTP-only; credentials ride gRPC metadata too (Arize's
# default exporter is gRPC), so they accept either OTLP transport.
owned_transports: Final = tuple(
exporter_transport(spec.kind) for spec in config.exporters if spec.owner == callback_name
)
self._project_routable = "http" in owned_transports
self._credential_routable = "http" in owned_transports or "grpc" in owned_transports
self._warned_project_unroutable = False
self._warned_credential_unroutable = False
def release(self, provider: TracerProvider | None) -> None:
"""Drop one open-span count; shut a retired provider down once drained.
@ -207,7 +231,7 @@ class TenantTracerCache:
concurrent overflow eviction can't shut it down between selection and
the caller's span start. The caller must ``release`` it exactly once.
"""
credential_headers: Final = dynamic_otlp_headers(self._callback_name, dynamic_params) or _NO_HEADERS
credential_headers: Final = self._credential_headers(dynamic_params)
project_headers: Final = self._project_headers(auth_metadata)
service_name: Final = tenant_service_name(auth_metadata)
if not credential_headers and not project_headers and service_name is None:
@ -231,7 +255,7 @@ class TenantTracerCache:
_shutdown_provider(evicted)
return TenantRoute(
tracer=get_tracer(provider, self._tracer_name),
detached=bool(project_headers),
detached=bool(project_headers) or bool(credential_headers),
provider=provider,
)
@ -275,6 +299,26 @@ class TenantTracerCache:
self._open_span_counts.pop(overflowed, None)
return overflowed
def _credential_headers(self, dynamic_params: StandardCallbackDynamicParams | None) -> Mapping[str, str]:
"""The per-request dynamic OTLP credentials, if this cache can apply them.
A callback owning only a console/in_memory exporter has nowhere to stamp
them, so the span would export to the operator's default backend
unchanged; routing there and detaching would orphan it on the very
backend that holds its parent. Warn once and keep the default tracer.
"""
requested: Final = dynamic_otlp_headers(self._callback_name, dynamic_params) or _NO_HEADERS
if not requested or self._credential_routable:
return requested
if not self._warned_credential_unroutable:
self._warned_credential_unroutable = True
verbose_logger.warning(
"OTel V2: %s request carries dynamic credentials, but the callback owns no "
"OTLP exporter to stamp them onto; spans export to the default backend.",
self._callback_name,
)
return _NO_HEADERS
def _project_headers(self, auth_metadata: Mapping[str, str] | None) -> Mapping[str, str]:
"""The per-request project-routing headers, if this cache can apply them.

16
litellm/integrations/websearch_interception/handler.py
View file

@ -10,7 +10,7 @@ import asyncio
import math
import uuid
from collections.abc import AsyncIterator, Mapping, Sequence
from typing import TYPE_CHECKING, Any, Final, TypedDict, cast
from typing import TYPE_CHECKING, Any, Final, TypedDict, TypeVar, cast
from typing_extensions import ReadOnly
@ -77,6 +77,10 @@ WEBSEARCH_EMIT_NATIVE_BLOCKS_KEY: Final = "_websearch_interception_emit_native_b
# ``web_search_tool_result`` blocks to inject into the final response.
WEBSEARCH_NATIVE_BLOCKS_METADATA_KEY: Final = "websearch_native_blocks"
_RESPONSE_CONTENT_FIELD: Final = "content"
_ResponseT: Final = TypeVar("_ResponseT")
class _PlanMetadataView(TypedDict):
websearch_native_blocks: Sequence[Mapping[str, object]] | None
@ -948,17 +952,17 @@ class WebSearchInterceptionLogger(CustomLogger):
)
@staticmethod
def _inject_native_blocks(response: Any, native_blocks: Sequence[Mapping[str, object]]) -> Any:
def _inject_native_blocks(response: _ResponseT, native_blocks: Sequence[Mapping[str, object]]) -> _ResponseT:
"""Prepend native blocks to response content, dict or object form."""
if not native_blocks:
return response
if isinstance(response, dict):
existing = response.get("content") or []
response["content"] = list(native_blocks) + list(existing)
existing = response.get(_RESPONSE_CONTENT_FIELD) or []
response[_RESPONSE_CONTENT_FIELD] = list(native_blocks) + list(existing)
return response
existing = getattr(response, "content", None) or []
existing = getattr(response, _RESPONSE_CONTENT_FIELD, None) or []
try:
response.content = list(native_blocks) + list(existing)
setattr(response, _RESPONSE_CONTENT_FIELD, list(native_blocks) + list(existing))
except (AttributeError, TypeError):
# Object refused write — fall through and leave the response
# untouched rather than crash the request.

14
litellm/litellm_core_utils/litellm_logging.py
View file

@ -547,6 +547,9 @@ class Logging(LiteLLMLoggingBaseClass):
# Init Caching related details
self.caching_details: CachingDetails | None = None
# Timing for results that cannot carry ``_hidden_params`` (plain-dict /v1/messages
# responses and the bridge stream wrappers); see ``update_response_metadata``.
self.response_timing_metrics: Mapping[str, float] = {} # mutable-ok: kept deep-copyable
# Passthrough endpoint guardrails config for field targeting
self.passthrough_guardrails_config: dict[str, Any] | None = None
@ -566,6 +569,10 @@ class Logging(LiteLLMLoggingBaseClass):
self._defer_async_logging: bool = False
self._enqueue_deferred_logging: Callable[[], None] | None = None
def set_response_timing_metrics(self, timing_metrics: Mapping[str, float]) -> None:
"""Keep ``_response_ms`` / ``litellm_overhead_time_ms`` for a result that has no ``_hidden_params``."""
self.response_timing_metrics = dict(timing_metrics) # mutable-ok: kept deep-copyable
def process_dynamic_callbacks(self):
"""
Initializes CustomLogger compatible callbacks in self.dynamic_* callbacks
@ -6005,6 +6012,13 @@ def get_standard_logging_object_payload(
clean_hidden_params: Final = StandardLoggingPayloadSetup.get_hidden_params(hidden_params)
if clean_hidden_params["response_cost"] is None and raw_response_cost is not None:
clean_hidden_params["response_cost"] = llm_response_cost
if clean_hidden_params["litellm_overhead_time_ms"] is None and status == "success":
# /v1/messages dict results and the bridge stream wrappers keep it on the logging object;
# failure payloads stay None like every response type that carries its own _hidden_params
timing_metrics: Final = (
getattr(logging_obj, "response_timing_metrics", None) or {} # mutable-ok: empty fallback
)
clean_hidden_params["litellm_overhead_time_ms"] = timing_metrics.get("litellm_overhead_time_ms")
model_cost_information: Final = StandardLoggingPayloadSetup.get_model_cost_information(
base_model=base_model,

124
litellm/litellm_core_utils/llm_response_utils/response_metadata.py
View file

@ -1,6 +1,9 @@
import datetime
from collections.abc import Mapping
from typing import Any, Final
import httpx
from litellm.constants import LITELLM_DETAILED_TIMING
from litellm.litellm_core_utils.core_helpers import process_response_headers
from litellm.litellm_core_utils.llm_response_utils.get_api_base import get_api_base
@ -13,6 +16,39 @@ from litellm.types.utils import (
)
def response_timing_metrics(
start_time: datetime.datetime,
end_time: datetime.datetime,
logging_obj: LiteLLMLoggingObject,
include_overhead: bool = True,
) -> Mapping[str, float]:
"""``_response_ms`` for the whole call, plus ``litellm_overhead_time_ms`` when it can be derived.
On a cache hit the overhead is the total minus the cache read; otherwise it is the total minus
the provider call (``llm_api_duration_ms``). It is omitted when neither duration was recorded,
and when ``include_overhead`` is False because the two durations cover different windows.
"""
total_response_time_ms: Final = (end_time - start_time).total_seconds() * 1000
if not include_overhead:
return {"_response_ms": total_response_time_ms} # mutable-ok: read-only timing result
caching_details: Final = logging_obj.caching_details
cache_duration_ms: Final = (
caching_details.get("cache_duration_ms")
if caching_details is not None and caching_details.get("cache_hit") is True
else None
)
llm_api_duration_ms: Final = logging_obj.model_call_details.get("llm_api_duration_ms")
if cache_duration_ms is not None:
overhead_ms: float | None = total_response_time_ms - cache_duration_ms
elif llm_api_duration_ms is not None:
overhead_ms = round(total_response_time_ms - llm_api_duration_ms, 4)
else:
overhead_ms = None
if overhead_ms is None:
return {"_response_ms": total_response_time_ms}
return {"_response_ms": total_response_time_ms, "litellm_overhead_time_ms": overhead_ms}
class ResponseMetadata:
"""
Handles setting and managing `_hidden_params`, `response_time_ms`, and `litellm_overhead_time_ms` for LiteLLM responses
@ -25,11 +61,7 @@ class ResponseMetadata:
@property
def supports_response_time(self) -> bool:
"""Check if response type supports timing metrics"""
return (
isinstance(self.result, ModelResponse)
or isinstance(self.result, EmbeddingResponse)
or isinstance(self.result, TranscriptionResponse)
)
return isinstance(self.result, (ModelResponse, EmbeddingResponse, TranscriptionResponse))
def set_hidden_params(self, logging_obj: LiteLLMLoggingObject, model: str | None, kwargs: dict) -> None:
"""Set hidden parameters on the response"""
@ -45,14 +77,14 @@ class ResponseMetadata:
result=self.result, litellm_model_name=model, router_model_id=model_id
),
"additional_headers": process_response_headers(
self._get_value_from_hidden_params("additional_headers") or {},
self._get_additional_headers_from_hidden_params() or {},
preserve_litellm_internal_headers=True,
),
"litellm_model_name": model,
}
self._update_hidden_params(new_params)
def _update_hidden_params(self, new_params: dict) -> None:
def _update_hidden_params(self, new_params: Mapping[str, object]) -> None:
"""
Update hidden params - handles when self._hidden_params is a dict or HiddenParams object
"""
@ -64,51 +96,38 @@ class ResponseMetadata:
for key, value in new_params.items():
setattr(self._hidden_params, key, value)
def _get_value_from_hidden_params(self, key: str) -> Any | None:
"""Get value from hidden params - handles when self._hidden_params is a dict or HiddenParams object"""
def _get_additional_headers_from_hidden_params(self) -> httpx.Headers | dict[str, str] | None:
"""Get `additional_headers` from hidden params - handles when self._hidden_params is a dict or HiddenParams object"""
if isinstance(self._hidden_params, dict):
return self._hidden_params.get(key, None)
return self._hidden_params.get("additional_headers", None)
elif isinstance(self._hidden_params, HiddenParams):
return getattr(self._hidden_params, key, None)
return getattr(self._hidden_params, "additional_headers", None)
def set_timing_metrics(
self,
start_time: datetime.datetime,
end_time: datetime.datetime,
logging_obj: LiteLLMLoggingObject,
include_overhead: bool = True,
) -> None:
"""Set response timing metrics"""
total_response_time_ms: Final = (end_time - start_time).total_seconds() * 1000
timing_metrics: Final = response_timing_metrics(start_time, end_time, logging_obj, include_overhead)
total_response_time_ms: Final = timing_metrics["_response_ms"]
# Set total response time if supported
if self.supports_response_time:
self.result._response_ms = total_response_time_ms
#########################################################
# 1. Add _response_ms total duration
# 1. Add _response_ms total duration and the LiteLLM overhead within it
# (total minus the cache read on a cache hit, else total minus the provider call)
#########################################################
self._update_hidden_params(
{
"_response_ms": total_response_time_ms,
}
)
self._update_hidden_params(timing_metrics)
#########################################################
# 2. Add LiteLLM overhead duration
# 2. Add callback processing duration
#########################################################
llm_api_duration_ms: Final = logging_obj.model_call_details.get("llm_api_duration_ms")
if llm_api_duration_ms is not None:
overhead_ms = round(total_response_time_ms - llm_api_duration_ms, 4)
self._update_hidden_params(
{
"litellm_overhead_time_ms": overhead_ms,
}
)
#########################################################
# 3. Add callback processing duration
#########################################################
callback_duration_ms: Final = getattr(logging_obj, "callback_duration_ms", None)
callback_duration_ms: Final[float | None] = getattr(logging_obj, "callback_duration_ms", None)
if callback_duration_ms is not None:
self._update_hidden_params(
{
@ -117,36 +136,21 @@ class ResponseMetadata:
)
#########################################################
# 4. Add duration for reading from cache
# In this case overhead from litellm is the difference between the cache read duration and the total response time
#########################################################
if (
logging_obj.caching_details is not None
and logging_obj.caching_details.get("cache_hit") is True
and (cache_duration_ms := logging_obj.caching_details.get("cache_duration_ms")) is not None
):
overhead_ms = total_response_time_ms - cache_duration_ms
self._update_hidden_params(
{
"litellm_overhead_time_ms": overhead_ms,
}
)
#########################################################
# 5. Detailed per-phase timing (opt-in via env var)
# 3. Detailed per-phase timing (opt-in via env var)
#########################################################
llm_api_duration_ms: Final = logging_obj.model_call_details.get("llm_api_duration_ms")
if LITELLM_DETAILED_TIMING and llm_api_duration_ms is not None:
detailed: Final[dict] = {
detailed: Final[dict[str, float]] = {
"timing_llm_api_ms": round(llm_api_duration_ms, 4),
}
# message copy time from Logging.__init__()
msg_copy_ms: Final = getattr(logging_obj, "message_copy_duration_ms", None)
msg_copy_ms: Final[float | None] = getattr(logging_obj, "message_copy_duration_ms", None)
if msg_copy_ms is not None:
detailed["timing_message_copy_ms"] = round(msg_copy_ms, 4)
# pre-processing = time from request start to LLM API call start
api_call_start: Final = logging_obj.model_call_details.get("api_call_start_time")
api_call_start: Final[datetime.datetime | None] = logging_obj.model_call_details.get("api_call_start_time")
if api_call_start is not None and start_time is not None:
pre_ms: Final = (api_call_start - start_time).total_seconds() * 1000
detailed["timing_pre_processing_ms"] = round(pre_ms, 4)
@ -170,6 +174,7 @@ def update_response_metadata(
kwargs: dict,
start_time: datetime.datetime,
end_time: datetime.datetime,
include_overhead: bool = True,
) -> None:
"""
Updates response metadata including hidden params and timing metrics
@ -177,11 +182,22 @@ def update_response_metadata(
- response._hidden_params
- response._hidden_params["litellm_overhead_time_ms"]
- response.response_time_ms
A result that cannot hold ``_hidden_params`` gets its timing on ``logging_obj`` instead.
Callers whose ``end_time`` covers more than the recorded provider call (a stream read to
completion) pass ``include_overhead=False``, since the overhead cannot be derived there.
"""
if result is None or not hasattr(result, "_hidden_params"):
if result is None:
return
if not hasattr(result, "_hidden_params"):
# /v1/messages returns a plain dict and the Anthropic / Responses bridge stream wrappers
# cannot hold ``_hidden_params``: keep only the timing on the logging object (no cost
# recompute) so the proxy headers and the standard logging payload can still read it.
logging_obj.set_response_timing_metrics(
response_timing_metrics(start_time, end_time, logging_obj, include_overhead)
)
return
metadata: Final = ResponseMetadata(result)
metadata.set_hidden_params(logging_obj, model, kwargs)
metadata.set_timing_metrics(start_time, end_time, logging_obj)
metadata.set_timing_metrics(start_time, end_time, logging_obj, include_overhead)
metadata.apply()

8
litellm/litellm_core_utils/streaming_handler.py
View file

@ -93,7 +93,7 @@ def print_verbose(print_statement: object):
@dataclass(frozen=True, slots=True)
class _ProviderChunkParsed:
response_obj: dict[str, Any]
response_obj: dict[str, object]
@dataclass(frozen=True, slots=True)
@ -1288,7 +1288,7 @@ class CustomStreamWrapper:
for key, value in anthropic_response_obj["provider_specific_fields"].items():
setattr(model_response, key, value)
response_obj = cast(dict[str, Any], anthropic_response_obj)
response_obj = cast(dict[str, object], anthropic_response_obj)
elif self.model == "replicate" or self.custom_llm_provider == "replicate":
response_obj = self.handle_replicate_chunk(chunk)
completion_obj["content"] = response_obj["text"]
@ -1444,7 +1444,7 @@ class CustomStreamWrapper:
if not isinstance(chunk, str):
raise ValueError(f"chunk is not a string: {chunk}")
response_obj = cast(
dict[str, Any],
dict[str, object],
litellm.CodestralTextCompletionConfig()._chunk_parser(chunk),
)
completion_obj["content"] = response_obj["text"]
@ -2551,7 +2551,7 @@ def calculate_total_usage(chunks: list[ModelResponse]) -> Usage:
prompt_tokens: int = 0
completion_tokens: int = 0
latest_usage_chunk = None
latest_usage_chunk: Usage | Mapping[str, int] | None = None
prompt_tokens_details: PromptTokensDetailsWrapper | None = None
completion_tokens_details: CompletionTokensDetailsWrapper | None = None
cache_creation_token_details: CacheCreationTokenDetails | None = None

4
litellm/litellm_core_utils/url_utils.py
View file

@ -417,7 +417,7 @@ def _extract_redirect_url(response: httpx.Response, request_url: str) -> str:
return str(httpx.URL(request_url).join(location))
def safe_get(client: Any, url: str, **kwargs: Any) -> Any:
def safe_get(client: Any, url: str, **kwargs: Any) -> httpx.Response:
"""
Fetch a user-supplied URL with SSRF protection on every redirect hop.
@ -460,7 +460,7 @@ def safe_get(client: Any, url: str, **kwargs: Any) -> Any:
raise SSRFError("Too many redirects")
async def async_safe_get(client: Any, url: str, **kwargs: Any) -> Any:
async def async_safe_get(client: Any, url: str, **kwargs: Any) -> httpx.Response:
"""Async version of safe_get."""
if not getattr(litellm, "user_url_validation", True):
kwargs.setdefault("follow_redirects", True)

59
litellm/llms/anthropic/batches/transformation.py
View file

@ -1,9 +1,11 @@
import json
import time
from collections.abc import Mapping
from typing import TYPE_CHECKING, Any, Final, Literal, cast
import httpx
from httpx import Headers, Response
from typing_extensions import ReadOnly, TypedDict
from litellm.litellm_core_utils.url_utils import encode_url_path_segment
from litellm.llms.base_llm.batches.transformation import BaseBatchesConfig
@ -21,6 +23,29 @@ else:
LoggingClass = Any
class AnthropicBatchRequestCounts(TypedDict, total=False):
"""The ``request_counts`` object of an Anthropic Message Batch."""
processing: ReadOnly[int]
succeeded: ReadOnly[int]
errored: ReadOnly[int]
canceled: ReadOnly[int]
expired: ReadOnly[int]
class AnthropicMessageBatch(TypedDict, total=False):
"""The fields of an Anthropic Message Batch that map onto an OpenAI Batch."""
id: ReadOnly[str]
processing_status: ReadOnly[str]
created_at: ReadOnly[str | None]
ended_at: ReadOnly[str | None]
expires_at: ReadOnly[str | None]
cancel_initiated_at: ReadOnly[str | None]
archived_at: ReadOnly[str | None]
request_counts: ReadOnly[AnthropicBatchRequestCounts]
class AnthropicBatchesConfig(BaseBatchesConfig):
def __init__(self):
from ..chat.transformation import AnthropicConfig
@ -85,7 +110,7 @@ class AnthropicBatchesConfig(BaseBatchesConfig):
create_batch_data: CreateBatchRequest,
optional_params: dict,
litellm_params: dict,
) -> bytes | str | dict[str, Any]:
) -> bytes | str | dict[str, object]:
"""
Transform the batch creation request to Anthropic format.
@ -135,7 +160,7 @@ class AnthropicBatchesConfig(BaseBatchesConfig):
batch_id: str,
optional_params: dict,
litellm_params: dict,
) -> bytes | str | dict[str, Any]:
) -> bytes | str | dict[str, object]:
"""
Transform batch retrieval request for Anthropic.
@ -154,7 +179,7 @@ class AnthropicBatchesConfig(BaseBatchesConfig):
) -> LiteLLMBatch:
"""Transform Anthropic MessageBatch retrieval response to LiteLLM format."""
try:
response_data: Final = raw_response.json()
response_data: Final[AnthropicMessageBatch] = raw_response.json()
except Exception as e:
raise ValueError(f"Failed to parse Anthropic batch response: {e}")
@ -163,18 +188,20 @@ class AnthropicBatchesConfig(BaseBatchesConfig):
processing_status: Final = response_data.get("processing_status", "in_progress")
# Map Anthropic processing_status to OpenAI status
status_mapping: dict[
str,
Literal[
"validating",
"failed",
"in_progress",
"finalizing",
"completed",
"expired",
"cancelling",
"cancelled",
],
status_mapping: Final[
Mapping[
str,
Literal[
"validating",
"failed",
"in_progress",
"finalizing",
"completed",
"expired",
"cancelling",
"cancelled",
],
]
] = {
"in_progress": "in_progress",
"canceling": "cancelling",
@ -281,7 +308,7 @@ class AnthropicBatchesConfig(BaseBatchesConfig):
if not line:
continue
try:
response_json = json.loads(line)
response_json: Mapping[str, Mapping[str, dict[str, object]]] = json.loads(line)
# Update model_response with the parsed JSON
completion_response = response_json["result"]["message"]
transformed_response = self.anthropic_chat_config.transform_parsed_response(

4
litellm/llms/anthropic/experimental_pass_through/responses_adapters/handler.py
View file

@ -160,7 +160,7 @@ class LiteLLMMessagesToResponsesAPIHandler:
top_k: int | None = None,
top_p: float | None = None,
output_format: AnthropicOutputSchema | None = None,
**kwargs,
**kwargs: object,
) -> AnthropicMessagesResponse | AsyncIterator[bytes]:
responses_kwargs: Final = _build_responses_kwargs(
max_tokens=max_tokens,
@ -214,7 +214,7 @@ class LiteLLMMessagesToResponsesAPIHandler:
top_p: float | None = None,
output_format: AnthropicOutputSchema | None = None,
_is_async: bool = False,
**kwargs,
**kwargs: object,
) -> (
AnthropicMessagesResponse
| AsyncIterator[bytes]

18
litellm/llms/bedrock/realtime/handler.py
View file

@ -94,7 +94,7 @@ class BedrockRealtime(BaseAWSLLM):
aws_sts_endpoint: str | None = None,
aws_bedrock_runtime_endpoint: str | None = None,
aws_external_id: str | None = None,
**kwargs,
**kwargs: object,
):
"""
Establish bidirectional streaming connection with Bedrock Nova Sonic.
@ -166,13 +166,16 @@ class BedrockRealtime(BaseAWSLLM):
)
bedrock_client: Final = BedrockRuntimeClient(config=config)
async def open_bidirectional_stream() -> BedrockBidirectionalStream:
return await bedrock_client.invoke_model_with_bidirectional_stream(
InvokeModelWithBidirectionalStreamOperationInput(model_id=model)
)
transformation_config: Final = BedrockRealtimeConfig()
try:
# Initialize the bidirectional stream
bedrock_stream: Final = await bedrock_client.invoke_model_with_bidirectional_stream(
InvokeModelWithBidirectionalStreamOperationInput(model_id=model)
)
bedrock_stream: Final = await open_bidirectional_stream()
verbose_proxy_logger.debug("Bedrock Realtime: Bidirectional stream established")
@ -243,10 +246,11 @@ class BedrockRealtime(BaseAWSLLM):
InvokeModelWithBidirectionalStreamInputChunk,
)
def build_input_chunk(payload: bytes) -> object:
return InvokeModelWithBidirectionalStreamInputChunk(value=BidirectionalInputPayloadPart(bytes_=payload))
async def send_to_bedrock(bedrock_message: str) -> None:
event: Final = InvokeModelWithBidirectionalStreamInputChunk(
value=BidirectionalInputPayloadPart(bytes_=bedrock_message.encode("utf-8"))
)
event: Final = build_input_chunk(bedrock_message.encode("utf-8"))
await bedrock_stream.input_stream.send(event)
verbose_proxy_logger.debug("Bedrock Realtime: Sent to Bedrock: %s", bedrock_message[:200])

4
litellm/llms/chatgpt/chat/streaming_utils.py
View file

@ -49,10 +49,10 @@ class ChatGPTToolCallNormalizer:
def __getattr__(self, name: str) -> object:
return getattr(self._stream, name)
def __iter__(self):
def __iter__(self) -> "ChatGPTToolCallNormalizer":
return self
def __aiter__(self):
def __aiter__(self) -> "ChatGPTToolCallNormalizer":
return self
def __next__(self) -> ModelResponseStream:

31
litellm/llms/compactifai/chat/transformation.py
View file

@ -2,13 +2,16 @@
CompactifAI chat completion transformation
"""
from collections.abc import Mapping, Sequence
from typing import TYPE_CHECKING, Any, Final
import httpx
from typing_extensions import ReadOnly, TypedDict
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.llms.openai.common_utils import OpenAIError
from litellm.secret_managers.main import get_secret_str
from litellm.types.llms.openai import AllMessageValues
from litellm.types.utils import ModelResponse
from ...openai.chat.gpt_transformation import OpenAIGPTConfig
@ -23,6 +26,18 @@ else:
LiteLLMLoggingObj = Any
class CompactifAIResponseFields(TypedDict, total=False):
"""The chat completion fields of a CompactifAI response body."""
id: ReadOnly[str]
choices: ReadOnly[Sequence[Mapping[str, object]]]
created: ReadOnly[int]
model: ReadOnly[str]
system_fingerprint: ReadOnly[str | None]
usage: ReadOnly[Mapping[str, object]]
object: ReadOnly[str]
class CompactifAIChatConfig(OpenAIGPTConfig):
"""
Configuration class for CompactifAI chat completions.
@ -47,10 +62,10 @@ class CompactifAIChatConfig(OpenAIGPTConfig):
raw_response: httpx.Response,
model_response: ModelResponse,
logging_obj: LiteLLMLoggingObj,
request_data: dict,
messages: list,
optional_params: dict,
litellm_params: dict,
request_data: Mapping[str, object],
messages: Sequence[AllMessageValues],
optional_params: Mapping[str, object],
litellm_params: Mapping[str, object],
encoding: "tiktoken.Encoding | None",
api_key: str | None = None,
json_mode: bool | None = None,
@ -81,14 +96,18 @@ class CompactifAIChatConfig(OpenAIGPTConfig):
message["content"] = tool_calls[0]["function"].get("arguments", "")
message["tool_calls"] = None
returned_response: Final = ModelResponse(**response_json)
response_fields: Final[CompactifAIResponseFields] = response_json
returned_response: Final = ModelResponse(**response_fields)
# Set model name with provider prefix
returned_response.model = f"compactifai/{model}"
return returned_response
def get_error_class(self, error_message: str, status_code: int, headers: dict | httpx.Headers) -> BaseLLMException:
def get_error_class(
self, error_message: str, status_code: int, headers: dict[str, str] | httpx.Headers
) -> BaseLLMException:
"""
Get the appropriate error class for CompactifAI errors.
Since CompactifAI is OpenAI-compatible, we use OpenAI error handling.

179
litellm/llms/custom_httpx/container_handler.py
View file

@ -6,11 +6,12 @@ endpoint defined in endpoints.json, eliminating the need for individual handler
"""
import json
from collections.abc import Coroutine
from collections.abc import Coroutine, Mapping, Sequence
from pathlib import Path
from typing import TYPE_CHECKING, Any, Final
import httpx
from typing_extensions import NotRequired, ReadOnly, TypedDict
import litellm
from litellm.litellm_core_utils.url_utils import encode_url_path_segment
@ -32,26 +33,58 @@ if TYPE_CHECKING:
from litellm.llms.base_llm.containers.transformation import BaseContainerConfig
class EndpointConfig(TypedDict):
"""One endpoint entry of ``litellm/containers/endpoints.json``."""
name: ReadOnly[str]
async_name: ReadOnly[str]
path: ReadOnly[str]
method: ReadOnly[str]
path_params: ReadOnly[Sequence[str]]
query_params: ReadOnly[Sequence[str]]
response_type: ReadOnly[str]
is_multipart: NotRequired[ReadOnly[bool]]
returns_binary: NotRequired[ReadOnly[bool]]
class EndpointsConfig(TypedDict):
"""The parsed ``litellm/containers/endpoints.json`` document."""
endpoints: ReadOnly[Sequence[EndpointConfig]]
class ContainerErrorDetail(TypedDict, total=False):
"""The ``error`` object of a container API error body."""
message: ReadOnly[str]
class ContainerResponseBody(TypedDict, total=False):
"""The fields this handler reads off a container API JSON body."""
error: ReadOnly[ContainerErrorDetail]
_ContainerResponseModel = ContainerFileListResponse | ContainerFileObject | DeleteContainerFileResponse
# Response type mapping
RESPONSE_TYPES: Final[dict[str, type]] = {
RESPONSE_TYPES: Final[Mapping[str, type[_ContainerResponseModel]]] = {
"ContainerFileListResponse": ContainerFileListResponse,
"ContainerFileObject": ContainerFileObject,
"DeleteContainerFileResponse": DeleteContainerFileResponse,
}
ContainerEndpointResponse = (
ContainerFileListResponse | ContainerFileObject | DeleteContainerFileResponse | bytes | dict[str, object]
)
ContainerEndpointResponse = _ContainerResponseModel | bytes | ContainerResponseBody
def _load_endpoints_config() -> dict:
def _load_endpoints_config() -> EndpointsConfig:
"""Load the endpoints configuration from JSON file."""
config_path: Final = Path(__file__).parent.parent.parent / "containers" / "endpoints.json"
with open(config_path) as f:
return json.load(f)
def _get_endpoint_config(endpoint_name: str) -> dict | None:
def _get_endpoint_config(endpoint_name: str) -> EndpointConfig | None:
"""Get config for a specific endpoint by name."""
config: Final = _load_endpoints_config()
for endpoint in config["endpoints"]:
@ -60,10 +93,15 @@ def _get_endpoint_config(endpoint_name: str) -> dict | None:
return None
def _response_model(response_type_name: str) -> type[_ContainerResponseModel] | None:
"""The pydantic model a container endpoint's ``response_type`` names."""
return RESPONSE_TYPES.get(response_type_name)
def _build_url(
api_base: str,
path_template: str,
path_params: dict[str, str],
path_params: Mapping[str, object],
) -> str:
"""Build the full URL by substituting path parameters.
@ -93,16 +131,12 @@ def _build_url(
def _build_query_params(
query_param_names: list,
kwargs: dict[str, Any],
) -> dict[str, str]:
query_param_names: Sequence[str],
kwargs: Mapping[str, object],
) -> dict[str, object]:
"""Build query parameters from kwargs."""
params: Final = {}
for param_name in query_param_names:
value = kwargs.get(param_name)
if value is not None:
params[param_name] = str(value) if not isinstance(value, str) else value
return params
supplied: Final = ((param_name, kwargs.get(param_name)) for param_name in query_param_names)
return {name: value if isinstance(value, str) else str(value) for name, value in supplied if value is not None}
def _error_message_from_response(response: httpx.Response) -> str:
@ -136,24 +170,24 @@ def _transform_response(
if returns_binary:
return response.content
response_json: Final = response.json()
response_json: Final[ContainerResponseBody] = response.json()
if "error" in response_json:
raise BaseLLMException(
status_code=response.status_code,
message=response_json.get("error", {}).get("message", str(response_json)),
message=response_json["error"].get("message", str(response_json)),
headers=dict(response.headers),
)
response_type: Final = RESPONSE_TYPES.get(response_type_name)
response_type: Final = _response_model(response_type_name)
if response_type:
return response_type(**response_json)
return response_type.model_validate(response_json)
return response_json
def _prepare_multipart_file_upload(
file: Any,
headers: dict[str, Any],
) -> tuple:
headers: dict[str, object],
) -> tuple[dict[str, tuple[str, bytes, str]], dict[str, object]]:
"""
Prepare file and headers for multipart upload.
@ -178,6 +212,52 @@ def _prepare_multipart_file_upload(
return files, headers_copy
def _request_headers(
container_provider_config: "BaseContainerConfig",
extra_headers: dict[str, object] | None,
litellm_params: GenericLiteLLMParams,
) -> dict[str, object]:
"""The provider auth headers for a container request."""
return container_provider_config.validate_environment(
headers=extra_headers or {},
api_key=litellm_params.get("api_key", None),
)
def _request_api_base(
container_provider_config: "BaseContainerConfig",
litellm_params: GenericLiteLLMParams,
) -> str:
"""The provider base URL for a container request."""
return container_provider_config.get_complete_url(
api_base=litellm_params.get("api_base", None),
litellm_params=dict(litellm_params),
)
def _sync_http_client(
client: HTTPHandler | AsyncHTTPHandler | None,
litellm_params: GenericLiteLLMParams,
) -> HTTPHandler:
"""The sync HTTP client for a container request, reusing the caller's when usable."""
if client is None or not isinstance(client, HTTPHandler):
return _get_httpx_client(params={"ssl_verify": litellm_params.get("ssl_verify", None)})
return client
def _async_http_client(
client: HTTPHandler | AsyncHTTPHandler | None,
litellm_params: GenericLiteLLMParams,
) -> AsyncHTTPHandler:
"""The async HTTP client for a container request, reusing the caller's when usable."""
if client is None or not isinstance(client, AsyncHTTPHandler):
return get_async_httpx_client(
llm_provider=litellm.LlmProviders.OPENAI,
params={"ssl_verify": litellm_params.get("ssl_verify", None)},
)
return client
class GenericContainerHandler:
"""
Generic handler for container file API endpoints.
@ -192,13 +272,13 @@ class GenericContainerHandler:
container_provider_config: "BaseContainerConfig",
litellm_params: GenericLiteLLMParams,
logging_obj: "LiteLLMLoggingObj",
extra_headers: dict[str, Any] | None = None,
extra_query: dict[str, Any] | None = None,
extra_headers: dict[str, object] | None = None,
extra_query: dict[str, object] | None = None,
timeout: float | httpx.Timeout = 600,
_is_async: bool = False,
client: HTTPHandler | AsyncHTTPHandler | None = None,
**kwargs,
) -> Any | Coroutine[Any, Any, Any]:
**kwargs: object,
) -> Any | Coroutine[object, object, Any]:
"""
Generic handler for any container file endpoint.
@ -245,11 +325,11 @@ class GenericContainerHandler:
container_provider_config: "BaseContainerConfig",
litellm_params: GenericLiteLLMParams,
logging_obj: "LiteLLMLoggingObj",
extra_headers: dict[str, Any] | None = None,
extra_query: dict[str, Any] | None = None,
extra_headers: dict[str, object] | None = None,
extra_query: dict[str, object] | None = None,
timeout: float | httpx.Timeout = 600,
client: HTTPHandler | AsyncHTTPHandler | None = None,
**kwargs,
**kwargs: object,
) -> Any:
"""Synchronous request handler."""
endpoint_config: Final = _get_endpoint_config(endpoint_name)
@ -257,23 +337,14 @@ class GenericContainerHandler:
raise ValueError(f"Unknown endpoint: {endpoint_name}")
# Get HTTP client
if client is None or not isinstance(client, HTTPHandler):
http_client = _get_httpx_client(params={"ssl_verify": litellm_params.get("ssl_verify", None)})
else:
http_client = client
http_client: Final = _sync_http_client(client, litellm_params)
# Build request
headers = container_provider_config.validate_environment(
headers=extra_headers or {},
api_key=litellm_params.get("api_key", None),
)
headers = _request_headers(container_provider_config, extra_headers, litellm_params)
if extra_headers:
headers.update(extra_headers)
api_base: Final = container_provider_config.get_complete_url(
api_base=litellm_params.get("api_base", None),
litellm_params=dict(litellm_params),
)
api_base: Final = _request_api_base(container_provider_config, litellm_params)
# Build URL with path params
path_params: Final = {p: kwargs.get(p, "") for p in endpoint_config.get("path_params", [])}
@ -334,11 +405,11 @@ class GenericContainerHandler:
container_provider_config: "BaseContainerConfig",
litellm_params: GenericLiteLLMParams,
logging_obj: "LiteLLMLoggingObj",
extra_headers: dict[str, Any] | None = None,
extra_query: dict[str, Any] | None = None,
extra_headers: dict[str, object] | None = None,
extra_query: dict[str, object] | None = None,
timeout: float | httpx.Timeout = 600,
client: HTTPHandler | AsyncHTTPHandler | None = None,
**kwargs,
**kwargs: object,
) -> Any:
"""Asynchronous request handler."""
endpoint_config: Final = _get_endpoint_config(endpoint_name)
@ -346,26 +417,14 @@ class GenericContainerHandler:
raise ValueError(f"Unknown endpoint: {endpoint_name}")
# Get HTTP client
if client is None or not isinstance(client, AsyncHTTPHandler):
http_client = get_async_httpx_client(
llm_provider=litellm.LlmProviders.OPENAI,
params={"ssl_verify": litellm_params.get("ssl_verify", None)},
)
else:
http_client = client
http_client: Final = _async_http_client(client, litellm_params)
# Build request
headers = container_provider_config.validate_environment(
headers=extra_headers or {},
api_key=litellm_params.get("api_key", None),
)
headers = _request_headers(container_provider_config, extra_headers, litellm_params)
if extra_headers:
headers.update(extra_headers)
api_base: Final = container_provider_config.get_complete_url(
api_base=litellm_params.get("api_base", None),
litellm_params=dict(litellm_params),
)
api_base: Final = _request_api_base(container_provider_config, litellm_params)
# Build URL with path params
path_params: Final = {p: kwargs.get(p, "") for p in endpoint_config.get("path_params", [])}

6
litellm/llms/custom_httpx/http_handler.py
View file

@ -9,7 +9,7 @@ import threading
import time
from collections.abc import AsyncIterable, Callable, Iterable, Mapping
from http.cookiejar import CookieJar, DefaultCookiePolicy
from typing import TYPE_CHECKING, Any, ClassVar, Final, Optional, TypeAlias, TypedDict
from typing import TYPE_CHECKING, Any, ClassVar, Final, NoReturn, Optional, TypeAlias, TypedDict
import certifi
import httpx
@ -447,7 +447,7 @@ def _safe_read_response(response: httpx.Response, timeout: float | None = None)
return b""
def _raise_masked_sync_error(e: httpx.HTTPStatusError, stream: bool) -> None:
def _raise_masked_sync_error(e: httpx.HTTPStatusError, stream: bool) -> NoReturn:
"""Raise a MaskedHTTPStatusError for sync HTTP handlers."""
if stream:
try:
@ -467,7 +467,7 @@ def _raise_masked_sync_error(e: httpx.HTTPStatusError, stream: bool) -> None:
raise MaskedHTTPStatusError(e, message=_text, text=_text) from None
async def _raise_masked_async_error(e: httpx.HTTPStatusError, stream: bool) -> None:
async def _raise_masked_async_error(e: httpx.HTTPStatusError, stream: bool) -> NoReturn:
"""Raise a MaskedHTTPStatusError for async HTTP handlers."""
if stream:
try:

2
litellm/llms/custom_httpx/llm_http_handler.py
View file

@ -2872,6 +2872,7 @@ class BaseLLMHTTPHandler:
headers=headers,
timeout=timeout or float(response_api_optional_request_params.get("timeout", 0)),
stream=stream,
logging_obj=logging_obj,
**body_kwargs,
)
@ -2903,6 +2904,7 @@ class BaseLLMHTTPHandler:
url=api_base,
headers=headers,
timeout=timeout or float(response_api_optional_request_params.get("timeout", 0)),
logging_obj=logging_obj,
**body_kwargs,
)

35
litellm/llms/e2b/sandbox/transformation.py
View file

@ -8,7 +8,7 @@ Talks to e2b's REST API directly over httpx (no e2b SDK dependency):
"""
import json
from typing import Final, cast
from typing import Final
import httpx
@ -68,13 +68,10 @@ class E2BSandboxConfig(BaseSandboxConfig):
if metadata:
body["metadata"] = metadata
response: Final = cast(
httpx.Response,
await self._http(client).post(
url=f"{base}/sandboxes",
headers={"X-API-Key": key, "Content-Type": "application/json"},
json=body,
),
response: Final = await self._http(client).post(
url=f"{base}/sandboxes",
headers={"X-API-Key": key, "Content-Type": "application/json"},
json=body,
)
data: Final = response.json()
@ -117,14 +114,11 @@ class E2BSandboxConfig(BaseSandboxConfig):
headers["E2B-Traffic-Access-Token"] = traffic_token
url: Final = f"https://{JUPYTER_PORT}-{handle.id}.{handle.domain}/execute"
response: Final = cast(
httpx.Response,
await self._http(client).post(
url=url,
headers=headers,
json={"code": code, "context_id": None, "env_vars": env_vars},
stream=True,
),
response: Final = await self._http(client).post(
url=url,
headers=headers,
json={"code": code, "context_id": None, "env_vars": env_vars},
stream=True,
)
lines: Final = await self._read_capped_lines(response)
return self._parse_lines(lines)
@ -142,12 +136,9 @@ class E2BSandboxConfig(BaseSandboxConfig):
key: Final = api_key or handle._hidden_params.get("api_key") or self.validate_environment()
base: Final = api_base or handle._hidden_params.get("api_base") or E2B_API_BASE
try:
response: Final = cast(
httpx.Response,
await self._http(client).delete(
url=f"{base}/sandboxes/{handle.id}",
headers={"X-API-Key": key},
),
response: Final = await self._http(client).delete(
url=f"{base}/sandboxes/{handle.id}",
headers={"X-API-Key": key},
)
except httpx.HTTPStatusError as e:
if e.response.status_code == 404:

45
litellm/llms/gdc/chat/transformation.py
View file

@ -6,11 +6,25 @@ import json
import os
import re
import threading
from typing import Any, Final
from collections.abc import Callable
from typing import Any, Final, Protocol
from urllib.parse import urlsplit
import litellm
from litellm.llms.openai_like.chat.transformation import OpenAILikeChatConfig
from litellm.types.llms.openai import AllMessageValues
class _GDCHAudienceCredentials(Protocol):
"""A GDCH service account credential already bound to an audience, ready to mint a bearer token."""
@property
def valid(self) -> bool: ...
@property
def token(self) -> str: ...
def refresh(self, request: object) -> None: ...
class GDCGeminiConfig(OpenAILikeChatConfig):
@ -21,7 +35,7 @@ class GDCGeminiConfig(OpenAILikeChatConfig):
def __init__(self, **kwargs: Any) -> None:
super().__init__(**kwargs)
self._creds_lock = threading.Lock()
self._gdch_creds_cache: dict = {}
self._gdch_creds_cache: dict[tuple[str, str], _GDCHAudienceCredentials] = {}
def get_supported_openai_params(self, model: str) -> list:
return [
@ -110,7 +124,7 @@ class GDCGeminiConfig(OpenAILikeChatConfig):
return f"{api_base}/v1/projects/{project}/locations/{location}/chat/completions"
def _read_env_bool(self, val: Any, env_var: str, default: bool = True) -> bool | str:
def _read_env_bool(self, val: bool | str | None, env_var: str, default: bool = True) -> bool | str:
def _parse(s: str) -> bool | str:
cleaned: Final = s.strip().lower()
if cleaned in ("false", "0", "no", "off"):
@ -129,7 +143,7 @@ class GDCGeminiConfig(OpenAILikeChatConfig):
return default
return _parse(_env_val)
def _fetch_auth(self, gdch_creds: Any, ssl_verify: bool | str) -> None:
def _fetch_auth(self, gdch_creds: _GDCHAudienceCredentials, ssl_verify: bool | str) -> None:
import requests
from google.auth.transport import requests as auth_requests
@ -138,13 +152,24 @@ class GDCGeminiConfig(OpenAILikeChatConfig):
auth_request: Final = auth_requests.Request(session=auth_session)
gdch_creds.refresh(auth_request)
def _cached_fetch_token(self, creds: Any, audience: str, ssl_verify: bool | str, api_key: str | None = None) -> str:
def _with_gdch_audience(self, creds: object, audience: str) -> _GDCHAudienceCredentials:
"""The credential rebound to ``audience``, which GDCH requires before a token refresh."""
bind_audience: Final[Callable[[str], _GDCHAudienceCredentials] | None] = getattr(
creds, "with_gdch_audience", None
)
if bind_audience is None:
raise AttributeError("GDC credentials must expose with_gdch_audience to be bound to a request audience")
return bind_audience(audience)
def _cached_fetch_token(
self, creds: object, audience: str, ssl_verify: bool | str, api_key: str | None = None
) -> str:
# Key cache by both audience and credential identity to prevent cross-caller contamination
cache_key: Final = (audience.rstrip("/"), api_key or str(id(creds)))
with self._creds_lock:
if cache_key not in self._gdch_creds_cache:
self._gdch_creds_cache[cache_key] = creds.with_gdch_audience(audience.rstrip("/"))
self._gdch_creds_cache[cache_key] = self._with_gdch_audience(creds, audience.rstrip("/"))
gdch_creds: Final = self._gdch_creds_cache[cache_key]
@ -155,7 +180,7 @@ class GDCGeminiConfig(OpenAILikeChatConfig):
return token
def _load_creds_from_key(self, api_key: str) -> tuple[Any, bool]:
def _load_creds_from_key(self, api_key: str) -> tuple[object | None, bool]:
import google.auth
try:
@ -175,7 +200,7 @@ class GDCGeminiConfig(OpenAILikeChatConfig):
self,
headers: dict,
model: str,
messages: list[Any],
messages: list[AllMessageValues],
optional_params: dict,
litellm_params: dict,
api_key: str | None = None,
@ -230,7 +255,7 @@ class GDCGeminiConfig(OpenAILikeChatConfig):
if self._read_env_bool(litellm_params.get("gdc_token_caching"), "GDC_TOKEN_CACHING", default=False):
token = self._cached_fetch_token(creds, audience, ssl_verify, api_key)
else:
gdch_creds: Final = creds.with_gdch_audience(audience)
gdch_creds: Final = self._with_gdch_audience(creds, audience)
self._fetch_auth(gdch_creds, ssl_verify)
token = gdch_creds.token
headers["Authorization"] = f"Bearer {token}"
@ -252,7 +277,7 @@ class GDCGeminiConfig(OpenAILikeChatConfig):
def transform_request(
self,
model: str,
messages: list[Any],
messages: list[AllMessageValues],
optional_params: dict,
litellm_params: dict,
headers: dict,

30
litellm/llms/infinity/rerank/transformation.py
View file

@ -4,10 +4,11 @@ Transformation logic from Cohere's /v1/rerank format to Infinity's `/v1/rerank`
Why separate file? Make it easy to see how transformation works
"""
from collections.abc import Mapping
from collections.abc import Mapping, Sequence
from typing import Final
import httpx
from typing_extensions import NotRequired, ReadOnly, TypedDict
import litellm
from litellm._uuid import uuid
@ -26,6 +27,31 @@ from litellm.types.rerank import (
from ..common_utils import InfinityError
class _InfinityRerankUsage(TypedDict, extra_items=ReadOnly[int]):
"""The token counters Infinity reports in the ``usage`` block of a rerank response."""
class _InfinityRerankResult(TypedDict):
"""One scored document in an Infinity ``/v1/rerank`` response."""
index: ReadOnly[int]
relevance_score: ReadOnly[float]
document: ReadOnly[str]
class _InfinityRerankResponse(TypedDict):
"""The JSON body returned by Infinity's ``/v1/rerank`` endpoint."""
id: ReadOnly[NotRequired[str]]
usage: ReadOnly[NotRequired[_InfinityRerankUsage]]
results: ReadOnly[Sequence[_InfinityRerankResult]]
def _parse_rerank_response(raw_response: httpx.Response) -> _InfinityRerankResponse:
"""Read the untyped JSON body of an Infinity rerank response."""
return raw_response.json()
class InfinityRerankConfig(CohereRerankConfig):
def get_complete_url(
self,
@ -82,7 +108,7 @@ class InfinityRerankConfig(CohereRerankConfig):
No transformation required, Infinity follows Cohere API response format
"""
try:
raw_response_json: Final = raw_response.json()
raw_response_json: Final = _parse_rerank_response(raw_response)
except Exception:
raise InfinityError(message=raw_response.text, status_code=raw_response.status_code)

76
litellm/llms/litellm_proxy/skills/code_execution.py
View file

@ -13,12 +13,57 @@ Generated files are returned directly in the response - no separate storage need
import base64
import json
from collections.abc import Sequence
from enum import Enum
from typing import Any, Final
from typing import Any, Final, Protocol
from typing_extensions import NotRequired, ReadOnly, TypedDict
from litellm._logging import verbose_logger
class _ToolCallFunction(Protocol):
"""Function payload of an assistant tool call."""
name: str | None
arguments: str
class _ToolCall(Protocol):
"""Tool call requested by the assistant on a chat completion choice."""
id: str
function: _ToolCallFunction
class _AssistantMessage(Protocol):
"""Assistant message carried by a chat completion choice."""
content: str | None
tool_calls: Sequence[_ToolCall] | None
class _CompletionChoice(Protocol):
"""Single choice of a chat completion response."""
finish_reason: str
message: _AssistantMessage
class _SandboxFile(TypedDict):
"""File generated inside the sandbox during a code execution run."""
name: ReadOnly[str]
mime_type: ReadOnly[str]
content_base64: ReadOnly[str]
class _CodeExecutionArguments(TypedDict):
"""Arguments the model passes to the `litellm_code_execution` tool."""
code: NotRequired[ReadOnly[str]]
class LiteLLMInternalTools(str, Enum):
"""
Enum for internal LiteLLM tools that are injected into requests.
@ -30,7 +75,7 @@ class LiteLLMInternalTools(str, Enum):
CODE_EXECUTION = "litellm_code_execution"
def get_litellm_code_execution_tool() -> dict[str, Any]:
def get_litellm_code_execution_tool() -> dict[str, object]:
"""
Returns the litellm_code_execution tool definition in OpenAI format.
@ -51,7 +96,7 @@ def get_litellm_code_execution_tool() -> dict[str, Any]:
}
def get_litellm_code_execution_tool_anthropic() -> dict[str, Any]:
def get_litellm_code_execution_tool_anthropic() -> dict[str, object]:
"""
Returns the litellm_code_execution tool definition in Anthropic/messages API format.
@ -103,7 +148,7 @@ class CodeExecutionHandler:
skill_files: dict[str, bytes],
skill_id: str | None = None,
**kwargs,
) -> dict[str, Any]:
) -> dict[str, object]:
"""
Execute an LLM call with automatic code execution handling.
@ -134,8 +179,8 @@ class CodeExecutionHandler:
)
current_messages: Final = list(messages)
generated_files: Final[list[dict[str, Any]]] = [] # Files returned directly
execution_results: Final[list[dict]] = []
generated_files: Final[list[dict[str, object]]] = [] # Files returned directly
execution_results: Final[list[dict[str, object]]] = []
executor: Final = SkillsSandboxExecutor(timeout=self.sandbox_timeout)
response: Any = None # Initialize to avoid possibly unbound error
@ -151,11 +196,12 @@ class CodeExecutionHandler:
**kwargs,
)
assistant_message = response.choices[0].message
stop_reason = response.choices[0].finish_reason
choice: _CompletionChoice = response.choices[0]
assistant_message = choice.message
stop_reason: str = choice.finish_reason
# Build assistant message for conversation history
assistant_msg_dict: dict[str, Any] = {
assistant_msg_dict: dict[str, object] = {
"role": "assistant",
"content": assistant_message.content,
}
@ -190,8 +236,8 @@ class CodeExecutionHandler:
if tool_name == LiteLLMInternalTools.CODE_EXECUTION.value:
# Execute code in sandbox
try:
args = json.loads(tool_call.function.arguments)
code = args.get("code", "")
args: _CodeExecutionArguments = json.loads(tool_call.function.arguments)
code: str = args.get("code", "")
verbose_logger.debug("CodeExecutionHandler: Executing code (%s chars)", len(code))
@ -202,13 +248,15 @@ class CodeExecutionHandler:
verbose_logger.debug("CodeExecutionHandler: Execution result: %s", exec_result)
sandbox_files: Sequence[_SandboxFile] = exec_result["files"]
execution_results.append(
{
"iteration": iteration,
"success": exec_result["success"],
"output": exec_result["output"],
"error": exec_result["error"],
"files": [f["name"] for f in exec_result["files"]],
"files": [f["name"] for f in sandbox_files],
}
)
@ -216,9 +264,9 @@ class CodeExecutionHandler:
tool_result = exec_result["output"] or ""
# Collect generated files (returned directly, no storage)
if exec_result["files"]:
if sandbox_files:
tool_result += "\n\nGenerated files:"
for f in exec_result["files"]:
for f in sandbox_files:
file_content = base64.b64decode(f["content_base64"])
# Add to generated files list (returned in response)
generated_files.append(

36
litellm/llms/ollama/completion/handler.py
View file

@ -4,16 +4,32 @@ Ollama /chat/completion calls handled in llm_http_handler.py
[TODO]: migrate embeddings to a base handler as well.
"""
from typing import Any, Final
from collections.abc import Mapping, Sequence
from typing import Any, Final, Protocol, TypedDict
from typing_extensions import NotRequired, ReadOnly
import litellm
from litellm.types.utils import EmbeddingResponse
class TokenEncoder(Protocol):
"""The tokenizer surface used to estimate prompt tokens."""
def encode(self, text: str, /) -> Sequence[int]: ...
class OllamaEmbeddingResponse(TypedDict):
"""Body of an Ollama ``/api/embed`` response."""
embeddings: ReadOnly[list[list[float]]]
prompt_eval_count: ReadOnly[NotRequired[int]]
def _prepare_ollama_embedding_payload(
model: str, prompts: list[str], optional_params: dict[str, Any]
) -> dict[str, Any]:
data: Final[dict[str, Any]] = {"model": model, "input": prompts}
model: str, prompts: list[str], optional_params: Mapping[str, object]
) -> dict[str, object]:
data: Final[dict[str, object]] = {"model": model, "input": prompts}
special_optional_params: Final = ["truncate", "options", "keep_alive", "dimensions"]
for k, v in optional_params.items():
@ -27,12 +43,12 @@ def _prepare_ollama_embedding_payload(
def _process_ollama_embedding_response(
response_json: dict,
response_json: OllamaEmbeddingResponse,
prompts: list[str],
model: str,
model_response: EmbeddingResponse,
logging_obj: Any,
encoding: Any,
encoding: TokenEncoder | None,
) -> EmbeddingResponse:
output_data: Final = []
embeddings: Final[list[list[float]]] = response_json["embeddings"]
@ -72,7 +88,7 @@ async def ollama_aembeddings(
model_response: EmbeddingResponse,
optional_params: dict,
logging_obj: Any,
encoding: Any,
encoding: TokenEncoder | None,
):
if not api_base.endswith("/api/embed"):
api_base += "/api/embed"
@ -80,7 +96,7 @@ async def ollama_aembeddings(
data: Final = _prepare_ollama_embedding_payload(model, prompts, optional_params)
response: Final = await litellm.module_level_aclient.post(url=api_base, json=data)
response_json: Final = response.json()
response_json: Final[OllamaEmbeddingResponse] = response.json()
return _process_ollama_embedding_response(
response_json=response_json,
@ -99,7 +115,7 @@ def ollama_embeddings(
optional_params: dict,
model_response: EmbeddingResponse,
logging_obj: Any,
encoding: Any = None,
encoding: TokenEncoder | None = None,
):
if not api_base.endswith("/api/embed"):
api_base += "/api/embed"
@ -107,7 +123,7 @@ def ollama_embeddings(
data: Final = _prepare_ollama_embedding_payload(model, prompts, optional_params)
response: Final = litellm.module_level_client.post(url=api_base, json=data)
response_json: Final = response.json()
response_json: Final[OllamaEmbeddingResponse] = response.json()
return _process_ollama_embedding_response(
response_json=response_json,

72
litellm/llms/openai/containers/transformation.py
View file

@ -1,6 +1,8 @@
from typing import TYPE_CHECKING, Any, Final
from collections.abc import Mapping
from typing import TYPE_CHECKING, Any, Final, Literal
import httpx
from typing_extensions import ReadOnly, TypedDict
import litellm
from litellm.litellm_core_utils.llm_cost_calc.tool_call_cost_tracking import (
@ -11,9 +13,11 @@ from litellm.secret_managers.main import get_secret_str
from litellm.types.containers.main import (
ContainerCreateOptionalRequestParams,
ContainerFileListResponse,
ContainerFileObject,
ContainerListResponse,
ContainerObject,
DeleteContainerResult,
ExpiresAfter,
)
from litellm.types.router import GenericLiteLLMParams
@ -32,6 +36,46 @@ else:
BaseLLMException = Any
class OpenAIContainerPayload(TypedDict):
"""The JSON body OpenAI returns for a single container."""
id: ReadOnly[str]
object: ReadOnly[Literal["container"]]
created_at: ReadOnly[int]
status: ReadOnly[str]
expires_after: ReadOnly[ExpiresAfter | None]
last_active_at: ReadOnly[int | None]
name: ReadOnly[str | None]
class OpenAIContainerListPayload(TypedDict):
"""The JSON body OpenAI returns for a page of containers."""
object: ReadOnly[Literal["list"]]
data: ReadOnly[list[ContainerObject]]
first_id: ReadOnly[str | None]
last_id: ReadOnly[str | None]
has_more: ReadOnly[bool]
class OpenAIContainerDeletedPayload(TypedDict):
"""The JSON body OpenAI returns for a deleted container."""
id: ReadOnly[str]
object: ReadOnly[Literal["container.deleted"]]
deleted: ReadOnly[bool]
class OpenAIContainerFileListPayload(TypedDict):
"""The JSON body OpenAI returns for a page of container files."""
object: ReadOnly[Literal["list"]]
data: ReadOnly[list[ContainerFileObject]]
first_id: ReadOnly[str | None]
last_id: ReadOnly[str | None]
has_more: ReadOnly[bool]
class OpenAIContainerConfig(BaseContainerConfig):
"""Configuration class for OpenAI container API."""
@ -87,7 +131,7 @@ class OpenAIContainerConfig(BaseContainerConfig):
def transform_container_create_request(
self,
name: str,
container_create_optional_request_params: dict,
container_create_optional_request_params: Mapping[str, object],
litellm_params: GenericLiteLLMParams,
headers: dict,
) -> dict:
@ -111,7 +155,7 @@ class OpenAIContainerConfig(BaseContainerConfig):
logging_obj: LiteLLMLoggingObj,
) -> ContainerObject:
"""Transform the OpenAI container creation response."""
response_data: Final = raw_response.json()
response_data: Final[OpenAIContainerPayload] = raw_response.json()
# Transform the response data
container_obj: Final = ContainerObject(**response_data)
@ -140,7 +184,7 @@ class OpenAIContainerConfig(BaseContainerConfig):
after: str | None = None,
limit: int | None = None,
order: str | None = None,
extra_query: dict[str, Any] | None = None,
extra_query: Mapping[str, object] | None = None,
) -> tuple[str, dict]:
"""Transform the container list request for OpenAI API.
@ -151,7 +195,7 @@ class OpenAIContainerConfig(BaseContainerConfig):
url: Final = api_base
# Prepare query parameters
params: Final = {}
params: Final[dict[str, object]] = {}
if after is not None:
params["after"] = after
if limit is not None:
@ -171,7 +215,7 @@ class OpenAIContainerConfig(BaseContainerConfig):
logging_obj: LiteLLMLoggingObj,
) -> ContainerListResponse:
"""Transform the OpenAI container list response."""
response_data: Final = raw_response.json()
response_data: Final[OpenAIContainerListPayload] = raw_response.json()
# Transform the response data
container_list: Final = ContainerListResponse(**response_data)
@ -191,7 +235,7 @@ class OpenAIContainerConfig(BaseContainerConfig):
url: Final = join_container_api_base_path(api_base, f"/{encoded_container_id}")
# No additional data needed for GET request
data: Final[dict[str, Any]] = {}
data: Final[dict[str, object]] = {}
return url, data
@ -201,7 +245,7 @@ class OpenAIContainerConfig(BaseContainerConfig):
logging_obj: LiteLLMLoggingObj,
) -> ContainerObject:
"""Transform the OpenAI container retrieve response."""
response_data: Final = raw_response.json()
response_data: Final[OpenAIContainerPayload] = raw_response.json()
# Transform the response data
container_obj: Final = ContainerObject(**response_data)
@ -224,7 +268,7 @@ class OpenAIContainerConfig(BaseContainerConfig):
url: Final = join_container_api_base_path(api_base, f"/{encoded_container_id}")
# No data needed for DELETE request
data: Final[dict[str, Any]] = {}
data: Final[dict[str, object]] = {}
return url, data
@ -234,7 +278,7 @@ class OpenAIContainerConfig(BaseContainerConfig):
logging_obj: LiteLLMLoggingObj,
) -> DeleteContainerResult:
"""Transform the OpenAI container delete response."""
response_data: Final = raw_response.json()
response_data: Final[OpenAIContainerDeletedPayload] = raw_response.json()
# Transform the response data
delete_result: Final = DeleteContainerResult(**response_data)
@ -250,7 +294,7 @@ class OpenAIContainerConfig(BaseContainerConfig):
after: str | None = None,
limit: int | None = None,
order: str | None = None,
extra_query: dict[str, Any] | None = None,
extra_query: Mapping[str, object] | None = None,
) -> tuple[str, dict]:
"""Transform the container file list request for OpenAI API.
@ -262,7 +306,7 @@ class OpenAIContainerConfig(BaseContainerConfig):
url: Final = join_container_api_base_path(api_base, f"/{encoded_container_id}/files")
# Prepare query parameters
params: Final[dict[str, Any]] = {}
params: Final[dict[str, object]] = {}
if after is not None:
params["after"] = after
if limit is not None:
@ -282,7 +326,7 @@ class OpenAIContainerConfig(BaseContainerConfig):
logging_obj: LiteLLMLoggingObj,
) -> ContainerFileListResponse:
"""Transform the OpenAI container file list response."""
response_data: Final = raw_response.json()
response_data: Final[OpenAIContainerFileListPayload] = raw_response.json()
# Transform the response data
file_list: Final = ContainerFileListResponse(**response_data)
@ -308,7 +352,7 @@ class OpenAIContainerConfig(BaseContainerConfig):
url: Final = join_container_api_base_path(api_base, f"/{encoded_container_id}/files/{encoded_file_id}/content")
# No query parameters needed
params: Final[dict[str, Any]] = {}
params: Final[dict[str, object]] = {}
return url, params

57
litellm/llms/opensandbox/sandbox/transformation.py
View file

@ -1,7 +1,7 @@
import asyncio
import json
import time
from typing import Final, cast
from typing import Final
import httpx
@ -86,13 +86,10 @@ class OpenSandboxSandboxConfig(BaseSandboxConfig):
secure_access=secure_access,
)
response: Final = cast(
httpx.Response,
await self._http(client).post(
url=f"{base}/sandboxes",
headers=self._lifecycle_headers(key),
json=body,
),
response: Final = await self._http(client).post(
url=f"{base}/sandboxes",
headers=self._lifecycle_headers(key),
json=body,
)
data: Final = response.json()
sandbox_id: Final = str(data["id"])
@ -182,12 +179,9 @@ class OpenSandboxSandboxConfig(BaseSandboxConfig):
base: Final = str(handle._hidden_params.get("api_base") or self._api_base(api_base))
key: Final = self._api_key(api_key=api_key, handle=handle)
try:
response: Final = cast(
httpx.Response,
await self._http(client).delete(
url=f"{base}/sandboxes/{handle.id}",
headers=self._lifecycle_headers(key),
),
response: Final = await self._http(client).delete(
url=f"{base}/sandboxes/{handle.id}",
headers=self._lifecycle_headers(key),
)
except httpx.HTTPStatusError as e:
if e.response.status_code == 404:
@ -245,12 +239,9 @@ class OpenSandboxSandboxConfig(BaseSandboxConfig):
) -> None:
deadline: Final = time.monotonic() + ready_timeout
while True:
response = cast(
httpx.Response,
await self._http(client).get(
url=f"{api_base}/sandboxes/{sandbox_id}",
headers=headers,
),
response = await self._http(client).get(
url=f"{api_base}/sandboxes/{sandbox_id}",
headers=headers,
)
data = response.json()
state = self._sandbox_state(data)
@ -306,13 +297,10 @@ class OpenSandboxSandboxConfig(BaseSandboxConfig):
use_server_proxy: bool,
client: AsyncHTTPHandler | None,
) -> tuple[str, dict[str, str]]:
response: Final = cast(
httpx.Response,
await self._http(client).get(
url=f"{api_base}/sandboxes/{sandbox_id}/endpoints/{OPEN_SANDBOX_EXECD_PORT}",
headers=headers,
params={"use_server_proxy": use_server_proxy},
),
response: Final = await self._http(client).get(
url=f"{api_base}/sandboxes/{sandbox_id}/endpoints/{OPEN_SANDBOX_EXECD_PORT}",
headers=headers,
params={"use_server_proxy": use_server_proxy},
)
data: Final = response.json()
endpoint: Final = data.get("endpoint")
@ -329,15 +317,12 @@ class OpenSandboxSandboxConfig(BaseSandboxConfig):
client: AsyncHTTPHandler | None,
) -> list[str]:
timeout: Final = httpx.Timeout(connect=30.0, read=None, write=30.0, pool=None)
response: Final = cast(
httpx.Response,
await self._http(client).post(
url=url,
headers=headers,
timeout=timeout,
json=body,
stream=True,
),
response: Final = await self._http(client).post(
url=url,
headers=headers,
timeout=timeout,
json=body,
stream=True,
)
return await self._read_capped_lines(response)

8
litellm/llms/vertex_ai/vector_stores/rag_api/transformation.py
View file

@ -97,7 +97,7 @@ class VertexVectorStoreConfig(BaseVectorStoreConfig, VertexBase):
def __init__(self):
super().__init__()
def get_auth_credentials(self, litellm_params: dict) -> BaseVectorStoreAuthCredentials:
def get_auth_credentials(self, litellm_params: Mapping[str, object]) -> BaseVectorStoreAuthCredentials:
# Get credentials and project info
vertex_credentials: Final = self.get_vertex_ai_credentials(dict(litellm_params))
vertex_project: Final = self.get_vertex_ai_project(dict(litellm_params))
@ -122,7 +122,9 @@ class VertexVectorStoreConfig(BaseVectorStoreConfig, VertexBase):
"write": [("POST", "/ragCorpora")],
}
def validate_environment(self, headers: dict, litellm_params: GenericLiteLLMParams | None) -> dict:
def validate_environment(
self, headers: dict[str, str], litellm_params: GenericLiteLLMParams | None
) -> dict[str, str]:
"""
Validate and set up authentication for Vertex AI RAG API
"""
@ -135,7 +137,7 @@ class VertexVectorStoreConfig(BaseVectorStoreConfig, VertexBase):
def get_complete_url(
self,
api_base: str | None,
litellm_params: dict,
litellm_params: dict[str, object],
) -> str:
"""
Get the Base endpoint for Vertex AI RAG API

2
litellm/models/base.py
View file

@ -33,6 +33,6 @@ class DomainModel(BaseModel):
return cls(**record.dict())
return cls(**dict(record))
def to_db_dict(self, exclude_unset: bool = False) -> dict[str, Any]:
def to_db_dict(self, exclude_unset: bool = False) -> dict[str, object]:
"""Convert domain model to a dictionary for database operations."""
return self.model_dump(exclude_none=True, exclude_unset=exclude_unset)

28
litellm/proxy/_experimental/mcp_server/db.py
View file

@ -600,23 +600,19 @@ async def get_all_mcp_servers(
NULL approval_status predates the approval workflow, so those rows are kept explicitly rather
than dropped by a bare inequality, which SQL evaluates as NULL and would silently hide them.
"""
try:
where: Final[prisma_db_types.LiteLLM_MCPServerTableWhereInput] = (
{"approval_status": approval_status}
if approval_status is not None
# mutable-ok: prisma where-inputs must be plain dicts, and both `NOT` and `not` drop
# NULL rows (measured), so the OR is the only NULL-preserving way to exclude drafts
else {"OR": [{"approval_status": None}, {"approval_status": {"not": MCPApprovalStatus.draft}}]}
)
mcp_servers: Final = await _db_find_mcp_server_rows(prisma_client, where)
where: Final[prisma_db_types.LiteLLM_MCPServerTableWhereInput] = (
{"approval_status": approval_status}
if approval_status is not None
# mutable-ok: prisma where-inputs must be plain dicts, and both `NOT` and `not` drop
# NULL rows (measured), so the OR is the only NULL-preserving way to exclude drafts
else {"OR": [{"approval_status": None}, {"approval_status": {"not": MCPApprovalStatus.draft}}]}
)
mcp_servers: Final = await _db_find_mcp_server_rows(prisma_client, where)
tables: Final = [LiteLLM_MCPServerTable.model_validate(mcp_server.model_dump()) for mcp_server in mcp_servers]
for table in tables:
decrypt_global_env_var_values(table.env_vars)
return tables
except Exception as e:
verbose_proxy_logger.debug("litellm.proxy._experimental.mcp_server.db.py::get_all_mcp_servers - %s", e)
return []
tables: Final = [LiteLLM_MCPServerTable.model_validate(mcp_server.model_dump()) for mcp_server in mcp_servers]
for table in tables:
decrypt_global_env_var_values(table.env_vars)
return tables
async def get_mcp_server(prisma_client: PrismaClient, server_id: str) -> LiteLLM_MCPServerTable | None:

16
litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py
View file

@ -1096,8 +1096,7 @@ async def exchange_token_with_server(
headers={"Accept": "application/json", **token_request.headers},
data=token_data,
)
if response is not None:
response.raise_for_status()
response.raise_for_status()
except httpx.HTTPStatusError as exc:
fault: Final = classify_upstream_token_rejection(
exc.response,
@ -1119,11 +1118,6 @@ async def exchange_token_with_server(
)
return _bridge_mint_error_response("invalid_refresh")
return render_token_fault(fault)
if response is None:
raise HTTPException(
status_code=502,
detail="MCP upstream token endpoint returned no response",
)
token_response = response.json()
# Validate token response against server-configured rules before any storage.
@ -1536,16 +1530,10 @@ async def _post_dcr_registration(
headers=headers,
json=register_data,
)
if response is not None:
response.raise_for_status()
response.raise_for_status()
except httpx.HTTPStatusError as exc:
status_code, detail = dcr_fault_detail(classify_upstream_dcr_rejection(exc.response, log_context=server_id))
raise HTTPException(status_code=status_code, detail=detail) from exc
if response is None:
raise HTTPException(
status_code=502,
detail="MCP upstream registration endpoint returned no response",
)
return response

63
litellm/proxy/_experimental/mcp_server/oauth2_flow_backfill.py
View file

@ -36,7 +36,10 @@ a healed fleet has no null rows and the backfill exits after one query.
import json
from collections import Counter
from typing import Any, Final, Literal
from collections.abc import Mapping, Sequence
from typing import Final, Literal, Protocol
from pydantic import JsonValue
from litellm._logging import verbose_proxy_logger
from litellm.proxy._experimental.mcp_server.db import _decode_oauth_payload, decrypt_credentials
@ -55,9 +58,59 @@ BackfillRule = Literal[
_BACKFILL_AUDIT_ACTOR: Final = "oauth2_flow_backfill"
def _decrypted_credentials(raw_credentials: Any) -> MCPCredentials | None:
class _MCPServerRow(Protocol):
"""The ``LiteLLM_MCPServerTable`` columns this backfill reads."""
@property
def server_id(self) -> str: ...
@property
def authorization_url(self) -> str | None: ...
@property
def registration_url(self) -> str | None: ...
@property
def token_url(self) -> str | None: ...
@property
def credentials(self) -> str | Mapping[str, JsonValue] | None: ...
class _MCPUserCredentialRow(Protocol):
"""The ``LiteLLM_MCPUserCredentials`` columns this backfill reads."""
@property
def server_id(self) -> str: ...
@property
def credential_b64(self) -> str: ...
class _MCPServerTable(Protocol):
async def find_many(self, *, where: Mapping[str, object]) -> Sequence[_MCPServerRow]: ...
async def update_many(self, *, where: Mapping[str, object], data: Mapping[str, str]) -> object: ...
class _MCPUserCredentialsTable(Protocol):
async def find_many(self, *, where: Mapping[str, object]) -> Sequence[_MCPUserCredentialRow]: ...
def _mcp_server_table(prisma_client: PrismaClient) -> _MCPServerTable:
"""The MCP server table, typed so the untyped prisma client surface stops here."""
return prisma_client.db.litellm_mcpservertable
def _mcp_user_credentials_table(prisma_client: PrismaClient) -> _MCPUserCredentialsTable:
"""The per-user MCP credential table, typed so the untyped prisma client surface stops here."""
return prisma_client.db.litellm_mcpusercredentials
def _decrypted_credentials(raw_credentials: str | Mapping[str, JsonValue] | None) -> MCPCredentials | None:
if raw_credentials is None:
return None
parsed: JsonValue | Mapping[str, JsonValue]
if isinstance(raw_credentials, str):
try:
parsed = json.loads(raw_credentials)
@ -92,14 +145,14 @@ def classify_null_flow_row(
async def backfill_null_oauth2_flows(prisma_client: PrismaClient) -> dict[BackfillRule, int]:
"""Classify every ``auth_type=oauth2`` row whose ``oauth2_flow`` is null; stamp the provable
ones, warn on the ambiguous ones, and return counts per rule."""
null_rows: Final[list[Any]] = await prisma_client.db.litellm_mcpservertable.find_many(
null_rows: Final[Sequence[_MCPServerRow]] = await _mcp_server_table(prisma_client).find_many(
where={"auth_type": "oauth2", "oauth2_flow": None},
)
if not null_rows:
return {}
server_ids: Final = [row.server_id for row in null_rows]
token_rows: Final[list[Any]] = await prisma_client.db.litellm_mcpusercredentials.find_many(
token_rows: Final[Sequence[_MCPUserCredentialRow]] = await _mcp_user_credentials_table(prisma_client).find_many(
where={"server_id": {"in": server_ids}},
)
server_ids_with_oauth_tokens: Final[set[str]] = {
@ -141,7 +194,7 @@ async def backfill_null_oauth2_flows(prisma_client: PrismaClient) -> dict[Backfi
stamped_flows: Final = {flow for _, (flow, _) in classified if flow is not None}
for stamped_flow in stamped_flows:
server_ids_for_flow = [row.server_id for row, (row_flow, _) in classified if row_flow == stamped_flow]
await prisma_client.db.litellm_mcpservertable.update_many(
await _mcp_server_table(prisma_client).update_many(
where={"server_id": {"in": server_ids_for_flow}, "oauth2_flow": None},
data={"oauth2_flow": stamped_flow, "updated_by": _BACKFILL_AUDIT_ACTOR},
)

17
litellm/proxy/_experimental/mcp_server/outbound_credentials/client_credentials.py
View file

@ -19,9 +19,9 @@ Implements the client-credentials behavior contract for the v2 resolver:
identity.
The token-endpoint POST is injected (``M2MTokenEndpointPost``) so the grant orchestration is
testable without a live IdP; ``post_client_credentials_grant`` is the httpx edge and the one
place the untyped response boundary is contained. Failures are values: the source returns
``Result[OAuthToken, CredError]``; only the httpx edge touches exceptions.
testable without a live IdP; ``post_client_credentials_grant`` is the httpx edge. Failures are
values: the source returns ``Result[OAuthToken, CredError]``; only the httpx edge touches
exceptions.
"""
from __future__ import annotations
@ -95,18 +95,17 @@ async def post_client_credentials_grant(
) -> TokenEndpointOutcome:
"""POST the grant to the token endpoint and classify the transport outcome.
The httpx edge: litellm's handler is partially typed (and raises ``HTTPStatusError`` itself on
a 4xx/5xx), so the untyped boundary is contained here and every field the caller reads comes
out of a validated ``TokenEndpointOutcome``.
The httpx edge: litellm's handler raises ``HTTPStatusError`` itself on a 4xx/5xx, and every
field the caller reads comes out of a validated ``TokenEndpointOutcome``.
"""
from litellm.llms.custom_httpx.http_handler import ( # noqa: PLC0415 # defer heavy handler import to call time
get_async_httpx_client, # pyright: ignore[reportUnknownVariableType] # handler is partially typed
get_async_httpx_client, # pyright: ignore[reportUnknownVariableType] # handler factory params are coarsely typed
)
from litellm.types.llms.custom_http import httpxSpecialProvider # noqa: PLC0415 # deferred with the handler import
try:
client: Final = get_async_httpx_client(llm_provider=httpxSpecialProvider.Oauth2Check)
response = await client.post( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # handler is partially typed
response: Final = await client.post( # pyright: ignore[reportUnknownMemberType] # handler params are coarsely typed
url, headers={"Accept": "application/json", **headers}, data=form
)
except httpx.HTTPStatusError as status_err:
@ -114,8 +113,6 @@ async def post_client_credentials_grant(
return TokenEndpointDenied(status_code=status_code, detail=f"token endpoint returned HTTP {status_code}")
except Exception as exc: # noqa: BLE001 # any transport failure is the same outcome: unreachable
return TokenEndpointUnreachable(detail=str(exc))
if not isinstance(response, httpx.Response):
return TokenEndpointUnreachable(detail="token endpoint returned no response")
try:
body: Final = _TOKEN_BODY_ADAPTER.validate_json(response.content)
except ValidationError:

7
litellm/proxy/_experimental/mcp_server/outbound_credentials/token_endpoint.py
View file

@ -111,9 +111,6 @@ class TokenEndpointClient:
return Error(
CredError.of_upstream_unavailable("token exchange failed: token endpoint returned a non-JSON response")
)
if raw is None:
verbose_proxy_logger.warning("MCP token endpoint %s returned no response", endpoint)
return Error(CredError.of_upstream_unavailable("token exchange failed: no response from token endpoint"))
try:
parsed: Final = _TokenEndpointResponse.model_validate(raw)
except ValidationError:
@ -199,7 +196,7 @@ def _cache_ttl_seconds(expires_in: int | None) -> int:
)
async def _post_form(endpoint: str, data: dict[str, str]) -> object | None:
async def _post_form(endpoint: str, data: dict[str, str]) -> object:
# litellm's httpx handler and httpx.Response are only partially typed; the token endpoint
# returns a JSON object that `_TokenEndpointResponse` validates, so the untyped boundary is
# contained here. A non-2xx raises `httpx.HTTPStatusError`, an unreachable endpoint raises
@ -208,8 +205,6 @@ async def _post_form(endpoint: str, data: dict[str, str]) -> object | None:
# each to a CredError.
client = get_async_httpx_client(llm_provider=httpxSpecialProvider.MCP) # pyright: ignore[reportUnknownVariableType] # litellm http handler is untyped
response = await client.post(endpoint, data=data) # pyright: ignore[reportUnknownMemberType,reportUnknownVariableType] # litellm http handler is untyped
if response is None:
return None
response.raise_for_status()
return response.json() # pyright: ignore[reportAny] # untyped JSON; validated by _TokenEndpointResponse in fetch

93
litellm/proxy/_experimental/mcp_server/toolset_db.py
View file

@ -1,5 +1,9 @@
import json
from typing import Final
from collections.abc import Mapping, Sequence
from datetime import datetime
from typing import Final, Protocol
from typing_extensions import NotRequired, ReadOnly, TypedDict
from litellm._logging import verbose_proxy_logger
from litellm._uuid import uuid
@ -7,18 +11,73 @@ from litellm.proxy.utils import PrismaClient
from litellm.repositories.table_repositories import MCPToolsetRepository
from litellm.types.mcp_server.mcp_toolset import (
MCPToolset,
MCPToolsetTool,
NewMCPToolsetRequest,
UpdateMCPToolsetRequest,
)
def _toolset_from_row(row) -> MCPToolset:
class MCPToolsetFields(TypedDict):
"""The ``MCPToolset`` constructor keywords a toolset row expands into."""
toolset_id: ReadOnly[str]
toolset_name: ReadOnly[str]
description: NotRequired[ReadOnly[str | None]]
tools: NotRequired[ReadOnly[list[MCPToolsetTool]]]
created_at: NotRequired[ReadOnly[datetime | None]]
created_by: NotRequired[ReadOnly[str | None]]
updated_at: NotRequired[ReadOnly[datetime | None]]
updated_by: NotRequired[ReadOnly[str | None]]
class MCPToolsetRowData(TypedDict):
"""A toolset table row, whose ``tools`` column is stored as JSON."""
toolset_id: ReadOnly[str]
toolset_name: ReadOnly[str]
description: NotRequired[ReadOnly[str | None]]
tools: NotRequired[ReadOnly[str | list[MCPToolsetTool]]]
created_at: NotRequired[ReadOnly[datetime | None]]
created_by: NotRequired[ReadOnly[str | None]]
updated_at: NotRequired[ReadOnly[datetime | None]]
updated_by: NotRequired[ReadOnly[str | None]]
class MCPToolsetRow(Protocol):
"""A row of the toolset table, as the prisma client returns it."""
def model_dump(self) -> MCPToolsetRowData: ...
class MCPToolsetTable(Protocol):
"""The prisma table actions this module runs against the toolset table."""
async def create(self, data: Mapping[str, object]) -> MCPToolsetRow: ...
async def find_unique(self, where: Mapping[str, object]) -> MCPToolsetRow | None: ...
async def find_first(self, where: Mapping[str, object]) -> MCPToolsetRow | None: ...
async def find_many(self, where: Mapping[str, object]) -> Sequence[MCPToolsetRow]: ...
async def update(self, where: Mapping[str, object], data: Mapping[str, object]) -> MCPToolsetRow: ...
async def delete(self, where: Mapping[str, object]) -> MCPToolsetRow: ...
def _toolset_table(prisma_client: PrismaClient) -> MCPToolsetTable:
"""The toolset table actions of the prisma client."""
return MCPToolsetRepository(prisma_client).table
def _toolset_from_row(row: MCPToolsetRow) -> MCPToolset:
data: Final = row.model_dump()
tools = data.get("tools") or []
if isinstance(tools, str):
tools = json.loads(tools)
data["tools"] = tools
return MCPToolset(**data)
tools: Final = data.get("tools") or []
resolved: Final[MCPToolsetFields] = {
**data,
"tools": json.loads(tools) if isinstance(tools, str) else tools,
}
return MCPToolset(**resolved)
async def create_mcp_toolset(
@ -31,7 +90,7 @@ async def create_mcp_toolset(
data_dict["tools"] = json.dumps(data_dict.get("tools", []))
data_dict["created_by"] = touched_by
data_dict["updated_by"] = touched_by
row: Final = await MCPToolsetRepository(prisma_client).table.create(data=data_dict)
row: Final = await _toolset_table(prisma_client).create(data=data_dict)
return _toolset_from_row(row)
@ -39,7 +98,7 @@ async def get_mcp_toolset(
prisma_client: PrismaClient,
toolset_id: str,
) -> MCPToolset | None:
row: Final = await MCPToolsetRepository(prisma_client).table.find_unique(where={"toolset_id": toolset_id})
row: Final = await _toolset_table(prisma_client).find_unique(where={"toolset_id": toolset_id})
if row is None:
return None
return _toolset_from_row(row)
@ -47,13 +106,11 @@ async def get_mcp_toolset(
async def list_mcp_toolsets(
prisma_client: PrismaClient,
toolset_ids: list[str] | None = None,
) -> list[MCPToolset]:
toolset_ids: Sequence[str] | None = None,
) -> Sequence[MCPToolset]:
try:
where = {}
if toolset_ids is not None:
where = {"toolset_id": {"in": toolset_ids}}
rows: Final = await MCPToolsetRepository(prisma_client).table.find_many(where=where)
where: Final[Mapping[str, object]] = {} if toolset_ids is None else {"toolset_id": {"in": toolset_ids}}
rows: Final = await _toolset_table(prisma_client).find_many(where=where)
return [_toolset_from_row(r) for r in rows]
except Exception as e:
verbose_proxy_logger.warning("litellm.proxy._experimental.mcp_server.toolset_db::list_mcp_toolsets - %s", e)
@ -64,7 +121,7 @@ async def get_mcp_toolset_by_name(
prisma_client: PrismaClient,
toolset_name: str,
) -> MCPToolset | None:
row: Final = await MCPToolsetRepository(prisma_client).table.find_first(where={"toolset_name": toolset_name})
row: Final = await _toolset_table(prisma_client).find_first(where={"toolset_name": toolset_name})
if row is None:
return None
return _toolset_from_row(row)
@ -80,7 +137,7 @@ async def update_mcp_toolset(
data_dict["tools"] = json.dumps(data_dict["tools"])
data_dict["updated_by"] = touched_by
try:
row: Final = await MCPToolsetRepository(prisma_client).table.update(
row: Final = await _toolset_table(prisma_client).update(
where={"toolset_id": data.toolset_id},
data=data_dict,
)
@ -98,7 +155,7 @@ async def delete_mcp_toolset(
toolset_id: str,
) -> MCPToolset | None:
try:
row: Final = await MCPToolsetRepository(prisma_client).table.delete(where={"toolset_id": toolset_id})
row: Final = await _toolset_table(prisma_client).delete(where={"toolset_id": toolset_id})
except Exception as e:
from prisma.errors import RecordNotFoundError

2
litellm/proxy/_experimental/out/404.html
View file

File diff suppressed because one or more lines are too long

2
litellm/proxy/_experimental/out/404/index.html
View file

File diff suppressed because one or more lines are too long

8
litellm/proxy/_experimental/out/__next.!KGRhc2hib2FyZCk.__PAGE__.txt
View file

@ -1,9 +1,9 @@
1:"$Sreact.fragment"
2:I[347257,["/litellm-asset-prefix/_next/static/chunks/3_06chgeyldml.js","/litellm-asset-prefix/_next/static/chunks/26h-ny89yaww0.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"ClientPageRoot"]
3:I[871135,["/litellm-asset-prefix/_next/static/chunks/3_06chgeyldml.js","/litellm-asset-prefix/_next/static/chunks/26h-ny89yaww0.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js","/litellm-asset-prefix/_next/static/chunks/37t2cfzl_b58p.js","/litellm-asset-prefix/_next/static/chunks/1zf358k334atp.js","/litellm-asset-prefix/_next/static/chunks/3kil-7y33kpm9.js","/litellm-asset-prefix/_next/static/chunks/2i0218zvrsasm.js","/litellm-asset-prefix/_next/static/chunks/3ytz29phknzsy.js","/litellm-asset-prefix/_next/static/chunks/0aoel7yrv88fp.js","/litellm-asset-prefix/_next/static/chunks/1natmx9lu3mus.js","/litellm-asset-prefix/_next/static/chunks/1dmg55q8kht9j.js","/litellm-asset-prefix/_next/static/chunks/1lrl_8p0h2sbm.js","/litellm-asset-prefix/_next/static/chunks/1e-4-g6x6zyse.js","/litellm-asset-prefix/_next/static/chunks/0gygfcpmiijl8.js","/litellm-asset-prefix/_next/static/chunks/2cu4j3g1tldv4.js","/litellm-asset-prefix/_next/static/chunks/2kmqjpt047tjo.js","/litellm-asset-prefix/_next/static/chunks/1phty1k2nx8fx.js","/litellm-asset-prefix/_next/static/chunks/0u3cfuz-tf0wj.js","/litellm-asset-prefix/_next/static/chunks/39s4-rh6l9sa1.js","/litellm-asset-prefix/_next/static/chunks/0dylouuq8ak8p.js","/litellm-asset-prefix/_next/static/chunks/3a3jpg95umjho.js","/litellm-asset-prefix/_next/static/chunks/28hnu_qv5e_c_.js","/litellm-asset-prefix/_next/static/chunks/0qf1_0kt4uuxa.js","/litellm-asset-prefix/_next/static/chunks/0ab_ntohf1wik.js","/litellm-asset-prefix/_next/static/chunks/3kpec-qy1uzod.js"],"default"]
6:I[897367,["/litellm-asset-prefix/_next/static/chunks/3_06chgeyldml.js","/litellm-asset-prefix/_next/static/chunks/26h-ny89yaww0.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"OutletBoundary"]
2:I[347257,["/litellm-asset-prefix/_next/static/chunks/3s39b43k2vde7.js","/litellm-asset-prefix/_next/static/chunks/3kest3gurc9op.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"ClientPageRoot"]
3:I[871135,["/litellm-asset-prefix/_next/static/chunks/3s39b43k2vde7.js","/litellm-asset-prefix/_next/static/chunks/3kest3gurc9op.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js","/litellm-asset-prefix/_next/static/chunks/1a3mamulxkyhw.js","/litellm-asset-prefix/_next/static/chunks/2yqxc2yxa1-go.js","/litellm-asset-prefix/_next/static/chunks/27gjrlkmq245y.js","/litellm-asset-prefix/_next/static/chunks/0aoel7yrv88fp.js","/litellm-asset-prefix/_next/static/chunks/1nfnjvxf_0-3n.js","/litellm-asset-prefix/_next/static/chunks/2i0218zvrsasm.js","/litellm-asset-prefix/_next/static/chunks/34_wtpkkvqa3n.js","/litellm-asset-prefix/_next/static/chunks/3ytz29phknzsy.js","/litellm-asset-prefix/_next/static/chunks/1l61r88q65pjd.js","/litellm-asset-prefix/_next/static/chunks/0di-9qm-8ex8r.js","/litellm-asset-prefix/_next/static/chunks/3dy-3uqjux30s.js","/litellm-asset-prefix/_next/static/chunks/3wdy9040h4b13.js","/litellm-asset-prefix/_next/static/chunks/13rzpi4q1z_e8.js","/litellm-asset-prefix/_next/static/chunks/0wh5uu7sl34-i.js","/litellm-asset-prefix/_next/static/chunks/0cotqb-2hzyvs.js","/litellm-asset-prefix/_next/static/chunks/16xdxq7qvv37h.js","/litellm-asset-prefix/_next/static/chunks/0dylouuq8ak8p.js","/litellm-asset-prefix/_next/static/chunks/2tj1x2xl0npv1.js","/litellm-asset-prefix/_next/static/chunks/2eonl4rcemkdj.js","/litellm-asset-prefix/_next/static/chunks/0z7zg9587od6_.js","/litellm-asset-prefix/_next/static/chunks/0ab_ntohf1wik.js","/litellm-asset-prefix/_next/static/chunks/1v3m908ycsmt4.js"],"default"]
6:I[897367,["/litellm-asset-prefix/_next/static/chunks/3s39b43k2vde7.js","/litellm-asset-prefix/_next/static/chunks/3kest3gurc9op.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"OutletBoundary"]
7:"$Sreact.suspense"
0:{"rsc":["$","$1","c",{"children":[["$","$L2",null,{"Component":"$3","serverProvidedParams":{"searchParams":{},"params":{},"promises":["$@4","$@5"]}}],[["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/1e-4-g6x6zyse.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/0gygfcpmiijl8.js","async":true}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/2cu4j3g1tldv4.js","async":true}],["$","script","script-3",{"src":"/litellm-asset-prefix/_next/static/chunks/2kmqjpt047tjo.js","async":true}],["$","script","script-4",{"src":"/litellm-asset-prefix/_next/static/chunks/1phty1k2nx8fx.js","async":true}],["$","script","script-5",{"src":"/litellm-asset-prefix/_next/static/chunks/0u3cfuz-tf0wj.js","async":true}],["$","script","script-6",{"src":"/litellm-asset-prefix/_next/static/chunks/39s4-rh6l9sa1.js","async":true}],["$","script","script-7",{"src":"/litellm-asset-prefix/_next/static/chunks/0dylouuq8ak8p.js","async":true}],["$","script","script-8",{"src":"/litellm-asset-prefix/_next/static/chunks/3a3jpg95umjho.js","async":true}],["$","script","script-9",{"src":"/litellm-asset-prefix/_next/static/chunks/28hnu_qv5e_c_.js","async":true}],["$","script","script-10",{"src":"/litellm-asset-prefix/_next/static/chunks/0qf1_0kt4uuxa.js","async":true}],["$","script","script-11",{"src":"/litellm-asset-prefix/_next/static/chunks/0ab_ntohf1wik.js","async":true}],["$","script","script-12",{"src":"/litellm-asset-prefix/_next/static/chunks/3kpec-qy1uzod.js","async":true}]],["$","$L6",null,{"children":["$","$7",null,{"name":"Next.MetadataOutlet","children":"$@8"}]}]]}],"isPartial":false,"staleTime":300,"varyParams":null,"buildId":"0cE25rDXvGu3tj4HWOGKy"}
0:{"rsc":["$","$1","c",{"children":[["$","$L2",null,{"Component":"$3","serverProvidedParams":{"searchParams":{},"params":{},"promises":["$@4","$@5"]}}],[["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/0di-9qm-8ex8r.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/3dy-3uqjux30s.js","async":true}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/3wdy9040h4b13.js","async":true}],["$","script","script-3",{"src":"/litellm-asset-prefix/_next/static/chunks/13rzpi4q1z_e8.js","async":true}],["$","script","script-4",{"src":"/litellm-asset-prefix/_next/static/chunks/0wh5uu7sl34-i.js","async":true}],["$","script","script-5",{"src":"/litellm-asset-prefix/_next/static/chunks/0cotqb-2hzyvs.js","async":true}],["$","script","script-6",{"src":"/litellm-asset-prefix/_next/static/chunks/16xdxq7qvv37h.js","async":true}],["$","script","script-7",{"src":"/litellm-asset-prefix/_next/static/chunks/0dylouuq8ak8p.js","async":true}],["$","script","script-8",{"src":"/litellm-asset-prefix/_next/static/chunks/2tj1x2xl0npv1.js","async":true}],["$","script","script-9",{"src":"/litellm-asset-prefix/_next/static/chunks/2eonl4rcemkdj.js","async":true}],["$","script","script-10",{"src":"/litellm-asset-prefix/_next/static/chunks/0z7zg9587od6_.js","async":true}],["$","script","script-11",{"src":"/litellm-asset-prefix/_next/static/chunks/0ab_ntohf1wik.js","async":true}],["$","script","script-12",{"src":"/litellm-asset-prefix/_next/static/chunks/1v3m908ycsmt4.js","async":true}]],["$","$L6",null,{"children":["$","$7",null,{"name":"Next.MetadataOutlet","children":"$@8"}]}]]}],"isPartial":false,"staleTime":300,"varyParams":null,"buildId":"YAsRgSxdV-OcBfib_67Dt"}
4:{}
5:"$0:rsc:props:children:0:props:serverProvidedParams:params"
8:null

10
litellm/proxy/_experimental/out/__next.!KGRhc2hib2FyZCk.txt
View file

@ -1,7 +1,7 @@
1:"$Sreact.fragment"
2:I[92825,["/litellm-asset-prefix/_next/static/chunks/3_06chgeyldml.js","/litellm-asset-prefix/_next/static/chunks/26h-ny89yaww0.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"ClientSegmentRoot"]
3:I[216370,["/litellm-asset-prefix/_next/static/chunks/3_06chgeyldml.js","/litellm-asset-prefix/_next/static/chunks/26h-ny89yaww0.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js","/litellm-asset-prefix/_next/static/chunks/37t2cfzl_b58p.js","/litellm-asset-prefix/_next/static/chunks/1zf358k334atp.js","/litellm-asset-prefix/_next/static/chunks/3kil-7y33kpm9.js","/litellm-asset-prefix/_next/static/chunks/2i0218zvrsasm.js","/litellm-asset-prefix/_next/static/chunks/3ytz29phknzsy.js","/litellm-asset-prefix/_next/static/chunks/0aoel7yrv88fp.js","/litellm-asset-prefix/_next/static/chunks/1natmx9lu3mus.js","/litellm-asset-prefix/_next/static/chunks/1dmg55q8kht9j.js","/litellm-asset-prefix/_next/static/chunks/1lrl_8p0h2sbm.js"],"default"]
4:I[339756,["/litellm-asset-prefix/_next/static/chunks/3_06chgeyldml.js","/litellm-asset-prefix/_next/static/chunks/26h-ny89yaww0.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"default"]
5:I[837457,["/litellm-asset-prefix/_next/static/chunks/3_06chgeyldml.js","/litellm-asset-prefix/_next/static/chunks/26h-ny89yaww0.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"default"]
0:{"rsc":["$","$1","c",{"children":[[["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/37t2cfzl_b58p.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/1zf358k334atp.js","async":true}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/3kil-7y33kpm9.js","async":true}],["$","script","script-3",{"src":"/litellm-asset-prefix/_next/static/chunks/2i0218zvrsasm.js","async":true}],["$","script","script-4",{"src":"/litellm-asset-prefix/_next/static/chunks/3ytz29phknzsy.js","async":true}],["$","script","script-5",{"src":"/litellm-asset-prefix/_next/static/chunks/0aoel7yrv88fp.js","async":true}],["$","script","script-6",{"src":"/litellm-asset-prefix/_next/static/chunks/1natmx9lu3mus.js","async":true}],["$","script","script-7",{"src":"/litellm-asset-prefix/_next/static/chunks/1dmg55q8kht9j.js","async":true}],["$","script","script-8",{"src":"/litellm-asset-prefix/_next/static/chunks/1lrl_8p0h2sbm.js","async":true}]],["$","$L2",null,{"Component":"$3","slots":{"children":["$","$L4",null,{"parallelRouterKey":"children","template":["$","$L5",null,{}],"notFound":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":404}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],[]]}]},"serverProvidedParams":{"params":{},"promises":["$@6"]}}]]}],"isPartial":false,"staleTime":300,"varyParams":null,"buildId":"0cE25rDXvGu3tj4HWOGKy"}
2:I[92825,["/litellm-asset-prefix/_next/static/chunks/3s39b43k2vde7.js","/litellm-asset-prefix/_next/static/chunks/3kest3gurc9op.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"ClientSegmentRoot"]
3:I[216370,["/litellm-asset-prefix/_next/static/chunks/3s39b43k2vde7.js","/litellm-asset-prefix/_next/static/chunks/3kest3gurc9op.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js","/litellm-asset-prefix/_next/static/chunks/1a3mamulxkyhw.js","/litellm-asset-prefix/_next/static/chunks/2yqxc2yxa1-go.js","/litellm-asset-prefix/_next/static/chunks/27gjrlkmq245y.js","/litellm-asset-prefix/_next/static/chunks/0aoel7yrv88fp.js","/litellm-asset-prefix/_next/static/chunks/1nfnjvxf_0-3n.js","/litellm-asset-prefix/_next/static/chunks/2i0218zvrsasm.js","/litellm-asset-prefix/_next/static/chunks/34_wtpkkvqa3n.js","/litellm-asset-prefix/_next/static/chunks/3ytz29phknzsy.js","/litellm-asset-prefix/_next/static/chunks/1l61r88q65pjd.js"],"default"]
4:I[339756,["/litellm-asset-prefix/_next/static/chunks/3s39b43k2vde7.js","/litellm-asset-prefix/_next/static/chunks/3kest3gurc9op.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"default"]
5:I[837457,["/litellm-asset-prefix/_next/static/chunks/3s39b43k2vde7.js","/litellm-asset-prefix/_next/static/chunks/3kest3gurc9op.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"default"]
0:{"rsc":["$","$1","c",{"children":[[["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/1a3mamulxkyhw.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/2yqxc2yxa1-go.js","async":true}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/27gjrlkmq245y.js","async":true}],["$","script","script-3",{"src":"/litellm-asset-prefix/_next/static/chunks/0aoel7yrv88fp.js","async":true}],["$","script","script-4",{"src":"/litellm-asset-prefix/_next/static/chunks/1nfnjvxf_0-3n.js","async":true}],["$","script","script-5",{"src":"/litellm-asset-prefix/_next/static/chunks/2i0218zvrsasm.js","async":true}],["$","script","script-6",{"src":"/litellm-asset-prefix/_next/static/chunks/34_wtpkkvqa3n.js","async":true}],["$","script","script-7",{"src":"/litellm-asset-prefix/_next/static/chunks/3ytz29phknzsy.js","async":true}],["$","script","script-8",{"src":"/litellm-asset-prefix/_next/static/chunks/1l61r88q65pjd.js","async":true}]],["$","$L2",null,{"Component":"$3","slots":{"children":["$","$L4",null,{"parallelRouterKey":"children","template":["$","$L5",null,{}],"notFound":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":404}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],[]]}]},"serverProvidedParams":{"params":{},"promises":["$@6"]}}]]}],"isPartial":false,"staleTime":300,"varyParams":null,"buildId":"YAsRgSxdV-OcBfib_67Dt"}
6:"$0:rsc:props:children:1:props:serverProvidedParams:params"

40
litellm/proxy/_experimental/out/__next._full.txt
View file

File diff suppressed because one or more lines are too long

8
litellm/proxy/_experimental/out/__next._head.txt
View file

@ -1,6 +1,6 @@
1:"$Sreact.fragment"
2:I[897367,["/litellm-asset-prefix/_next/static/chunks/3_06chgeyldml.js","/litellm-asset-prefix/_next/static/chunks/26h-ny89yaww0.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"ViewportBoundary"]
3:I[897367,["/litellm-asset-prefix/_next/static/chunks/3_06chgeyldml.js","/litellm-asset-prefix/_next/static/chunks/26h-ny89yaww0.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"MetadataBoundary"]
2:I[897367,["/litellm-asset-prefix/_next/static/chunks/3s39b43k2vde7.js","/litellm-asset-prefix/_next/static/chunks/3kest3gurc9op.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"ViewportBoundary"]
3:I[897367,["/litellm-asset-prefix/_next/static/chunks/3s39b43k2vde7.js","/litellm-asset-prefix/_next/static/chunks/3kest3gurc9op.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"MetadataBoundary"]
4:"$Sreact.suspense"
5:I[27201,["/litellm-asset-prefix/_next/static/chunks/3_06chgeyldml.js","/litellm-asset-prefix/_next/static/chunks/26h-ny89yaww0.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"IconMark"]
0:{"rsc":["$","$1","h",{"children":[null,["$","$L2",null,{"children":[["$","meta","0",{"charSet":"utf-8"}],["$","meta","1",{"name":"viewport","content":"width=device-width, initial-scale=1"}]]}],["$","div",null,{"hidden":true,"children":["$","$L3",null,{"children":["$","$4",null,{"name":"Next.Metadata","children":[["$","title","0",{"children":"LiteLLM Dashboard"}],["$","meta","1",{"name":"description","content":"LiteLLM Proxy Admin UI"}],["$","link","2",{"rel":"icon","href":"/favicon.ico?favicon.3arlap5n8tyzg.ico","sizes":"48x48","type":"image/x-icon"}],["$","link","3",{"rel":"icon","href":"/get_favicon"}],["$","$L5","4",{}]]}]}]}],["$","meta",null,{"name":"next-size-adjust","content":""}]]}],"isPartial":false,"staleTime":300,"varyParams":null,"buildId":"0cE25rDXvGu3tj4HWOGKy"}
5:I[27201,["/litellm-asset-prefix/_next/static/chunks/3s39b43k2vde7.js","/litellm-asset-prefix/_next/static/chunks/3kest3gurc9op.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"IconMark"]
0:{"rsc":["$","$1","h",{"children":[null,["$","$L2",null,{"children":[["$","meta","0",{"charSet":"utf-8"}],["$","meta","1",{"name":"viewport","content":"width=device-width, initial-scale=1"}]]}],["$","div",null,{"hidden":true,"children":["$","$L3",null,{"children":["$","$4",null,{"name":"Next.Metadata","children":[["$","title","0",{"children":"LiteLLM Dashboard"}],["$","meta","1",{"name":"description","content":"LiteLLM Proxy Admin UI"}],["$","link","2",{"rel":"icon","href":"/favicon.ico?favicon.3arlap5n8tyzg.ico","sizes":"48x48","type":"image/x-icon"}],["$","link","3",{"rel":"icon","href":"/get_favicon"}],["$","$L5","4",{}]]}]}]}],["$","meta",null,{"name":"next-size-adjust","content":""}]]}],"isPartial":false,"staleTime":300,"varyParams":null,"buildId":"YAsRgSxdV-OcBfib_67Dt"}

18
litellm/proxy/_experimental/out/__next._index.txt
View file

@ -1,11 +1,11 @@
1:"$Sreact.fragment"
2:I[363178,["/litellm-asset-prefix/_next/static/chunks/3_06chgeyldml.js","/litellm-asset-prefix/_next/static/chunks/26h-ny89yaww0.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"ThemeProvider"]
3:I[12985,["/litellm-asset-prefix/_next/static/chunks/3_06chgeyldml.js","/litellm-asset-prefix/_next/static/chunks/26h-ny89yaww0.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"NuqsAdapter"]
4:I[867271,["/litellm-asset-prefix/_next/static/chunks/3_06chgeyldml.js","/litellm-asset-prefix/_next/static/chunks/26h-ny89yaww0.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"default"]
5:I[557951,["/litellm-asset-prefix/_next/static/chunks/3_06chgeyldml.js","/litellm-asset-prefix/_next/static/chunks/26h-ny89yaww0.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"AuthProvider"]
6:I[339756,["/litellm-asset-prefix/_next/static/chunks/3_06chgeyldml.js","/litellm-asset-prefix/_next/static/chunks/26h-ny89yaww0.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"default"]
7:I[837457,["/litellm-asset-prefix/_next/static/chunks/3_06chgeyldml.js","/litellm-asset-prefix/_next/static/chunks/26h-ny89yaww0.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"default"]
8:I[713354,["/litellm-asset-prefix/_next/static/chunks/3_06chgeyldml.js","/litellm-asset-prefix/_next/static/chunks/26h-ny89yaww0.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"Toaster"]
2:I[363178,["/litellm-asset-prefix/_next/static/chunks/3s39b43k2vde7.js","/litellm-asset-prefix/_next/static/chunks/3kest3gurc9op.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"ThemeProvider"]
3:I[12985,["/litellm-asset-prefix/_next/static/chunks/3s39b43k2vde7.js","/litellm-asset-prefix/_next/static/chunks/3kest3gurc9op.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"NuqsAdapter"]
4:I[867271,["/litellm-asset-prefix/_next/static/chunks/3s39b43k2vde7.js","/litellm-asset-prefix/_next/static/chunks/3kest3gurc9op.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"default"]
5:I[557951,["/litellm-asset-prefix/_next/static/chunks/3s39b43k2vde7.js","/litellm-asset-prefix/_next/static/chunks/3kest3gurc9op.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"AuthProvider"]
6:I[339756,["/litellm-asset-prefix/_next/static/chunks/3s39b43k2vde7.js","/litellm-asset-prefix/_next/static/chunks/3kest3gurc9op.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"default"]
7:I[837457,["/litellm-asset-prefix/_next/static/chunks/3s39b43k2vde7.js","/litellm-asset-prefix/_next/static/chunks/3kest3gurc9op.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"default"]
8:I[713354,["/litellm-asset-prefix/_next/static/chunks/3s39b43k2vde7.js","/litellm-asset-prefix/_next/static/chunks/3kest3gurc9op.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"Toaster"]
:HL["/litellm-asset-prefix/_next/static/chunks/1kid9zr1--h6y.css","style"]
:HL["/litellm-asset-prefix/_next/static/chunks/2hicgq-mjp8vy.css","style"]
0:{"rsc":["$","$1","c",{"children":[[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/1kid9zr1--h6y.css","precedence":"next"}],["$","link","1",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/2hicgq-mjp8vy.css","precedence":"next"}],["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/3_06chgeyldml.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/26h-ny89yaww0.js","async":true}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js","async":true}]],["$","html",null,{"lang":"en","suppressHydrationWarning":true,"children":["$","body",null,{"className":"inter_5972bc34-module__OU16Qa__className","children":["$","$L2",null,{"attribute":"class","defaultTheme":"light","enableSystem":true,"disableTransitionOnChange":true,"children":["$","$L3",null,{"children":["$","$L4",null,{"children":[["$","$L5",null,{"children":["$","$L6",null,{"parallelRouterKey":"children","template":["$","$L7",null,{}],"notFound":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":404}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],[]]}]}],["$","$L8",null,{}]]}]}]}]}]}]]}],"isPartial":false,"staleTime":300,"varyParams":null,"buildId":"0cE25rDXvGu3tj4HWOGKy"}
:HL["/litellm-asset-prefix/_next/static/chunks/3idmblk6vi8i5.css","style"]
0:{"rsc":["$","$1","c",{"children":[[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/1kid9zr1--h6y.css","precedence":"next"}],["$","link","1",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/3idmblk6vi8i5.css","precedence":"next"}],["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/3s39b43k2vde7.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/3kest3gurc9op.js","async":true}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js","async":true}]],["$","html",null,{"lang":"en","suppressHydrationWarning":true,"children":["$","body",null,{"className":"inter_5972bc34-module__OU16Qa__className","children":["$","$L2",null,{"attribute":"class","defaultTheme":"light","enableSystem":true,"disableTransitionOnChange":true,"children":["$","$L3",null,{"children":["$","$L4",null,{"children":[["$","$L5",null,{"children":["$","$L6",null,{"parallelRouterKey":"children","template":["$","$L7",null,{}],"notFound":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":404}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],[]]}]}],["$","$L8",null,{}]]}]}]}]}]}]]}],"isPartial":false,"staleTime":300,"varyParams":null,"buildId":"YAsRgSxdV-OcBfib_67Dt"}

4
litellm/proxy/_experimental/out/__next._tree.txt
View file

@ -1,4 +1,4 @@
:HL["/litellm-asset-prefix/_next/static/chunks/1kid9zr1--h6y.css","style"]
:HL["/litellm-asset-prefix/_next/static/chunks/2hicgq-mjp8vy.css","style"]
:HL["/litellm-asset-prefix/_next/static/chunks/3idmblk6vi8i5.css","style"]
:HL["/litellm-asset-prefix/_next/static/media/83afe278b6a6bb3c-s.p.2bn3s6zvc0dyp.woff2","font",{"crossOrigin":"","type":"font/woff2"}]
0:{"tree":{"name":"","param":null,"prefetchHints":16,"slots":{"children":{"name":"(dashboard)","param":null,"prefetchHints":0,"slots":{"children":{"name":"__PAGE__","param":null,"prefetchHints":0,"slots":null}}}}},"staleTime":300,"buildId":"0cE25rDXvGu3tj4HWOGKy"}
0:{"tree":{"name":"","param":null,"prefetchHints":16,"slots":{"children":{"name":"(dashboard)","param":null,"prefetchHints":0,"slots":{"children":{"name":"__PAGE__","param":null,"prefetchHints":0,"slots":null}}}}},"staleTime":300,"buildId":"YAsRgSxdV-OcBfib_67Dt"}

0
litellm/proxy/_experimental/out/_next/static/0cE25rDXvGu3tj4HWOGKy/_buildManifest.js → litellm/proxy/_experimental/out/_next/static/YAsRgSxdV-OcBfib_67Dt/_buildManifest.js
View file

0
litellm/proxy/_experimental/out/_next/static/0cE25rDXvGu3tj4HWOGKy/_clientMiddlewareManifest.js → litellm/proxy/_experimental/out/_next/static/YAsRgSxdV-OcBfib_67Dt/_clientMiddlewareManifest.js
View file

0
litellm/proxy/_experimental/out/_next/static/0cE25rDXvGu3tj4HWOGKy/_ssgManifest.js → litellm/proxy/_experimental/out/_next/static/YAsRgSxdV-OcBfib_67Dt/_ssgManifest.js
View file

1
litellm/proxy/_experimental/out/_next/static/chunks/0-dst_pi7co_a.js Normal file
View file

File diff suppressed because one or more lines are too long

1
litellm/proxy/_experimental/out/_next/static/chunks/0002gr7w0f3nn.js Normal file
View file

File diff suppressed because one or more lines are too long

1
litellm/proxy/_experimental/out/_next/static/chunks/00dvxqp6f0f6s.js Normal file
View file

File diff suppressed because one or more lines are too long

1
litellm/proxy/_experimental/out/_next/static/chunks/00mhzot068d2m.js
View file

File diff suppressed because one or more lines are too long

1
litellm/proxy/_experimental/out/_next/static/chunks/01benr9g1pe74.js Normal file
View file

File diff suppressed because one or more lines are too long

1
litellm/proxy/_experimental/out/_next/static/chunks/01i10m3msnar9.js
View file

File diff suppressed because one or more lines are too long

1
litellm/proxy/_experimental/out/_next/static/chunks/01t0ca9m9cblp.js
View file

File diff suppressed because one or more lines are too long

1
litellm/proxy/_experimental/out/_next/static/chunks/022gv-s8rsuep.js
View file

File diff suppressed because one or more lines are too long

1
litellm/proxy/_experimental/out/_next/static/chunks/027qywrv12iu8.js Normal file
View file

File diff suppressed because one or more lines are too long

1
litellm/proxy/_experimental/out/_next/static/chunks/02aj56rzfo-nr.js
View file

File diff suppressed because one or more lines are too long

1
litellm/proxy/_experimental/out/_next/static/chunks/03fte74pbliq5.js
View file

File diff suppressed because one or more lines are too long

1
litellm/proxy/_experimental/out/_next/static/chunks/03k5rtnvgsg9q.js Normal file
View file

File diff suppressed because one or more lines are too long

1
litellm/proxy/_experimental/out/_next/static/chunks/054k4q5uh06vi.js
View file

File diff suppressed because one or more lines are too long

2
litellm/proxy/_experimental/out/_next/static/chunks/1vjk83xj1xp-n.js → litellm/proxy/_experimental/out/_next/static/chunks/05jpqw44c6aj2.js
View file

File diff suppressed because one or more lines are too long

1
litellm/proxy/_experimental/out/_next/static/chunks/05jtp8xqp3j0x.js Normal file
View file

File diff suppressed because one or more lines are too long

1
litellm/proxy/_experimental/out/_next/static/chunks/05qxpjomf8mhm.js
View file

File diff suppressed because one or more lines are too long

1
litellm/proxy/_experimental/out/_next/static/chunks/079c6mpwr9q3x.js
View file

File diff suppressed because one or more lines are too long

1
litellm/proxy/_experimental/out/_next/static/chunks/07vruwfvhfop5.js
View file

File diff suppressed because one or more lines are too long

1
litellm/proxy/_experimental/out/_next/static/chunks/07y20ohq6ygp4.js Normal file
View file

File diff suppressed because one or more lines are too long

2
litellm/proxy/_experimental/out/_next/static/chunks/2tqkirw-qhcfg.js → litellm/proxy/_experimental/out/_next/static/chunks/08-1iq_vq49mx.js
View file

File diff suppressed because one or more lines are too long

1
litellm/proxy/_experimental/out/_next/static/chunks/08lua1iopk_79.js Normal file
View file

File diff suppressed because one or more lines are too long

1
litellm/proxy/_experimental/out/_next/static/chunks/0_bflj-notfn6.js
View file

File diff suppressed because one or more lines are too long

1
litellm/proxy/_experimental/out/_next/static/chunks/0a3n_ovfo3c5s.js Normal file
View file

File diff suppressed because one or more lines are too long

1
litellm/proxy/_experimental/out/_next/static/chunks/0c2lerwwie30s.js
View file

File diff suppressed because one or more lines are too long

1
litellm/proxy/_experimental/out/_next/static/chunks/0ci-hazx_vz-j.js Normal file
View file

File diff suppressed because one or more lines are too long

1
litellm/proxy/_experimental/out/_next/static/chunks/0coby3gy7zzwi.js
View file

File diff suppressed because one or more lines are too long

1
litellm/proxy/_experimental/out/_next/static/chunks/0cotqb-2hzyvs.js Normal file
View file

File diff suppressed because one or more lines are too long

1
litellm/proxy/_experimental/out/_next/static/chunks/0d4xeknobwogp.js Normal file
View file

File diff suppressed because one or more lines are too long

1
litellm/proxy/_experimental/out/_next/static/chunks/0di-9qm-8ex8r.js Normal file
View file

File diff suppressed because one or more lines are too long

1
litellm/proxy/_experimental/out/_next/static/chunks/0dn8lan-q2jre.js
View file

File diff suppressed because one or more lines are too long

1
litellm/proxy/_experimental/out/_next/static/chunks/0dnt68i2qq-dg.js Normal file
View file

File diff suppressed because one or more lines are too long

1
litellm/proxy/_experimental/out/_next/static/chunks/0e80y6a9ghn2s.js
View file

File diff suppressed because one or more lines are too long

1
litellm/proxy/_experimental/out/_next/static/chunks/0g-j8z905_xfh.js
View file

File diff suppressed because one or more lines are too long

2
litellm/proxy/_experimental/out/_next/static/chunks/2qapx8_h7ir44.js → litellm/proxy/_experimental/out/_next/static/chunks/0gb6pr-exq8__.js
View file

File diff suppressed because one or more lines are too long

1
litellm/proxy/_experimental/out/_next/static/chunks/0gb7mj1nkcqmd.js Normal file
View file

File diff suppressed because one or more lines are too long

1
litellm/proxy/_experimental/out/_next/static/chunks/0i6-ixfyudd4f.js
View file

File diff suppressed because one or more lines are too long

1
litellm/proxy/_experimental/out/_next/static/chunks/0ikamdtw78iln.js Normal file
View file

File diff suppressed because one or more lines are too long

1
litellm/proxy/_experimental/out/_next/static/chunks/0iv9a33o4--6a.js
View file

File diff suppressed because one or more lines are too long

1
litellm/proxy/_experimental/out/_next/static/chunks/0j23_osi2t23b.js
View file

File diff suppressed because one or more lines are too long

1
litellm/proxy/_experimental/out/_next/static/chunks/0l8gk73gef2gr.js Normal file
View file

File diff suppressed because one or more lines are too long

1
litellm/proxy/_experimental/out/_next/static/chunks/0lgjier0jo0da.js Normal file
View file

File diff suppressed because one or more lines are too long

1
litellm/proxy/_experimental/out/_next/static/chunks/0m3x0p_sp4c11.js Normal file
View file

File diff suppressed because one or more lines are too long

2
litellm/proxy/_experimental/out/_next/static/chunks/3fj6j4vgjqp_9.js → litellm/proxy/_experimental/out/_next/static/chunks/0n_d-fecc6ing.js
View file

File diff suppressed because one or more lines are too long

1
litellm/proxy/_experimental/out/_next/static/chunks/0nv-vje-mizhj.js Normal file
View file

File diff suppressed because one or more lines are too long

1
litellm/proxy/_experimental/out/_next/static/chunks/0o2bf40gidns3.js
View file

File diff suppressed because one or more lines are too long

1
litellm/proxy/_experimental/out/_next/static/chunks/0ob_vs6vpubam.js Normal file
View file

File diff suppressed because one or more lines are too long

1
litellm/proxy/_experimental/out/_next/static/chunks/0ocldevv8nr5j.js
View file

File diff suppressed because one or more lines are too long

1
litellm/proxy/_experimental/out/_next/static/chunks/0p2ty6d6s6ikf.js Normal file
View file

File diff suppressed because one or more lines are too long

Some files were not shown because too many files have changed in this diff Show more