scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-03 02:22:24 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 81

Merge branch 'litellm_internal_staging' into fix-session-sidebar-sort-order

Browse source
This commit is contained in:
Vanika Dangi 2026-06-11 08:35:45 +05:30 • committed by GitHub
commit 8cb1e534a8
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
679 changed files with 42140 additions and 7949 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

166
.circleci/config.yml
View file

@ -111,6 +111,28 @@ commands:
- wait_for_service:
url: tcp://localhost:6379
timeout: "60"
start_openai_record_replay_proxy:
description: "Start the record/replay proxy (tests/_openai_record_replay_proxy.py) on host port 8090 and wait until healthy. Models whose api_base points here replay recorded provider responses, so the E2E run neither pays for nor depends on the live provider. The default upstream is OpenAI; a non-OpenAI model must point its api_base at /__recorder_upstream/<host>/ so the recorder forwards there instead of defaulting to OpenAI. Run after uv deps are synced."
steps:
- run:
name: Start record/replay proxy
background: true
command: |
CASSETTE_REDIS_URL="$CASSETTE_REDIS_URL" \
RECORDER_UPSTREAM_BASE_URL="https://api.openai.com" \
uv run --no-sync python tests/_openai_record_replay_proxy.py --host 0.0.0.0 --port 8090
- run:
name: Wait for record/replay proxy
command: |
for i in $(seq 1 30); do
if curl -sf http://localhost:8090/__recorder_health >/dev/null 2>&1; then
echo "record/replay proxy is up"
exit 0
fi
sleep 1
done
echo "record/replay proxy did not become ready" >&2
exit 1
setup_litellm_enterprise_pip:
steps:
- run:
@ -1625,25 +1647,7 @@ jobs:
command: |
zstd -d litellm-docker-database.tar.zst --stdout | docker load
docker tag litellm-docker-database:ci my-app:latest
- run:
name: Start OpenAI image record/replay proxy
background: true
command: |
CASSETTE_REDIS_URL="$CASSETTE_REDIS_URL" \
RECORDER_UPSTREAM_BASE_URL="https://api.openai.com" \
uv run --no-sync python tests/_openai_record_replay_proxy.py --host 0.0.0.0 --port 8090
- run:
name: Wait for record/replay proxy
command: |
for i in $(seq 1 30); do
if curl -sf http://localhost:8090/__recorder_health >/dev/null 2>&1; then
echo "record/replay proxy is up"
exit 0
fi
sleep 1
done
echo "record/replay proxy did not become ready" >&2
exit 1
- start_openai_record_replay_proxy
- run:
name: Run Docker container
command: |
@ -1674,7 +1678,7 @@ jobs:
-e LANGFUSE_PROJECT2_PUBLIC=$LANGFUSE_PROJECT2_PUBLIC \
-e LANGFUSE_PROJECT1_SECRET=$LANGFUSE_PROJECT1_SECRET \
-e LANGFUSE_PROJECT2_SECRET=$LANGFUSE_PROJECT2_SECRET \
-e IMAGE_GEN_RECORDER_BASE_URL=http://host.docker.internal:8090/v1 \
-e RECORDER_OPENAI_BASE_URL=http://host.docker.internal:8090/v1 \
--add-host host.docker.internal:host-gateway \
--name my-app \
-v $(pwd)/proxy_server_config.yaml:/app/config.yaml \
@ -1812,6 +1816,7 @@ jobs:
command: |
zstd -d litellm-docker-database.tar.zst --stdout | docker load
docker images | grep litellm-docker-database
- start_openai_record_replay_proxy
- run:
name: Run Docker container
# intentionally give bad redis credentials here
@ -1835,6 +1840,7 @@ jobs:
-e DD_SITE=$DD_SITE \
-e AWS_REGION_NAME=$AWS_REGION_NAME \
-e COHERE_API_KEY=$COHERE_API_KEY \
-e RECORDER_COHERE_BASE_URL=http://host.docker.internal:8090/__recorder_upstream/api.cohere.com \
-e GCS_FLUSH_INTERVAL="1" \
--add-host host.docker.internal:host-gateway \
--name my-app \
@ -2400,6 +2406,7 @@ jobs:
command: |
zstd -d litellm-docker-database.tar.zst --stdout | docker load
docker images | grep litellm-docker-database
- start_openai_record_replay_proxy
- run:
name: Run Docker container with test config
command: |
@ -2408,6 +2415,7 @@ jobs:
-e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \
-e LITELLM_MASTER_KEY="sk-1234" \
-e ANTHROPIC_API_KEY=$ANTHROPIC_API_KEY \
-e RECORDER_ANTHROPIC_BASE_URL=http://host.docker.internal:8090/__recorder_upstream/api.anthropic.com \
-e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \
-e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \
-e AWS_REGION_NAME="us-east-1" \
@ -2682,6 +2690,122 @@ jobs:
path: ui/litellm-dashboard/playwright-report
destination: e2e-playwright-report
e2e_ui_testing_server_root_path:
docker:
- image: cimg/python:3.12-browsers@sha256:b432899af01c9a311bf74f4f22e9ada2e5306d4b1b4383f8d29e1228a5844ef2
auth:
username: ${DOCKERHUB_USERNAME}
password: ${DOCKERHUB_PASSWORD}
- image: cimg/postgres:16.0@sha256:b125148bc76e8e8eee5eb3ad6020a3a14110a14e8192f1c645128afebe2e2f84
environment:
POSTGRES_USER: e2euser
POSTGRES_PASSWORD: e2epassword
POSTGRES_DB: litellm_e2e
resource_class: large
working_directory: ~/project
environment:
DATABASE_URL: "postgresql://e2euser:e2epassword@localhost:5432/litellm_e2e"
CI: "true"
# The whole job exercises the proxy mounted under a prefix. SERVER_ROOT_PATH
# is read both by the proxy at boot (to rewrite the built UI bundle in place)
# and by migration.serverRootPath.config.ts, which refuses to run without it.
SERVER_ROOT_PATH: "/litellm"
steps:
- checkout
- setup_google_dns
- install_uv
- restore_cache:
keys:
- v1-uv-cache-{{ checksum "uv.lock" }}
- run:
name: Install Python dependencies
command: |
uv sync --frozen --all-groups --all-extras --python 3.12
uv run --no-sync python -m prisma generate --schema litellm/proxy/schema.prisma
- save_cache:
key: v1-uv-cache-{{ checksum "uv.lock" }}
paths:
- ~/.cache/uv
- restore_cache:
keys:
- ui-e2e-node-deps-v2-{{ checksum "ui/litellm-dashboard/package-lock.json" }}
- run:
name: Install Node dependencies and Playwright
command: |
cd ui/litellm-dashboard
npm ci
npx playwright install chromium
- save_cache:
key: ui-e2e-node-deps-v2-{{ checksum "ui/litellm-dashboard/package-lock.json" }}
paths:
- ui/litellm-dashboard/node_modules
- ~/.cache/ms-playwright
- run:
name: Build UI from source
command: |
cd ui/litellm-dashboard
npm run build
rm -rf ../../litellm/proxy/_experimental/out
mv out ../../litellm/proxy/_experimental/out
find ../../litellm/proxy/_experimental/out -name '*.html' ! -name 'index.html' | while read -r f; do
d="${f%.html}"; mkdir -p "$d"; mv "$f" "$d/index.html"
done
- wait_for_service:
url: tcp://localhost:5432
timeout: "30"
- run:
name: Push Prisma schema
command: uv run --no-sync python -m prisma db push --schema litellm/proxy/schema.prisma --accept-data-loss
- run:
name: Seed database
command: |
PGPASSWORD=e2epassword psql -h localhost -p 5432 -U e2euser -d litellm_e2e \
-f ui/litellm-dashboard/e2e_tests/fixtures/seed.sql
- run:
name: Start mock LLM server
command: uv run --no-sync python ui/litellm-dashboard/e2e_tests/fixtures/mock_llm_server/server.py
background: true
- run:
name: Start LiteLLM proxy under a server root path
environment:
LITELLM_MASTER_KEY: "sk-1234"
MOCK_LLM_URL: "http://127.0.0.1:8090/v1"
DISABLE_SCHEMA_UPDATE: "true"
# Output flows to this step's own log, so a boot crash is visible here
# rather than swallowed by a downstream readiness probe.
command: |
LITELLM_LICENSE="$LITELLM_LICENSE" \
uv run --no-sync python -m litellm.proxy.proxy_cli \
--config ui/litellm-dashboard/e2e_tests/fixtures/config.yml \
--port 4000
background: true
- run:
name: Wait for prefixed proxy to be ready
command: |
for i in $(seq 1 60); do
HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" --max-time 5 -H "Authorization: Bearer sk-1234" http://127.0.0.1:4000/litellm/health 2>/dev/null || true)
if [ "$HTTP_CODE" = "200" ]; then
echo "Prefixed proxy is ready"
exit 0
fi
sleep 2
done
echo "Prefixed proxy failed to start; see the 'Start LiteLLM proxy under a server root path' step for the boot log"
exit 1
- run:
name: Run migration smoke under SERVER_ROOT_PATH
command: |
cd ui/litellm-dashboard
LITELLM_LICENSE="$LITELLM_LICENSE" \
npx playwright test --config e2e_tests/migration.serverRootPath.config.ts
no_output_timeout: 10m
- store_artifacts:
path: ui/litellm-dashboard/test-results
destination: e2e-server-root-path-test-results
- store_artifacts:
path: ui/litellm-dashboard/playwright-report
destination: e2e-server-root-path-playwright-report
build_docker_database_image:
machine:
image: ubuntu-2204:2024.04.1
@ -2787,6 +2911,8 @@ workflows:
filters: *main_branches
- e2e_ui_testing:
filters: *main_branches
- e2e_ui_testing_server_root_path:
filters: *main_branches
- build_and_test:
requires:
- build_docker_database_image

33
.github/workflows/create-release.yml vendored
View file

@ -52,6 +52,22 @@ jobs:
// are stable maintenance releases, not pre-releases.
const isPrerelease = /(?:rc|nightly|alpha|beta|[-.]dev)/i.test(tag);
// A stable release should only claim the repo "latest" badge when its
// version is >= the current latest. Otherwise a backport (e.g. 1.84.6)
// would steal "latest" from a newer line (e.g. 1.88.1).
const versionKey = (rawTag) => {
const m = String(rawTag).match(/^v?(\d+)\.(\d+)\.(\d+)/);
if (!m) return null;
const maintenance = String(rawTag).match(/(?:\.post|\.patch\.)(\d+)/i);
return [Number(m[1]), Number(m[2]), Number(m[3]), maintenance ? Number(maintenance[1]) : 0];
};
const isAtLeast = (a, b) => {
for (let i = 0; i < a.length; i++) {
if (a[i] !== b[i]) return a[i] > b[i];
}
return true;
};
const cosignSection = [
`## Verify Docker Image Signature`,
``,
@ -90,6 +106,22 @@ jobs:
].join('\n');
try {
let makeLatest = "false";
const newVersion = versionKey(tag);
if (!isPrerelease && newVersion) {
let latestVersion = null;
try {
const latest = await github.rest.repos.getLatestRelease({
owner: context.repo.owner,
repo: context.repo.repo,
});
latestVersion = versionKey(latest.data.tag_name);
} catch (error) {
if (error.status !== 404) throw error;
}
makeLatest = (!latestVersion || isAtLeast(newVersion, latestVersion)) ? "true" : "false";
}
const response = await github.rest.repos.createRelease({
draft: true,
generate_release_notes: true,
@ -108,6 +140,7 @@ jobs:
release_id: response.data.id,
body: updatedBody,
draft: false,
make_latest: makeLatest,
});
} catch (error) {

2
.github/workflows/test-unit-misc.yml vendored
View file

@ -28,6 +28,8 @@ jobs:
tests/test_litellm/completion_extras
tests/test_litellm/containers
tests/test_litellm/experimental_mcp_client
tests/test_litellm/models
tests/test_litellm/repositories
tests/test_litellm/images
tests/test_litellm/interactions
tests/test_litellm/passthrough

18
ARCHITECTURE.md
View file

@ -240,6 +240,24 @@ graph LR
7. `DBSpendUpdateWriter.update_database()` queues spend increments to Redis
8. Background job `update_spend` flushes queued spend to PostgreSQL every 60s
### Data Access Layer (Models & Repositories)
Database entities and the operations on them live in two packages at the root of `litellm/` so both the gateway (`proxy/`) and the SDK can use them without importing proxy internals:
- `litellm/models/` holds the canonical Pydantic definitions for every persisted entity (`LiteLLM_VerificationToken`, `LiteLLM_TeamTable`, `LiteLLM_UserTable`, etc.). `proxy/_types.py` re-exports these for backwards compatibility, so existing imports keep working.
- `litellm/repositories/` holds the data-access layer. `BaseRepository[T]` provides the generic CRUD (`find_by_id`, `find_many`, `create`, `update`, `delete`, `count`, `exists`); entity repositories such as `VerificationTokenRepository`, `TeamRepository`, and `UserRepository` add domain-specific queries and writes on top of it.
Conventions to follow when touching this layer:
| Concern | How it's handled |
|---------|------------------|
| JSON columns | Prisma `Json` columns are stored as JSON strings. Repositories `json.dumps()` on write and `json.loads()` on read (see `_to_model` and the `_build_*_data` helpers). |
| Archive-then-delete | `delete_team` / `delete_token` copy the row into the `LiteLLM_Deleted*` table and delete the original inside a single `prisma_client.db.tx()` transaction. Archive payloads are built explicitly so only columns that exist on the archive table are written. |
| Column vs. field names | Where a model field differs from its DB column (for example `org_id` maps to the `organization_id` column), the repository translates in both directions rather than relying on Pydantic to guess. |
| Array mutations | Adds use Prisma's atomic `push` (`add_member`, `add_admin`, `add_models`) to avoid read-modify-write races. Removals fall back to read-modify-write because Prisma has no atomic array remove. |
To add a new entity, define the model under `litellm/models/`, re-export it from `proxy/_types.py` if existing code imports it from there, and add a repository under `litellm/repositories/` (subclass `BaseRepository` for plain CRUD, or add bespoke methods when the entity needs encryption, archiving, or atomic array updates). Mirror the tests in `tests/test_litellm/repositories/`.
---
## 2. SDK Request Flow

13
CLAUDE.md
View file

@ -52,6 +52,19 @@ Do not put names of customers or customer company names in code, PRs, and issues
CI supply-chain safety: Never pipe a remote script into a shell (`curl ... | bash`, `wget ... | sh`); download the artifact to a file, verify its SHA-256 checksum, then install. Pin every external tool to a specific version with a full URL (not `latest` or `stable`). Verify checksums for all downloaded binaries, using the provider's official `.sha256` / `.sha256sum` sidecar when available. These rules apply to every download in CI
Follow these coding conventions for new/updated code (a three-line fix in a legacy file shouldn't trigger huge drive-by refactors):
- Composition over inheritance
- Never-nester: early returns over deep nesting
- Don't throw; model failures as values (One function (e.g., raise_public) maps error union to existing public exception contracts via exhaustive match + assert_never)
- No mutation; instead of mutable lists and dicts, prefer tuples, NamedTuples, frozen dataclasses, etc.
- Use dependency injection
- Fully typed; no `Any` or coarse types like dict[str, Any]. Every function parameter must be strongly typed
- Use tagged unions + match
- No monster files or god objects
Follow conventional commits for commit names and PR titles
## Think Before Coding
**Don't assume. Don't hide confusion. Surface tradeoffs**

4
litellm-proxy-extras/pyproject.toml
View file

@ -1,6 +1,6 @@
[project]
name = "litellm-proxy-extras"
version = "0.4.73"
version = "0.4.74"
description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
readme = "README.md"
requires-python = ">=3.9"
@ -26,7 +26,7 @@ required-version = ">=0.10.9"
module-root = ""
[tool.commitizen]
version = "0.4.73"
version = "0.4.74"
version_files = [
"pyproject.toml:^version",
"../pyproject.toml:litellm-proxy-extras==",

20
litellm/__init__.py
View file

@ -16,8 +16,17 @@ import os
# Load .env before any other litellm imports so env vars (e.g. LITELLM_UI_SESSION_DURATION) are available
import dotenv as _dotenv
def _dev_env_hot_reload_enabled() -> bool:
"""The proxy exports this flag when started with ``--reload``. A reloaded
worker is a fresh process that inherits the reloader's environment, so an
edited ``.env`` value stays masked by the stale inherited one unless we
let the file win; overriding makes the edit take effect on reload."""
return os.getenv("LITELLM_DEV_ENV_HOT_RELOAD") == "True"
if os.getenv("LITELLM_MODE", "DEV") == "DEV":
_dotenv.load_dotenv()
_dotenv.load_dotenv(override=_dev_env_hot_reload_enabled())
from typing import (
Callable,
@ -433,6 +442,13 @@ custom_prometheus_metadata_labels: List[str] = []
custom_prometheus_tags: List[str] = []
prometheus_metrics_config: Optional[List] = None
prometheus_emit_stream_label: bool = False
# Opt-in: emit `rate_limit_category` and `rate_limit_type` labels on
# `litellm_proxy_failed_requests_metric`. Off by default to preserve the
# pre-unification label set so existing dashboards / recording rules keyed on
# that metric keep matching after upgrade. Enable when downstream consumers
# are ready to split 429s by source (vendor vs. litellm) and dimension
# (RPM/TPM/concurrent/budget).
prometheus_emit_rate_limit_labels: bool = False
prometheus_user_budget_label_include_email_alias: bool = False
prometheus_end_user_metrics_max_series_per_metric: Optional[int] = 10000
prometheus_end_user_metrics_ttl_seconds: Optional[float] = 3600.0
@ -1294,6 +1310,8 @@ from .exceptions import (
NotFoundError,
PermissionDeniedError,
RateLimitError,
RateLimitErrorCategory,
RateLimitType,
ServiceUnavailableError,
BadGatewayError,
OpenAIError,

2
litellm/anthropic_beta_headers_config.json
View file

@ -129,7 +129,7 @@
"bash_20241022": null,
"bash_20250124": null,
"code-execution-2025-08-25": null,
"compact-2026-01-12": null,
"compact-2026-01-12": "compact-2026-01-12",
"computer-use-2025-01-24": "computer-use-2025-01-24",
"computer-use-2025-11-24": "computer-use-2025-11-24",
"context-1m-2025-08-07": "context-1m-2025-08-07",

81
litellm/caching/caching.py
View file

@ -309,9 +309,13 @@ class Cache:
param_value = kwargs[param]
cache_key += f"{str(param)}: {str(param_value)}"
verbose_logger.debug("\nCreated cache key: %s", cache_key)
hashed_cache_key = Cache._get_hashed_cache_key(cache_key)
hashed_cache_key = self._add_namespace_to_cache_key(hashed_cache_key, **kwargs)
verbose_logger.debug(
"\nCreated cache key: %s (source material length: %d)",
hashed_cache_key,
len(cache_key),
)
# Remove preset_cache_key from kwargs to avoid "got multiple values" TypeError
# when kwargs already contains preset_cache_key from upstream callers
kwargs_for_preset = {k: v for k, v in kwargs.items() if k != "preset_cache_key"}
@ -497,6 +501,34 @@ class Cache:
return cached_response
return cached_result
@staticmethod
def _get_safe_cache_lookup_kwargs(kwargs: Dict[str, Any]) -> Dict[str, Any]:
cache_lookup_kwargs: Dict[str, Any] = {}
for prompt_kwarg in ("messages", "input"):
if prompt_kwarg in kwargs:
cache_lookup_kwargs[prompt_kwarg] = kwargs[prompt_kwarg]
if isinstance(kwargs.get("metadata"), dict):
cache_lookup_kwargs["metadata"] = {}
return cache_lookup_kwargs
@staticmethod
def _update_metadata_from_cache_lookup_kwargs(
original_kwargs: Dict[str, Any], cache_lookup_kwargs: Dict[str, Any]
) -> None:
original_metadata = original_kwargs.get("metadata")
cache_lookup_metadata = cache_lookup_kwargs.get("metadata")
if not isinstance(original_metadata, dict) or not isinstance(
cache_lookup_metadata, dict
):
return
if "semantic-similarity" in cache_lookup_metadata:
original_metadata["semantic-similarity"] = cache_lookup_metadata[
"semantic-similarity"
]
def get_cache(self, dynamic_cache_object: Optional[BaseCache] = None, **kwargs):
"""
Retrieves the cached result for the given arguments.
@ -511,7 +543,6 @@ class Cache:
try: # never block execution
if self.should_use_cache(**kwargs) is not True:
return
messages = kwargs.get("messages", [])
if "cache_key" in kwargs:
cache_key = kwargs["cache_key"]
else:
@ -523,12 +554,19 @@ class Cache:
or cache_control_args.get("s-max-age")
or float("inf")
)
cache_lookup_kwargs = self._get_safe_cache_lookup_kwargs(kwargs)
if dynamic_cache_object is not None:
cached_result = dynamic_cache_object.get_cache(
cache_key, messages=messages
cache_key, **cache_lookup_kwargs
)
else:
cached_result = self.cache.get_cache(cache_key, messages=messages)
cached_result = self.cache.get_cache(
cache_key, **cache_lookup_kwargs
)
self._update_metadata_from_cache_lookup_kwargs(
original_kwargs=kwargs,
cache_lookup_kwargs=cache_lookup_kwargs,
)
return self._get_cache_logic(
cached_result=cached_result, max_age=max_age
)
@ -549,7 +587,6 @@ class Cache:
if self.should_use_cache(**kwargs) is not True:
return
kwargs.get("messages", [])
if "cache_key" in kwargs:
cache_key = kwargs["cache_key"]
else:
@ -654,6 +691,7 @@ class Cache:
self,
embedding_response: Any,
model: Optional[str],
prompt_tokens: Optional[int] = None,
prompt_tokens_details: Optional[dict] = None,
) -> CachedEmbedding:
"""
@ -666,6 +704,7 @@ class Cache:
"index": embedding_response.get("index"),
"object": embedding_response.get("object"),
"model": model,
"prompt_tokens": prompt_tokens,
"prompt_tokens_details": prompt_tokens_details,
}
elif hasattr(embedding_response, "model_dump"):
@ -675,6 +714,7 @@ class Cache:
"index": data.get("index"),
"object": data.get("object"),
"model": model,
"prompt_tokens": prompt_tokens,
"prompt_tokens_details": prompt_tokens_details,
}
else:
@ -684,6 +724,7 @@ class Cache:
"index": data.get("index"),
"object": data.get("object"),
"model": model,
"prompt_tokens": prompt_tokens,
"prompt_tokens_details": prompt_tokens_details,
}
except KeyError as e:
@ -732,6 +773,29 @@ class Cache:
per_item[key] = value
return per_item if per_item else None
def _get_per_item_prompt_tokens(
self,
result: EmbeddingResponse,
idx_in_result_data: int,
) -> Optional[int]:
"""
Extract the per-item prompt_tokens from a response for caching.
Single-item responses store the full usage.prompt_tokens. Multi-item
responses distribute it evenly (with remainder) so that summing all
per-item values on retrieval reconstructs the original total.
"""
if result.usage is None or result.usage.prompt_tokens is None:
return None
total = result.usage.prompt_tokens
num_items = len(result.data)
if num_items <= 1:
return total
quotient, remainder = divmod(total, num_items)
return quotient + (1 if idx_in_result_data < remainder else 0)
def add_embedding_response_to_cache(
self,
result: EmbeddingResponse,
@ -743,7 +807,11 @@ class Cache:
kwargs["cache_key"] = preset_cache_key
embedding_response = result.data[idx_in_result_data]
# Extract per-item prompt_tokens_details from response usage
# Extract per-item prompt_tokens + details from response usage
prompt_tokens = self._get_per_item_prompt_tokens(
result=result,
idx_in_result_data=idx_in_result_data,
)
prompt_tokens_details = self._get_per_item_prompt_tokens_details(
result=result,
idx_in_result_data=idx_in_result_data,
@ -754,6 +822,7 @@ class Cache:
embedding_dict: CachedEmbedding = self._convert_to_cached_embedding(
embedding_response,
model_name,
prompt_tokens=prompt_tokens,
prompt_tokens_details=prompt_tokens_details,
)

7
litellm/caching/caching_handler.py
View file

@ -394,7 +394,7 @@ class LLMCachingHandler:
return cr["model"]
return None
def _process_async_embedding_cached_response(
def _process_async_embedding_cached_response( # noqa: PLR0915
self,
final_embedding_cached_response: Optional[EmbeddingResponse],
cached_result: List[Optional[CachedEmbedding]],
@ -456,7 +456,10 @@ class LLMCachingHandler:
index=idx,
object="embedding",
)
if isinstance(kwargs_input_as_list[idx], str):
cached_prompt_tokens = cr.get("prompt_tokens")
if cached_prompt_tokens is not None:
prompt_tokens += cached_prompt_tokens
elif isinstance(kwargs_input_as_list[idx], str):
from litellm.utils import token_counter
prompt_tokens += token_counter(

105
litellm/caching/redis_semantic_cache.py
View file

@ -213,6 +213,78 @@ class RedisSemanticCache(BaseCache):
ttl = int(ttl)
return ttl
@classmethod
def _get_prompt_from_kwargs(cls, **kwargs) -> Optional[str]:
"""
Extract a semantic-cache prompt from chat or Responses API request kwargs.
"""
messages = kwargs.get("messages")
if messages:
return get_str_from_messages(messages)
if "input" not in kwargs:
return None
prompt_parts: List[str] = []
cls._collect_responses_input_text(kwargs.get("input"), prompt_parts)
prompt = "\n".join(prompt_parts).strip()
return prompt or None
@classmethod
def _collect_responses_input_text(cls, value: Any, prompt_parts: List[str]) -> None:
value = cls._coerce_response_input_value(value)
if value is None:
return
if isinstance(value, str):
stripped_value = value.strip()
if stripped_value:
prompt_parts.append(stripped_value)
return
if isinstance(value, (list, tuple)):
for item in value:
cls._collect_responses_input_text(item, prompt_parts)
return
if isinstance(value, dict):
content = value.get("content")
if content is not None:
cls._collect_responses_input_text(content, prompt_parts)
return
for text_key in ("text", "output", "input_text", "output_text"):
text_value = value.get(text_key)
if isinstance(text_value, str):
stripped_text = text_value.strip()
if stripped_text:
prompt_parts.append(stripped_text)
return
return
content = getattr(value, "content", None)
if content is not None:
cls._collect_responses_input_text(content, prompt_parts)
return
for text_key in ("text", "output", "input_text", "output_text"):
text_value = getattr(value, text_key, None)
if isinstance(text_value, str):
stripped_text = text_value.strip()
if stripped_text:
prompt_parts.append(stripped_text)
return
@staticmethod
def _coerce_response_input_value(value: Any) -> Any:
model_dump = getattr(value, "model_dump", None)
if callable(model_dump):
return model_dump()
dict_method = getattr(value, "dict", None)
if callable(dict_method):
return dict_method()
return value
def _get_embedding(self, prompt: str) -> List[float]:
"""
Generate an embedding vector for the given prompt using the configured embedding model.
@ -278,13 +350,11 @@ class RedisSemanticCache(BaseCache):
value_str: Optional[str] = None
try:
# Extract the prompt from messages
messages = kwargs.get("messages", [])
if not messages:
print_verbose("No messages provided for semantic caching")
prompt = self._get_prompt_from_kwargs(**kwargs)
if prompt is None:
print_verbose("No prompt provided for semantic caching")
return
prompt = get_str_from_messages(messages)
value_str = str(value)
store_kwargs: Dict[str, Any] = {
@ -315,14 +385,12 @@ class RedisSemanticCache(BaseCache):
print_verbose(f"Redis semantic-cache get_cache, kwargs: {kwargs}")
try:
# Extract the prompt from messages
messages = kwargs.get("messages", [])
if not messages:
print_verbose("No messages provided for semantic cache lookup")
prompt = self._get_prompt_from_kwargs(**kwargs)
if prompt is None:
print_verbose("No prompt provided for semantic cache lookup")
kwargs.setdefault("metadata", {})["semantic-similarity"] = 0.0
return None
prompt = get_str_from_messages(messages)
# Check the cache for semantically similar prompts in this exact
# LiteLLM cache-key scope.
check_kwargs: Dict[str, Any] = {
@ -428,13 +496,11 @@ class RedisSemanticCache(BaseCache):
print_verbose(f"Async Redis semantic-cache set_cache, kwargs: {kwargs}")
try:
# Extract the prompt from messages
messages = kwargs.get("messages", [])
if not messages:
print_verbose("No messages provided for semantic caching")
prompt = self._get_prompt_from_kwargs(**kwargs)
if prompt is None:
print_verbose("No prompt provided for semantic caching")
return
prompt = get_str_from_messages(messages)
value_str = str(value)
# Generate embedding for the value (response) to cache
@ -471,15 +537,12 @@ class RedisSemanticCache(BaseCache):
print_verbose(f"Async Redis semantic-cache get_cache, kwargs: {kwargs}")
try:
# Extract the prompt from messages
messages = kwargs.get("messages", [])
if not messages:
print_verbose("No messages provided for semantic cache lookup")
prompt = self._get_prompt_from_kwargs(**kwargs)
if prompt is None:
print_verbose("No prompt provided for semantic cache lookup")
kwargs.setdefault("metadata", {})["semantic-similarity"] = 0.0
return None
prompt = get_str_from_messages(messages)
# Generate embedding for the prompt
prompt_embedding = await self._get_async_embedding(prompt, **kwargs)

14
litellm/completion_extras/litellm_responses_transformation/transformation.py
View file

@ -402,6 +402,20 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
instructions,
) = self.convert_chat_completion_messages_to_responses_api(messages)
# OpenAI's Responses API rejects an empty input. For a system-only
# request, carry the system message as a system-role input item instead
# of instructions, mirroring how non-string system content is already
# handled in convert_chat_completion_messages_to_responses_api.
if not input_items and instructions is not None:
input_items = [
{
"type": "message",
"role": "system",
"content": [{"type": "input_text", "text": instructions}],
}
]
instructions = None
optional_params = self._extract_extra_body_params(optional_params)
# Build responses API request using the reverse transformation logic

2
litellm/constants.py
View file

@ -831,6 +831,7 @@ openai_compatible_providers: List = [
"nano-gpt", # Nano-GPT - JSON-configured provider
"poe", # Poe - JSON-configured provider
"chutes", # Chutes - JSON-configured provider
"parasail", # Parasail - JSON-configured provider
"featherless_ai",
"nscale",
"nebius",
@ -1157,6 +1158,7 @@ BEDROCK_CONVERSE_MODELS = [
"openai.gpt-oss-120b-1:0",
"anthropic.claude-haiku-4-5-20251001-v1:0",
"anthropic.claude-sonnet-4-5-20250929-v1:0",
"anthropic.claude-fable-5",
"anthropic.claude-opus-4-8",
"anthropic.claude-opus-4-7",
"anthropic.claude-opus-4-6-v1:0",

161
litellm/exceptions.py
View file

@ -9,13 +9,109 @@
## LiteLLM versions of the OpenAI Exception Types
from typing import Any, Dict, Optional
import enum
from typing import Any, Dict, Optional, Union
import httpx
import openai
from litellm.types.utils import LiteLLMCommonStrings
class RateLimitErrorCategory(str, enum.Enum):
"""
Category of a rate limit error, allowing callers to distinguish where the rate
limit originated. Exposed on every :class:`RateLimitError` instance via the
``category`` attribute.
Use these values to switch on the rate limit source, e.g.::
try:
...
except litellm.RateLimitError as e:
if e.category == RateLimitErrorCategory.LITELLM_RATE_LIMIT:
... # litellm's own limiter (key/team/user/model RPM/TPM/budget)
elif e.category == RateLimitErrorCategory.VENDOR_RATE_LIMIT:
... # the upstream LLM provider returned 429
"""
VENDOR_RATE_LIMIT = "vendor_rate_limit"
"""The upstream LLM provider returned a rate-limit response (e.g. OpenAI 429)."""
VENDOR_BATCH_RATE_LIMIT = "vendor_batch_rate_limit"
"""The upstream LLM provider returned a rate-limit response on a batch endpoint."""
LITELLM_RATE_LIMIT = "litellm_rate_limit"
"""LiteLLM's own rate limiter (key/team/user/model RPM/TPM, budget, parallel-requests, etc.) blocked the request."""
LITELLM_BATCH_RATE_LIMIT = "litellm_batch_rate_limit"
"""LiteLLM's own batch rate limiter (token/request budget across a batch input file) blocked the request."""
class RateLimitType(str, enum.Enum):
"""
The dimension that was exceeded when a rate-limit error fired.
This is orthogonal to :class:`RateLimitErrorCategory` — *category* tells
callers **who** rate-limited the request (the upstream vendor vs. one of
litellm's own limiters), while *type* tells them **which limit dimension**
was exceeded (an RPM ceiling, a TPM ceiling, a max-parallel-requests
ceiling, a budget cap, or a max-iterations cap).
Surfaced both on every :class:`RateLimitError` instance via the
``rate_limit_type`` attribute and on the structured
``StandardLoggingPayload.error_information.error_rate_limit_type`` field
so custom callbacks / metrics consumers can split rate-limit failures by
cause without parsing free-text error messages.
"""
REQUESTS = "requests"
"""Requests-per-minute (RPM) or requests-per-window ceiling exceeded."""
TOKENS = "tokens"
"""Tokens-per-minute (TPM) or tokens-per-window ceiling exceeded."""
CONCURRENT_REQUESTS = "concurrent_requests"
"""``max_parallel_requests`` — too many in-flight requests at once."""
BUDGET = "budget"
"""Spend budget cap reached (key, team, user, or per-session)."""
MAX_ITERATIONS = "max_iterations"
"""Per-session max-iterations cap reached (agent-style flows)."""
_RATE_LIMIT_CATEGORY_VALUES = frozenset(c.value for c in RateLimitErrorCategory)
_RATE_LIMIT_TYPE_VALUES = frozenset(t.value for t in RateLimitType)
def validate_rate_limit_category(value: Any) -> Optional[str]:
"""Return ``value`` only if it matches a known :class:`RateLimitErrorCategory`.
Used at duck-typed read sites (StandardLoggingPayload extraction, Prometheus
labels) to reject `.category` strings set by unrelated third-party exceptions
— otherwise those would leak into custom-callback payloads and Prometheus
label cardinality.
"""
if isinstance(value, RateLimitErrorCategory):
return value.value
if isinstance(value, str) and value in _RATE_LIMIT_CATEGORY_VALUES:
return value
return None
def validate_rate_limit_type(value: Any) -> Optional[str]:
"""Return ``value`` only if it matches a known :class:`RateLimitType`.
See :func:`validate_rate_limit_category` for the rationale.
"""
if isinstance(value, RateLimitType):
return value.value
if isinstance(value, str) and value in _RATE_LIMIT_TYPE_VALUES:
return value
return None
_MINIMAL_ERROR_RESPONSE: Optional[httpx.Response] = None
@ -321,6 +417,18 @@ class PermissionDeniedError(openai.PermissionDeniedError): # type: ignore
class RateLimitError(openai.RateLimitError): # type: ignore
"""
Unified rate-limit error.
Every rate-limit condition surfaced by litellm — whether it originated from
an upstream LLM provider, a vendor batch endpoint, or one of litellm's own
proxy-side limiters (parallel-requests, dynamic-rate, batch-rate, budget,
max-iterations, etc.) — is raised as an instance of this class.
The :attr:`category` attribute lets callers distinguish the source. See
:class:`RateLimitErrorCategory` for the available values.
"""
def __init__(
self,
message,
@ -330,6 +438,12 @@ class RateLimitError(openai.RateLimitError): # type: ignore
litellm_debug_info: Optional[str] = None,
max_retries: Optional[int] = None,
num_retries: Optional[int] = None,
category: Union[str, RateLimitErrorCategory] = (
RateLimitErrorCategory.VENDOR_RATE_LIMIT
),
rate_limit_type: Optional[Union[str, RateLimitType]] = None,
headers: Optional[Dict[str, str]] = None,
detail: Any = None,
):
self.status_code = 429
self.message = "litellm.RateLimitError: {}".format(message)
@ -338,9 +452,39 @@ class RateLimitError(openai.RateLimitError): # type: ignore
self.litellm_debug_info = litellm_debug_info
self.max_retries = max_retries
self.num_retries = num_retries
self.category = (
category.value if isinstance(category, RateLimitErrorCategory) else category
)
# Which dimension was exceeded — request count, token count, parallel
# requests, budget, max iterations. None when the source didn't
# classify the failure (e.g. legacy vendor 429 with no header hints).
self.rate_limit_type: Optional[str] = (
rate_limit_type.value
if isinstance(rate_limit_type, RateLimitType)
else rate_limit_type
)
# Headers explicitly attached to the error (e.g. retry-after,
# rate_limit_type, reset_at). Preserved across the proxy boundary so
# clients can react appropriately.
#
# IMPORTANT: we deliberately do NOT auto-populate self.headers from
# response.headers when only `response` is provided. A vendor 429 can
# set arbitrary response headers (Set-Cookie, CORS overrides, …); if
# those leaked into e.headers and a downstream proxy serializer
# forwarded them to the client, a malicious upstream could inject
# browser-interpreted headers for the proxy origin. Vendor response
# headers stay reachable on `e.response.headers` for callers that
# explicitly want them; only the proxy-supplied `headers=` kwarg
# makes it onto `self.headers`.
_response_headers = (
getattr(response, "headers", None) if response is not None else None
)
self.headers: Optional[Dict[str, str]] = (
{k: str(v) for k, v in headers.items()} if headers else None
)
# Mirrors FastAPI HTTPException.detail so the same instance can be
# serialized through both the ProxyException and HTTPException paths.
self.detail = detail if detail is not None else self.message
self.response = httpx.Response(
status_code=429,
headers=_response_headers,
@ -843,11 +987,24 @@ LITELLM_EXCEPTION_TYPES = [
class BudgetExceededError(Exception):
def __init__(
self, current_cost: float, max_budget: float, message: Optional[str] = None
self,
current_cost: float,
max_budget: float,
message: Optional[str] = None,
llm_provider: Optional[str] = None,
):
self.current_cost = current_cost
self.max_budget = max_budget
self.status_code = 429
self.llm_provider = llm_provider or ""
# Surface unified rate-limit fields without joining the RateLimitError
# hierarchy so existing `except BudgetExceededError:` handlers keep
# working; custom callbacks reading StandardLoggingPayload pick these
# up via the same `category` / `rate_limit_type` attributes the rest
# of the unified rate-limit error path uses. Stored as plain strings
# to match the normalization RateLimitError.__init__ performs.
self.category: str = RateLimitErrorCategory.LITELLM_RATE_LIMIT.value
self.rate_limit_type: str = RateLimitType.BUDGET.value
message = (
message
or f"Budget has been exceeded! Current cost: {current_cost}, Max budget: {max_budget}"

6
litellm/integrations/SlackAlerting/slack_alerting.py
View file

@ -37,6 +37,8 @@ from litellm.proxy._types import (
VirtualKeyEvent,
WebhookEvent,
)
from litellm.repositories.team_repository import TeamRepository
from litellm.repositories.user_repository import UserRepository
from litellm.types.integrations.slack_alerting import *
from ..email_templates.templates import *
@ -1231,7 +1233,7 @@ Model Info:
and recipient_user_id is not None
and prisma_client is not None
):
user_row = await prisma_client.db.litellm_usertable.find_unique(
user_row = await UserRepository(prisma_client).table.find_unique(
where={"user_id": recipient_user_id}
)
@ -1263,7 +1265,7 @@ Model Info:
team_id = webhook_event.team_id
team_name = "Default Team"
if team_id is not None and prisma_client is not None:
team_row = await prisma_client.db.litellm_teamtable.find_unique(
team_row = await TeamRepository(prisma_client).table.find_unique(
where={"team_id": team_id}
)
if team_row is not None:

9
litellm/integrations/compression_interception/handler.py
View file

@ -72,8 +72,13 @@ class CompressionInterceptionLogger(CustomLogger):
compression_params: CompressionInterceptionConfig = {}
if "compression_interception_params" in litellm_settings:
compression_params = litellm_settings["compression_interception_params"]
elif "compression_interception" in callback_specific_params:
compression_params = callback_specific_params["compression_interception"]
elif "compression_interception" in callback_specific_params and isinstance(
callback_specific_params["compression_interception"], dict
):
compression_params = cast(
CompressionInterceptionConfig,
callback_specific_params["compression_interception"],
)
return CompressionInterceptionLogger.from_config_yaml(compression_params)
async def async_pre_call_deployment_hook(

3
litellm/integrations/email_alerting.py
View file

@ -7,6 +7,7 @@ from typing import List, Optional
from litellm._logging import verbose_logger, verbose_proxy_logger
from litellm.proxy._types import WebhookEvent
from litellm.repositories.team_repository import TeamRepository
# we use this for the email header, please send a test email if you change this. verify it looks good on email
LITELLM_LOGO_URL = "https://litellm-listing.s3.amazonaws.com/litellm_logo.png"
@ -24,7 +25,7 @@ async def get_all_team_member_emails(team_id: Optional[str] = None) -> list:
if prisma_client is None:
raise Exception("Not connected to DB!")
team_row = await prisma_client.db.litellm_teamtable.find_unique(
team_row = await TeamRepository(prisma_client).table.find_unique(
where={
"team_id": team_id,
}

6
litellm/integrations/focus/database.py
View file

@ -80,11 +80,15 @@ class FocusLiteLLMDatabase:
vt.team_id,
vt.key_alias as api_key_alias,
tt.team_alias,
ut.user_email as user_email
ut.user_email as user_email,
COALESCE(vt.organization_id, tt.organization_id) as organization_id,
ot.organization_alias as organization_alias
FROM "LiteLLM_DailyUserSpend" dus
LEFT JOIN "LiteLLM_VerificationToken" vt ON dus.api_key = vt.token
LEFT JOIN "LiteLLM_TeamTable" tt ON vt.team_id = tt.team_id
LEFT JOIN "LiteLLM_UserTable" ut ON dus.user_id = ut.user_id
LEFT JOIN "LiteLLM_OrganizationTable" ot
ON ot.organization_id = COALESCE(vt.organization_id, tt.organization_id)
{where_clause}
ORDER BY dus.date DESC, dus.created_at DESC
{limit_clause}

2
litellm/integrations/focus/destinations/__init__.py
View file

@ -2,12 +2,14 @@
from .base import FocusDestination, FocusTimeWindow
from .factory import FocusDestinationFactory
from .gcs_destination import FocusGCSDestination
from .s3_destination import FocusS3Destination
from .vantage_destination import FocusVantageDestination
__all__ = [
"FocusDestination",
"FocusDestinationFactory",
"FocusGCSDestination",
"FocusTimeWindow",
"FocusS3Destination",
"FocusVantageDestination",

15
litellm/integrations/focus/destinations/factory.py
View file

@ -6,6 +6,7 @@ import os
from typing import Any, Dict, Optional
from .base import FocusDestination
from .gcs_destination import FocusGCSDestination
from .s3_destination import FocusS3Destination
from .vantage_destination import FocusVantageDestination
@ -29,6 +30,8 @@ class FocusDestinationFactory:
return FocusS3Destination(prefix=prefix, config=normalized_config)
if provider_lower == "vantage":
return FocusVantageDestination(prefix=prefix, config=normalized_config)
if provider_lower == "gcs":
return FocusGCSDestination(prefix=prefix, config=normalized_config)
raise NotImplementedError(
f"Provider '{provider}' not supported for Focus export"
)
@ -72,6 +75,18 @@ class FocusDestinationFactory:
"VANTAGE_INTEGRATION_TOKEN must be provided for Vantage exports"
)
return {k: v for k, v in resolved.items() if v is not None}
if provider == "gcs":
resolved = {
"bucket_name": overrides.get("bucket_name")
or os.getenv("FOCUS_GCS_BUCKET_NAME"),
"service_account_json": overrides.get("service_account_json")
or os.getenv("FOCUS_GCS_PATH_SERVICE_ACCOUNT"),
}
if not resolved.get("bucket_name"):
raise ValueError(
"FOCUS_GCS_BUCKET_NAME must be provided for GCS exports"
)
return {k: v for k, v in resolved.items() if v is not None}
raise NotImplementedError(
f"Provider '{provider}' not supported for Focus export configuration"
)

74
litellm/integrations/focus/destinations/gcs_destination.py Normal file
View file

@ -0,0 +1,74 @@
"""GCS destination for Focus export — reuses GCSBucketBase auth and httpx client."""
from __future__ import annotations
from datetime import timezone
from typing import Any, Optional
from litellm._logging import verbose_logger
from litellm.integrations.gcs_bucket.gcs_bucket_base import GCSBucketBase
from litellm.litellm_core_utils.cloud_storage_security import (
encode_gcs_object_name_for_url,
)
from .base import FocusDestination, FocusTimeWindow
class FocusGCSDestination(GCSBucketBase, FocusDestination):
"""Upload serialized Focus exports to GCS using the GCS JSON API."""
def __init__(
self,
*,
prefix: str,
config: Optional[dict[str, Any]] = None,
) -> None:
config = config or {}
bucket_name = config.get("bucket_name")
if not bucket_name:
raise ValueError("bucket_name must be provided for GCS destination")
super().__init__(bucket_name=bucket_name)
service_account_json = config.get("service_account_json")
if service_account_json is not None:
self.path_service_account_json = service_account_json
self.prefix = prefix.rstrip("/")
async def deliver(
self,
*,
content: bytes,
time_window: FocusTimeWindow,
filename: str,
) -> None:
object_name = self._build_object_key(time_window=time_window, filename=filename)
headers = await self.construct_request_headers(
service_account_json=self.path_service_account_json
)
headers["Content-Type"] = "application/octet-stream"
encoded_name = encode_gcs_object_name_for_url(object_name)
url = (
f"https://storage.googleapis.com/upload/storage/v1/b/"
f"{self.BUCKET_NAME}/o?uploadType=media&name={encoded_name}"
)
response = await self.async_httpx_client.post(
url=url, headers=headers, data=content
)
if response.status_code != 200:
raise RuntimeError(
f"GCS upload failed: status={response.status_code} body={response.text}"
)
verbose_logger.debug(
"Focus GCS: uploaded %d bytes to gs://%s/%s",
len(content),
self.BUCKET_NAME,
object_name,
)
def _build_object_key(self, *, time_window: FocusTimeWindow, filename: str) -> str:
start_utc = time_window.start_time.astimezone(timezone.utc)
date_component = f"date={start_utc.strftime('%Y-%m-%d')}"
parts = [self.prefix, date_component]
if time_window.frequency == "hourly":
parts.append(f"hour={start_utc.strftime('%H')}")
key_prefix = "/".join(filter(None, parts))
return f"{key_prefix}/{filename}" if key_prefix else filename

2
litellm/integrations/focus/transformer.py
View file

@ -12,6 +12,8 @@ from .schema import FOCUS_NORMALIZED_SCHEMA
_TAG_KEYS = (
"team_id",
"team_alias",
"organization_id",
"organization_alias",
"user_id",
"user_email",
"api_key_alias",

71
litellm/integrations/galileo.py
View file

@ -26,6 +26,7 @@ from litellm.llms.custom_httpx.http_handler import (
get_async_httpx_client,
httpxSpecialProvider,
)
from litellm.types.integrations.base_health_check import IntegrationHealthCheckStatus
GALILEO_CLOUD_API_BASE_URL = "https://api.galileo.ai"
# Cap the in-memory buffer so persistent flush failures (e.g. Galileo
@ -89,6 +90,52 @@ class GalileoObserve(CustomLogger):
return bool(self.api_key)
return bool(self.username and self.password)
async def async_health_check(self) -> IntegrationHealthCheckStatus:
try:
if not self.project_id:
return IntegrationHealthCheckStatus(
status="unhealthy",
error_message="GALILEO_PROJECT_ID environment variable not set",
)
if not self.base_url:
return IntegrationHealthCheckStatus(
status="unhealthy",
error_message="GALILEO_BASE_URL environment variable not set",
)
if not self.use_v2_api and (not self.username or not self.password):
return IntegrationHealthCheckStatus(
status="unhealthy",
error_message=(
"GALILEO_API_KEY or GALILEO_USERNAME and GALILEO_PASSWORD "
"environment variables must be set"
),
)
if not await self._ensure_headers():
return IntegrationHealthCheckStatus(
status="unhealthy",
error_message="Galileo authentication failed",
)
response = await self.async_httpx_handler.get(
url=f"{self.base_url}/current_user",
headers=self.headers,
)
if response.status_code >= 400:
return IntegrationHealthCheckStatus(
status="unhealthy",
error_message=(f"Galileo API returned HTTP {response.status_code}"),
)
return IntegrationHealthCheckStatus(status="healthy", error_message=None)
except Exception as e:
return IntegrationHealthCheckStatus(
status="unhealthy",
error_message=f"Galileo health check failed: {str(e)}",
)
async def async_set_galileo_headers(self) -> None:
galileo_login_response = await self.async_httpx_handler.post(
url=f"{self.base_url}/login",
@ -399,9 +446,9 @@ class GalileoObserve(CustomLogger):
return prompt
@staticmethod
def _serialize_galileo_output(value: Any) -> Optional[str]:
def _serialize_galileo_output(value: Any) -> str:
if value is None:
return None
return ""
if isinstance(value, str):
return value
@ -460,11 +507,11 @@ class GalileoObserve(CustomLogger):
response_obj: Any,
level: str = "DEFAULT",
status_message: Optional[str] = None,
) -> Tuple[str, Optional[str], Any]:
) -> Tuple[str, str, Any]:
"""
Mirror Langfuse _get_langfuse_input_output_content for Galileo ingest.
Returns (input_text, output_text, messages_for_span). output_text None skips ingest.
Returns (input_text, output_text, messages_for_span).
"""
call_type = kwargs.get("call_type")
prompt = self._build_prompt(kwargs)
@ -477,10 +524,11 @@ class GalileoObserve(CustomLogger):
return self._prompt_to_input_text(prompt), status_message, prompt
if response_obj is not None and (
call_type == "embedding"
call_type in ("embedding", "aembedding")
or isinstance(response_obj, litellm.EmbeddingResponse)
):
return self._prompt_to_input_text(prompt), None, prompt
# Match Langfuse OTEL: log embeddings without serializing vectors.
return self._prompt_to_input_text(prompt), "embedding-output", prompt
if response_obj is not None and isinstance(response_obj, litellm.ModelResponse):
output = self._get_chat_content_for_galileo(response_obj)
@ -549,7 +597,7 @@ class GalileoObserve(CustomLogger):
):
input_val = kwargs.get("input")
return (
self._serialize_galileo_output(input_val) or "",
self._serialize_galileo_output(input_val),
self._serialize_galileo_output(response_obj),
input_val,
)
@ -574,11 +622,11 @@ class GalileoObserve(CustomLogger):
kwargs.get("messages") or [],
)
return self._prompt_to_input_text(prompt), None, kwargs.get("messages") or []
return self._prompt_to_input_text(prompt), "", kwargs.get("messages") or []
def get_output_str_from_response(
self, response_obj: Any, kwargs: Dict[str, Any]
) -> Optional[str]:
) -> str:
_, output_text, _ = self._get_galileo_input_output_content(
kwargs=kwargs, response_obj=response_obj
)
@ -659,11 +707,6 @@ class GalileoObserve(CustomLogger):
input_text, output_text, messages = self._get_galileo_input_output_content(
kwargs=kwargs, response_obj=response_obj
)
if output_text is None:
verbose_logger.debug(
"Galileo Logger: skipping %s — no text output to log", _call_type
)
return
raw_start = slo.get("startTime")
raw_end = slo.get("endTime")

9
litellm/integrations/otel/logger.py
View file

@ -15,6 +15,7 @@ from litellm.integrations.otel.model.baggage import promoted_baggage
from litellm.integrations.otel.model.config import OpenTelemetryV2Config
from litellm.integrations.otel.plumbing.context import (
is_recordable_span,
request_root_span,
resolve_parent_context,
resolve_request_span_context,
set_request_baggage,
@ -435,8 +436,12 @@ class OpenTelemetryV2(CustomLogger):
attach(set_request_baggage(bag, context=get_current()))
# The server span was started by the instrumentor before this ran,
# so the Baggage processor (which only fires at span start) won't
# backfill it — stamp identity on it directly.
server_span = get_current_span()
# backfill it — stamp identity on it directly. Prefer the anchored
# root span over the ambient one so identity still lands on the
# server span when seeding from inside the live ``auth`` phase span
# (the auth-failure path), where ``get_current_span`` is the phase
# span, not the request's root.
server_span = request_root_span() or get_current_span()
if is_recordable_span(server_span):
# Re-capture the anchor here too: this runs post-auth with the
# server span active and covers entrypoints that bypass

119
litellm/integrations/prometheus.py
View file

@ -24,14 +24,18 @@ from typing import (
import litellm
from litellm._logging import print_verbose, verbose_logger
from litellm.integrations.custom_logger import CustomLogger
from litellm.integrations.prometheus_helpers.bounded_prometheus_series_tracker import (
BoundedPrometheusSeriesTracker,
from litellm.exceptions import (
validate_rate_limit_category,
validate_rate_limit_type,
)
from litellm.integrations.custom_logger import CustomLogger
from litellm.integrations.prometheus_helpers import (
PrometheusLabelFactoryContext,
_get_cached_end_user_id_for_cost_tracking,
)
from litellm.integrations.prometheus_helpers.bounded_prometheus_series_tracker import (
BoundedPrometheusSeriesTracker,
)
from litellm.litellm_core_utils.core_helpers import (
get_litellm_metadata_from_kwargs,
get_metadata_variable_name_from_kwargs,
@ -42,6 +46,9 @@ from litellm.proxy._types import (
LiteLLM_UserTable,
UserAPIKeyAuth,
)
from litellm.repositories.organization_repository import OrganizationRepository
from litellm.repositories.team_repository import TeamRepository
from litellm.repositories.user_repository import UserRepository
from litellm.types.integrations.prometheus import *
from litellm.types.integrations.prometheus import (
_sanitize_prometheus_label_name,
@ -78,6 +85,20 @@ class PrometheusLogger(CustomLogger):
# Always initialize label_filters, even for non-premium users
self.label_filters = self._parse_prometheus_config()
# Cache resolved label sets per metric. Several entries in
# ``PrometheusMetricLabels.get_labels`` read module-level toggles
# (e.g. ``litellm.prometheus_emit_stream_label``,
# ``litellm.prometheus_emit_rate_limit_labels``) that can be
# changed at runtime. Prometheus counters/gauges/histograms are
# created with a *fixed* ``labelnames`` set; if a runtime call
# to ``get_labels_for_metric`` returned a different set, the
# subsequent ``counter.labels(**_labels)`` would raise a
# ``ValueError`` from the prometheus client. Snapshotting at
# logger init time pins the label set for the lifetime of the
# logger so toggling these flags only takes effect after a
# restart, keeping init-time and runtime label sets in sync.
self._cached_metric_labels: Dict[str, List[str]] = {}
_custom_buckets = litellm.prometheus_latency_buckets
self.latency_buckets = (
tuple(_custom_buckets)
@ -1033,13 +1054,27 @@ class PrometheusLogger(CustomLogger):
self, metric_name: DEFINED_PROMETHEUS_METRICS
) -> List[str]:
"""
Get the labels for a metric, filtered if configured
Get the labels for a metric, filtered if configured.
The result is cached on the instance so the label set used to
construct each Prometheus metric at ``__init__`` time stays in lock
step with the label set passed to ``counter.labels(...)`` at
runtime, even if the underlying module-level toggles consulted by
:meth:`PrometheusMetricLabels.get_labels` (e.g.
``litellm.prometheus_emit_rate_limit_labels``,
``litellm.prometheus_emit_stream_label``) are flipped after the
logger has been created.
"""
cached = self._cached_metric_labels.get(metric_name)
if cached is not None:
return cached
# Get default labels for this metric from PrometheusMetricLabels
default_labels = PrometheusMetricLabels.get_labels(metric_name)
# If no label filtering is configured for this metric, use default labels
if metric_name not in self.label_filters:
self._cached_metric_labels[metric_name] = default_labels
return default_labels
# Get configured labels for this metric
@ -1050,6 +1085,7 @@ class PrometheusLogger(CustomLogger):
label for label in default_labels if label in configured_labels
]
self._cached_metric_labels[metric_name] = filtered_labels
return filtered_labels
def _track_end_user_metric_series(
@ -2029,14 +2065,8 @@ class PrometheusLogger(CustomLogger):
Proxy level tracking - failed client side requests
labelnames=[
"end_user",
"hashed_api_key",
"api_key_alias",
REQUESTED_MODEL,
"team",
"team_alias",
] + EXCEPTION_LABELS,
See :attr:`PrometheusMetricLabels.litellm_proxy_failed_requests_metric`
for the authoritative list of labels emitted on this metric.
"""
from litellm.litellm_core_utils.litellm_logging import (
StandardLoggingPayloadSetup,
@ -2059,6 +2089,9 @@ class PrometheusLogger(CustomLogger):
model_id = _metadata.get("model_info", {}).get("id") or request_data.get(
"model_info", {}
).get("id")
rate_limit_category, rate_limit_type = self._extract_rate_limit_labels(
original_exception
)
enum_values = UserAPIKeyLabelValues(
end_user=user_api_key_dict.end_user_id,
user=user_api_key_dict.user_id,
@ -2073,6 +2106,8 @@ class PrometheusLogger(CustomLogger):
status_code=str(status_code),
exception_status=str(status_code),
exception_class=self._get_exception_class_name(original_exception),
rate_limit_category=rate_limit_category,
rate_limit_type=rate_limit_type,
tags=_tags,
route=user_api_key_dict.request_route,
client_ip=_metadata.get("requester_ip_address"),
@ -2843,6 +2878,33 @@ class PrometheusLogger(CustomLogger):
@staticmethod
def _get_exception_class_name(exception: Exception) -> str:
# Some exception types pin the ``exception_class`` label to a legacy
# value for back-compat with existing dashboards (e.g. proxy-side 429s
# keep reporting as "HTTPException"). Honor that opt-in marker before
# deriving the label from the runtime class name. Reading it via
# ``getattr`` keeps this core integrations module free of a transitive
# ``fastapi`` dependency.
legacy_class_name = getattr(exception, "prometheus_exception_class_name", None)
if isinstance(legacy_class_name, str) and legacy_class_name:
return legacy_class_name
# Same back-compat reasoning for ``BudgetExceededError``: the unified
# rate-limit error work attached ``.llm_provider`` to budget errors
# too (so callbacks reading ``StandardLoggingPayload`` get provider
# attribution). Without this short-circuit, the provider prefix below
# would silently flip the label from "BudgetExceededError" to e.g.
# "Openai.BudgetExceededError" and break dashboards keyed on the
# original value.
try:
from litellm.exceptions import BudgetExceededError
except ImportError:
BudgetExceededError = None # type: ignore[assignment,misc]
if BudgetExceededError is not None and isinstance(
exception, BudgetExceededError
):
return "BudgetExceededError"
exception_class_name = ""
if hasattr(exception, "llm_provider"):
exception_class_name = getattr(exception, "llm_provider") or ""
@ -2857,6 +2919,27 @@ class PrometheusLogger(CustomLogger):
exception_class_name += exception.__class__.__name__
return exception_class_name
@staticmethod
def _extract_rate_limit_labels(
exception: Optional[Exception],
) -> Tuple[Optional[str], Optional[str]]:
"""
Pull the unified ``category`` / ``rate_limit_type`` fields off any
exception that declares them (``litellm.RateLimitError`` and bare-
Exception subclasses like ``BudgetExceededError``).
Values are validated against the :class:`RateLimitErrorCategory` /
:class:`RateLimitType` enums so unrelated third-party exceptions that
happen to declare ``.category`` / ``.rate_limit_type`` string attributes
can't leak garbage into Prometheus label cardinality.
"""
if exception is None:
return None, None
return (
validate_rate_limit_category(getattr(exception, "category", None)),
validate_rate_limit_type(getattr(exception, "rate_limit_type", None)),
)
async def log_success_fallback_event(
self, original_model_group: str, kwargs: dict, original_exception: Exception
):
@ -3198,12 +3281,12 @@ class PrometheusLogger(CustomLogger):
page_size: int, page: int
) -> Tuple[List[LiteLLM_UserTable], Optional[int]]:
skip = (page - 1) * page_size
users = await prisma_client.db.litellm_usertable.find_many(
users = await UserRepository(prisma_client).table.find_many(
skip=skip,
take=page_size,
order={"created_at": "desc"},
)
total_count = await prisma_client.db.litellm_usertable.count()
total_count = await UserRepository(prisma_client).table.count()
return users, total_count
await self._initialize_budget_metrics(
@ -3226,13 +3309,13 @@ class PrometheusLogger(CustomLogger):
async def fetch_orgs(page_size: int, page: int) -> Tuple[list, Optional[int]]:
skip = (page - 1) * page_size
orgs = await prisma_client.db.litellm_organizationtable.find_many(
orgs = await OrganizationRepository(prisma_client).table.find_many(
skip=skip,
take=page_size,
order={"created_at": "desc"},
include={"litellm_budget_table": True},
)
total_count = await prisma_client.db.litellm_organizationtable.count()
total_count = await OrganizationRepository(prisma_client).table.count()
return orgs, total_count
await self._initialize_budget_metrics(
@ -3300,14 +3383,14 @@ class PrometheusLogger(CustomLogger):
try:
# Get total user count
total_users = await prisma_client.db.litellm_usertable.count()
total_users = await UserRepository(prisma_client).table.count()
self.litellm_total_users_metric.set(total_users)
verbose_logger.debug(
f"Prometheus: set litellm_total_users to {total_users}"
)
# Get total team count
total_teams = await prisma_client.db.litellm_teamtable.count()
total_teams = await TeamRepository(prisma_client).table.count()
self.litellm_teams_count_metric.set(total_teams)
verbose_logger.debug(
f"Prometheus: set litellm_teams_count to {total_teams}"

9
litellm/integrations/websearch_interception/handler.py
View file

@ -1339,8 +1339,13 @@ class WebSearchInterceptionLogger(CustomLogger):
websearch_params: WebSearchInterceptionConfig = {}
if "websearch_interception_params" in litellm_settings:
websearch_params = litellm_settings["websearch_interception_params"]
elif "websearch_interception" in callback_specific_params:
websearch_params = callback_specific_params["websearch_interception"]
elif "websearch_interception" in callback_specific_params and isinstance(
callback_specific_params["websearch_interception"], dict
):
websearch_params = cast(
WebSearchInterceptionConfig,
callback_specific_params["websearch_interception"],
)
# Use classmethod to initialize from config
return WebSearchInterceptionLogger.from_config_yaml(websearch_params)

2
litellm/litellm_core_utils/cli_token_utils.py
View file

@ -37,7 +37,7 @@ def get_litellm_gateway_api_key(
"""
Get the stored CLI API key for use with LiteLLM SDK.
This function reads the token file created by `litellm-proxy login`
This function reads the token file created by `lite login`
and returns the API key for use in Python scripts.
Args:

6
litellm/litellm_core_utils/exception_mapping_utils.py
View file

@ -655,7 +655,11 @@ def exception_type( # type: ignore # noqa: PLR0915
custom_llm_provider == "anthropic"
or custom_llm_provider == "anthropic_text"
): # one of the anthropics
if "prompt is too long" in error_str or "prompt: length" in error_str:
if (
"prompt is too long" in error_str
or "prompt: length" in error_str
or ExceptionCheckers.is_error_str_context_window_exceeded(error_str)
):
exception_mapping_worked = True
raise ContextWindowExceededError(
message="AnthropicError - {}".format(error_str),

1
litellm/litellm_core_utils/get_litellm_params.py
View file

@ -34,6 +34,7 @@ _OPTIONAL_KWARGS_KEYS = frozenset(
"aws_bedrock_runtime_endpoint",
"tpm",
"rpm",
"use_xai_oauth",
}
)

19
litellm/litellm_core_utils/litellm_logging.py
View file

@ -37,6 +37,10 @@ from litellm import (
turn_off_message_logging,
)
from litellm._logging import _is_debugging_on, _redact_string, verbose_logger
from litellm.exceptions import (
validate_rate_limit_category,
validate_rate_limit_type,
)
from litellm._uuid import uuid
from litellm.batches.batch_utils import _handle_completed_batch
from litellm.caching.caching import DualCache, InMemoryCache
@ -5318,12 +5322,27 @@ class StandardLoggingPayloadSetup:
else str(original_exception)
)
# Duck-typed read so bare-Exception subclasses like
# `litellm.BudgetExceededError` can participate without joining the
# RateLimitError hierarchy (which would break `except BudgetExceededError`).
# Validated against the enum value sets so a third-party exception that
# happens to declare a `.category` or `.rate_limit_type` string attribute
# can't leak garbage into the payload or Prometheus label cardinality.
rate_limit_category = validate_rate_limit_category(
getattr(original_exception, "category", None)
)
rate_limit_type = validate_rate_limit_type(
getattr(original_exception, "rate_limit_type", None)
)
return StandardLoggingPayloadErrorInformation(
error_code=error_status,
error_class=error_class,
llm_provider=_llm_provider_in_exception,
traceback=traceback_info,
error_message=error_message if original_exception else "",
error_rate_limit_category=rate_limit_category,
error_rate_limit_type=rate_limit_type,
)
@staticmethod

5
litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py
View file

@ -633,11 +633,6 @@ def convert_to_model_response_object( # noqa: PLR0915
thinking_blocks = choice["message"]["thinking_blocks"]
provider_specific_fields["thinking_blocks"] = thinking_blocks
if reasoning_content:
provider_specific_fields["reasoning_content"] = (
reasoning_content
)
message = Message(
content=content,
role=choice["message"]["role"] or "assistant",

78
litellm/litellm_core_utils/prompt_templates/factory.py
View file

@ -3653,17 +3653,13 @@ from litellm.types.llms.bedrock import ContentBlock as BedrockContentBlock
from litellm.types.llms.bedrock import DocumentBlock as BedrockDocumentBlock
from litellm.types.llms.bedrock import ImageBlock as BedrockImageBlock
from litellm.types.llms.bedrock import SourceBlock as BedrockSourceBlock
from litellm.types.llms.bedrock import BedrockToolSpec
from litellm.types.llms.bedrock import ToolBlock as BedrockToolBlock
from litellm.types.llms.bedrock import (
ToolInputSchemaBlock as BedrockToolInputSchemaBlock,
)
from litellm.types.llms.bedrock import ToolJsonSchemaBlock as BedrockToolJsonSchemaBlock
from litellm.types.llms.bedrock import SearchResultBlock
from litellm.types.llms.bedrock import ToolResultBlock as BedrockToolResultBlock
from litellm.types.llms.bedrock import (
ToolResultContentBlock as BedrockToolResultContentBlock,
)
from litellm.types.llms.bedrock import ToolSpecBlock as BedrockToolSpecBlock
from litellm.types.llms.bedrock import ToolUseBlock as BedrockToolUseBlock
from litellm.types.llms.bedrock import VideoBlock as BedrockVideoBlock
@ -4294,6 +4290,49 @@ def _deduplicate_bedrock_tool_content(
return _deduplicate_bedrock_content_blocks(tool_content, "toolResult")
def _rename_duplicate_bedrock_document_names(
contents: List[BedrockMessageBlock],
) -> List[BedrockMessageBlock]:
"""
Rename duplicate document names across all messages in a Bedrock request.
Document names are derived from a content hash, so the same file appearing
in multiple conversation turns produces identical names and Bedrock rejects
the request with "Messages can not contain duplicate document names". The
first occurrence keeps its original name so prompt-cache prefixes stay
stable; later occurrences get a deterministic positional suffix
(``_2``, ``_3``, ...), bumped further if the suffixed name already
belongs to another document (e.g. an organic name ending in ``_2``).
"""
used_names: Set[str] = set()
for message in contents:
for block in message.get("content") or []:
document = block.get("document")
if isinstance(document, dict) and document.get("name"):
used_names.add(document["name"])
name_counts: Dict[str, int] = {}
for message in contents:
for block in message.get("content") or []:
document = block.get("document")
if not isinstance(document, dict):
continue
name = document.get("name")
if not name:
continue
count = name_counts.get(name, 0) + 1
name_counts[name] = count
if count > 1:
suffix = count
new_name = f"{name}_{suffix}"
while new_name in used_names:
suffix += 1
new_name = f"{name}_{suffix}"
used_names.add(new_name)
document["name"] = new_name
return contents
def _sort_bedrock_assistant_content_blocks(
blocks: List[BedrockContentBlock],
) -> List[BedrockContentBlock]:
@ -4942,7 +4981,7 @@ class BedrockConverseMessagesProcessor:
llm_provider=llm_provider,
)
return contents
return _rename_duplicate_bedrock_document_names(contents)
@staticmethod
def translate_thinking_blocks_to_reasoning_content_blocks(
@ -5364,7 +5403,7 @@ def _bedrock_converse_messages_pt( # noqa: PLR0915
llm_provider=llm_provider,
)
return contents
return _rename_duplicate_bedrock_document_names(contents)
def make_valid_bedrock_tool_name(input_tool_name: str) -> str:
@ -5496,6 +5535,7 @@ def _bedrock_tools_pt(
]
"""
from litellm.llms.bedrock.common_utils import (
get_bedrock_base_model,
normalize_json_schema_custom_types_to_object,
)
from litellm.litellm_core_utils.prompt_templates.common_utils import unpack_defs
@ -5503,6 +5543,11 @@ def _bedrock_tools_pt(
_valid_json_schema_root_types = frozenset(
("array", "boolean", "integer", "null", "number", "object", "string")
)
# Only Claude on Bedrock honours strict tool schemas; other families
# (Nova, Llama, GPT-OSS) reject the strict field outright.
supports_strict_tools = bool(
model and get_bedrock_base_model(model).startswith("anthropic")
)
tool_block_list: List[BedrockToolBlock] = []
for tool_idx, tool in enumerate(tools):
# Check if tool is already a BedrockToolBlock (e.g., systemTool for Nova grounding)
@ -5548,17 +5593,16 @@ def _bedrock_tools_pt(
normalize_json_schema_custom_types_to_object(parameters)
if parameters.get("type") not in _valid_json_schema_root_types:
parameters["type"] = "object"
tool_input_schema = BedrockToolInputSchemaBlock(
json=BedrockToolJsonSchemaBlock(
type=parameters["type"],
properties=parameters.get("properties", {}),
required=parameters.get("required", []),
)
tool_block = cast(
BedrockToolBlock,
BedrockToolSpec(
name=name,
description=description,
parameters=parameters,
strict=tool.get("function", {}).get("strict", None),
supports_strict_tools=supports_strict_tools,
),
)
tool_spec = BedrockToolSpecBlock(
inputSchema=tool_input_schema, name=name, description=description
)
tool_block = BedrockToolBlock(toolSpec=tool_spec)
tool_block_list.append(tool_block)
## ADD CACHE POINT TOOL BLOCK ##

10
litellm/litellm_core_utils/redact_messages.py
View file

@ -17,6 +17,10 @@ from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.core_helpers import (
get_metadata_variable_name_from_kwargs,
)
from litellm.llms.vertex_ai.common_utils import (
redact_vertex_ai_metadata_from_litellm_params,
redact_vertex_ai_metadata_from_logged_object,
)
from litellm.secret_managers.main import str_to_bool
from litellm.types.utils import StandardCallbackDynamicParams
@ -119,10 +123,12 @@ def _redact_standard_logging_object(model_call_details: dict):
# ResponsesAPIResponse format - redact content in output items
if isinstance(response.get("output"), list):
_redact_responses_api_output_dict(response["output"], redacted_str)
redact_vertex_ai_metadata_from_logged_object(response)
elif isinstance(response, dict) and "choices" in response:
# ModelResponse dict format - redact content in choices
if isinstance(response.get("choices"), list):
_redact_model_response_dict_choices(response["choices"], redacted_str)
redact_vertex_ai_metadata_from_logged_object(response)
elif isinstance(response, str):
standard_logging_object["response"] = redacted_str
else:
@ -164,6 +170,7 @@ def perform_redaction(model_call_details: dict, result):
model_call_details["prompt"] = ""
model_call_details["input"] = ""
_redact_standard_logging_object(model_call_details)
redact_vertex_ai_metadata_from_litellm_params(model_call_details)
# Redact streaming response
if (
@ -174,6 +181,7 @@ def perform_redaction(model_call_details: dict, result):
if hasattr(_streaming_response, "choices"):
for choice in _streaming_response.choices:
_redact_choice_content(choice)
redact_vertex_ai_metadata_from_logged_object(_streaming_response)
elif hasattr(_streaming_response, "output"):
_redact_responses_api_output(_streaming_response.output)
# Redact reasoning field in ResponsesAPIResponse
@ -200,12 +208,14 @@ def perform_redaction(model_call_details: dict, result):
if hasattr(_result, "choices") and _result.choices is not None:
for choice in _result.choices:
_redact_choice_content(choice)
redact_vertex_ai_metadata_from_logged_object(_result)
elif isinstance(_result, dict) and "choices" in _result:
# Handle dict representation of ModelResponse (e.g., from model_dump())
if _result.get("choices") is not None:
_redact_model_response_dict_choices(
_result["choices"], "redacted-by-litellm"
)
redact_vertex_ai_metadata_from_logged_object(_result)
elif isinstance(_result, dict) and "output" in _result:
if isinstance(_result.get("output"), list):
_redact_responses_api_output_dict(

49
litellm/litellm_core_utils/streaming_chunk_builder_utils.py
View file

@ -20,6 +20,7 @@ from litellm.types.utils import (
ServerToolUse,
Usage,
)
from litellm._logging import verbose_logger
from litellm.utils import print_verbose, token_counter
if TYPE_CHECKING:
@ -79,6 +80,54 @@ class ChunkProcessor:
model_response._hidden_params = chunk.get("_hidden_params", {})
return model_response
@staticmethod
def apply_provider_assembled_streaming_metadata(
response: ModelResponse,
chunks: List[Any],
logging_obj: Optional[Any] = None,
) -> None:
if not chunks:
return
model = getattr(response, "model", None)
if not model:
return
custom_llm_provider = None
if logging_obj is not None:
custom_llm_provider = logging_obj.model_call_details.get(
"custom_llm_provider"
)
try:
from litellm.litellm_core_utils.get_llm_provider_logic import (
get_llm_provider,
)
from litellm.types.utils import LlmProviders
from litellm.utils import ProviderConfigManager
if custom_llm_provider:
provider = LlmProviders(custom_llm_provider)
else:
_, provider_str, _, _ = get_llm_provider(model)
provider = LlmProviders(provider_str)
provider_config = ProviderConfigManager.get_provider_chat_config(
model=model,
provider=provider,
)
if provider_config is not None:
provider_config.apply_assembled_streaming_response_metadata(
response=response,
chunks=chunks,
)
except Exception as e:
verbose_logger.debug(
"apply_provider_assembled_streaming_metadata failed for model=%s: %s",
model,
e,
)
@staticmethod
def _get_chunk_id(chunks: List[Dict[str, Any]]) -> str:
"""

49
litellm/llms/anthropic/chat/transformation.py
View file

@ -1455,10 +1455,15 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
_value = self._map_stop_sequences(value)
if _value is not None:
optional_params["stop_sequences"] = _value
elif param == "temperature":
optional_params["temperature"] = value
elif param == "top_p":
optional_params["top_p"] = value
elif param == "temperature" or param == "top_p":
AnthropicConfig._apply_sampling_param(
optional_params=optional_params,
model=model,
param=param,
value=value,
drop_params=drop_params,
output_key=param,
)
elif param == "response_format" and isinstance(value, dict):
if any(
substring in model
@ -1607,6 +1612,15 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
)
return _tool
def should_strip_billing_metadata(self) -> bool:
"""
Whether to drop x-anthropic-billing-header system blocks before sending upstream.
The first-party Anthropic API uses these blocks for Claude Code attribution, so the
base config keeps them. Providers that reject them (e.g. Bedrock) override this to True.
"""
return False
def translate_system_message(
self, messages: List[AllMessageValues]
) -> List[AnthropicSystemMessageContent]:
@ -1614,7 +1628,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
Translate system message to anthropic format.
Removes system message from the original list and returns a new list of anthropic system message content.
Filters out system messages containing x-anthropic-billing-header metadata.
When should_strip_billing_metadata() is True, x-anthropic-billing-header system blocks are dropped.
"""
system_prompt_indices = []
anthropic_system_message_list: List[AnthropicSystemMessageContent] = []
@ -1626,10 +1640,9 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
# Skip empty text blocks - Anthropic API raises errors for empty text
if not system_message_block["content"]:
continue
# Skip system messages containing x-anthropic-billing-header metadata
if system_message_block["content"].startswith(
"x-anthropic-billing-header:"
):
if self.should_strip_billing_metadata() and system_message_block[
"content"
].startswith("x-anthropic-billing-header:"):
continue
anthropic_system_message_content = AnthropicSystemMessageContent(
type="text",
@ -1648,9 +1661,9 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
text_value = _content.get("text")
if _content.get("type") == "text" and not text_value:
continue
# Skip system messages containing x-anthropic-billing-header metadata
if (
_content.get("type") == "text"
self.should_strip_billing_metadata()
and _content.get("type") == "text"
and text_value
and text_value.startswith("x-anthropic-billing-header:")
):
@ -1967,6 +1980,20 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
optional_params.pop("is_vertex_request", None)
optional_params.pop("client_metadata", None)
# ``top_k`` is a provider-specific kwarg that bypasses
# ``map_openai_params``; gate it here, the single boundary shared by
# the direct Anthropic, Bedrock invoke, Vertex, and Azure paths.
top_k = optional_params.pop("top_k", None)
if top_k is not None:
AnthropicConfig._apply_sampling_param(
optional_params=optional_params,
model=model,
param="top_k",
value=top_k,
drop_params=litellm_params.get("drop_params") is True,
output_key="top_k",
)
data = {
"model": model,
"messages": anthropic_messages,

112
litellm/llms/anthropic/common_utils.py
View file

@ -272,23 +272,68 @@ class AnthropicModelInfo(BaseLLMModelInfo):
)
@staticmethod
def _supports_model_capability(model: str, key: str) -> bool:
"""Check a boolean capability ``key`` in the model map.
def _supports_sampling_params(model: str) -> bool:
"""Claude 4.7+ (Opus 4.7/4.8, Fable 5) removed sampling params: the API
rejects ``top_p``, ``top_k``, and any ``temperature`` other than 1 with
a 400 ("`temperature` is deprecated for this model").
Strips bedrock/vertex prefixes so a provider-routed Claude still
resolves to the Anthropic model-map entry.
"""
from litellm.utils import _supports_factory
Driven by the ``supports_sampling_params`` flag in the model map; the
name check remains only as a fallback for provider-routed ids whose
map entries predate the flag."""
flag = AnthropicModelInfo._get_model_capability(
model, "supports_sampling_params"
)
if flag is not None:
return flag
model_lower = model.lower()
return not any(
v in model_lower
for v in (
"fable",
"opus-4-7",
"opus_4_7",
"opus-4.7",
"opus_4.7",
"opus-4-8",
"opus_4_8",
"opus-4.8",
"opus_4.8",
)
)
try:
if _supports_factory(
model=model,
custom_llm_provider="anthropic",
key=key,
):
return True
except Exception:
pass
@staticmethod
def _apply_sampling_param(
optional_params: dict,
model: str,
param: str,
value: Any,
drop_params: bool,
output_key: str,
) -> None:
"""Forward ``temperature``/``top_p``/``top_k`` to
``optional_params[output_key]`` unless the model removed sampling
params, in which case drop the param (with drop_params) or raise a
clean client-side 400."""
if AnthropicModelInfo._supports_sampling_params(model) or (
param == "temperature" and value == 1
):
optional_params[output_key] = value
elif not (litellm.drop_params or drop_params):
supported_hint = (
"Only temperature=1 is supported. " if param == "temperature" else ""
)
raise litellm.utils.UnsupportedParamsError(
message=(
f"{model} does not support {param}={value}. {supported_hint}"
"To drop unsupported params, set `litellm.drop_params = True`."
),
status_code=400,
)
@staticmethod
def _model_map_lookup_candidates(model: str) -> List[str]:
"""Model-map keys to try for ``model``, stripping bedrock/vertex
prefixes so a provider-routed Claude still resolves to its entry."""
candidates = [model]
for prefix in (
"bedrock/converse/",
@ -307,15 +352,40 @@ class AnthropicModelInfo(BaseLLMModelInfo):
candidates.append(f"bedrock/{base}")
except Exception:
pass
return candidates
@staticmethod
def _get_model_capability(model: str, key: str) -> Optional[bool]:
"""Read boolean capability ``key`` from the model map, or None when
no entry declares it."""
try:
for cand in candidates:
if cand in litellm.model_cost and (
litellm.model_cost[cand].get(key) is True
):
return True
for cand in AnthropicModelInfo._model_map_lookup_candidates(model):
value = litellm.model_cost.get(cand, {}).get(key)
if isinstance(value, bool):
return value
except Exception:
pass
return False
return None
@staticmethod
def _supports_model_capability(model: str, key: str) -> bool:
"""Check a boolean capability ``key`` in the model map.
Strips bedrock/vertex prefixes so a provider-routed Claude still
resolves to the Anthropic model-map entry.
"""
from litellm.utils import _supports_factory
try:
if _supports_factory(
model=model,
custom_llm_provider="anthropic",
key=key,
):
return True
except Exception:
pass
return AnthropicModelInfo._get_model_capability(model, key) is True
@staticmethod
def _is_adaptive_thinking_model(model: str) -> bool:

98
litellm/llms/anthropic/experimental_pass_through/adapters/streaming_iterator.py
View file

@ -1,5 +1,6 @@
# What is this?
## Translates OpenAI call to Anthropic `/v1/messages` format
import copy
import json
import traceback
from collections import deque
@ -29,6 +30,98 @@ if TYPE_CHECKING:
from litellm.types.utils import ModelResponseStream
class _CombinedChunkSplitter:
"""
Splits a streaming chunk that carries BOTH response content and a
``finish_reason`` into two chunks: a content-only chunk followed by a
finish-only chunk.
``AnthropicStreamWrapper`` (via ``translate_streaming_openai_response_to_anthropic``)
assumes content and ``finish_reason`` never arrive in the same chunk — true for
real provider streams, but false for fake-streamed providers (e.g. Vertex AI
Gemma ``:predict``) where ``MockResponseIterator`` collapses the entire response
into a single chunk. Without this split the assumption causes all content to be
silently dropped (only the ``message_delta`` stop event is emitted).
Supports both sync and async iteration, since ``AnthropicStreamWrapper`` exposes
both ``__next__`` and ``__anext__``. An instance is single-mode: callers must
iterate it either synchronously or asynchronously, never both — the two modes
hold independent iterator references on the upstream stream and mixing them
would advance them out of sync.
"""
def __init__(self, completion_stream: Any):
self._stream = completion_stream
self._sync_iter: Optional[Iterator[Any]] = None
self._async_iter: Optional[AsyncIterator[Any]] = None
self._buffer: deque = deque()
@staticmethod
def _is_combined(chunk: Any) -> bool:
"""True if ``chunk`` carries response content AND a finish_reason."""
choices = getattr(chunk, "choices", None)
if not choices:
return False
choice = choices[0]
if getattr(choice, "finish_reason", None) is None:
return False
delta = getattr(choice, "delta", None)
if delta is None:
return False
return bool(
getattr(delta, "content", None)
or getattr(delta, "tool_calls", None)
or getattr(delta, "reasoning_content", None)
or getattr(delta, "thinking_blocks", None)
)
@staticmethod
def _split(chunk: Any) -> List[Any]:
"""Return ``[chunk]``, or ``[content_chunk, finish_chunk]`` if combined."""
if not _CombinedChunkSplitter._is_combined(chunk):
return [chunk]
# Content chunk: keep the delta payload, clear the finish_reason.
content_chunk = copy.deepcopy(chunk)
content_chunk.choices[0].finish_reason = None
# Finish chunk: keep finish_reason (and usage), clear the delta payload.
finish_chunk = copy.deepcopy(chunk)
finish_delta = finish_chunk.choices[0].delta
finish_delta.content = None
if hasattr(finish_delta, "tool_calls"):
finish_delta.tool_calls = None
if hasattr(finish_delta, "reasoning_content"):
finish_delta.reasoning_content = None
if hasattr(finish_delta, "thinking_blocks"):
finish_delta.thinking_blocks = None
return [content_chunk, finish_chunk]
def __iter__(self) -> "Iterator[Any]":
return self
def __next__(self) -> Any:
if self._buffer:
return self._buffer.popleft()
if self._sync_iter is None:
self._sync_iter = iter(self._stream)
chunk = next(self._sync_iter) # propagates StopIteration when exhausted
self._buffer.extend(self._split(chunk))
return self._buffer.popleft()
def __aiter__(self) -> "AsyncIterator[Any]":
return self
async def __anext__(self) -> Any:
if self._buffer:
return self._buffer.popleft()
if self._async_iter is None:
self._async_iter = self._stream.__aiter__()
chunk = await self._async_iter.__anext__() # propagates StopAsyncIteration
self._buffer.extend(self._split(chunk))
return self._buffer.popleft()
class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
"""
- first chunk return 'message_start'
@ -62,7 +155,10 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
compaction_block: Optional[CompactionBlock] = None,
iterations_usage: Optional[List[UsageIteration]] = None,
):
super().__init__(completion_stream)
# Wrap the upstream stream so chunks that carry both content and a
# finish_reason (fake-streamed providers) are split into two — see
# _CombinedChunkSplitter.
super().__init__(_CombinedChunkSplitter(completion_stream))
self.model = model
# Mapping of truncated tool names to original names (for OpenAI's 64-char limit)
self.tool_name_mapping = tool_name_mapping or {}

13
litellm/llms/anthropic/experimental_pass_through/messages/transformation.py
View file

@ -84,6 +84,15 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
if isinstance(content, list):
_process_content_list(content)
def should_strip_billing_metadata(self) -> bool:
"""
Whether to drop x-anthropic-billing-header system blocks before sending upstream.
The first-party Anthropic API uses these blocks for Claude Code attribution, so the
base config keeps them. Providers that reject them override this to True.
"""
return False
@staticmethod
def _filter_billing_headers_from_system(system_param):
"""
@ -286,14 +295,12 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
optional_params=anthropic_messages_optional_request_params,
)
# Filter out x-anthropic-billing-header from system messages
system_param = anthropic_messages_optional_request_params.get("system")
if system_param is not None:
if self.should_strip_billing_metadata() and system_param is not None:
filtered_system = self._filter_billing_headers_from_system(system_param)
if filtered_system is not None and len(filtered_system) > 0:
anthropic_messages_optional_request_params["system"] = filtered_system
else:
# Remove system parameter if all content was filtered out
anthropic_messages_optional_request_params.pop("system", None)
# Transform context_management from OpenAI format to Anthropic format if needed

16
litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py
View file

@ -155,10 +155,24 @@ class AnthropicResponsesStreamWrapper:
event.get("delta", "") if isinstance(event, dict) else ""
)
block_idx = (
self._item_id_to_block_index.get(item_id, self._current_block_index)
self._item_id_to_block_index.get(item_id, -1)
if item_id
else self._current_block_index
)
if block_idx < 0:
# Some providers (e.g. LMStudio) skip response.output_item.added,
# so no text block is open yet; synthesize content_block_start
# instead of emitting a delta with index -1
block_idx = self._next_block_index()
if item_id:
self._item_id_to_block_index[item_id] = block_idx
self._chunk_queue.append(
{
"type": "content_block_start",
"index": block_idx,
"content_block": {"type": "text", "text": ""},
}
)
self._chunk_queue.append(
{
"type": "content_block_delta",

39
litellm/llms/azure/azure.py
View file

@ -43,7 +43,10 @@ from .common_utils import (
process_azure_headers,
select_azure_base_url_or_endpoint,
)
from .image_generation import get_azure_image_generation_config
from .image_generation import (
AzureFoundryMAIImageGenerationConfig,
get_azure_image_generation_config,
)
from .image_generation.http_utils import azure_deployment_image_generation_json_body
@ -1097,10 +1100,14 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
)
def create_azure_base_url(
self, azure_client_params: dict, model: Optional[str]
self,
azure_client_params: dict,
model: Optional[str],
base_model: Optional[str] = None,
) -> str:
from litellm.llms.azure_ai.image_generation import (
AzureFoundryFluxImageGenerationConfig,
AzureFoundryMAIImageGenerationConfig,
)
api_base: str = azure_client_params.get(
@ -1112,6 +1119,12 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
if model is None:
model = ""
if AzureFoundryMAIImageGenerationConfig.is_mai_model(base_model or model):
return AzureFoundryMAIImageGenerationConfig.get_mai_image_generation_url(
api_base=api_base,
api_version=api_version,
)
# Handle FLUX 2 models on Azure AI which use a different URL pattern
# e.g., /providers/blackforestlabs/v1/flux-2-pro instead of /openai/deployments/{model}/images/generations
if AzureFoundryFluxImageGenerationConfig.is_flux2_model(model):
@ -1153,10 +1166,10 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
if api_base.endswith("/"):
api_base = api_base.rstrip("/")
api_version: str = azure_client_params.get("api_version", "")
# Use the deployment name (model) for URL construction, not the base_model from data
img_gen_api_base = self.create_azure_base_url(
azure_client_params=azure_client_params,
model=model or data.get("model", ""),
base_model=data.get("model", ""),
)
## LOGGING
@ -1285,9 +1298,10 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
if aimg_generation is True:
return self.aimage_generation(data=data, input=input, logging_obj=logging_obj, model_response=model_response, api_key=api_key, client=client, azure_client_params=azure_client_params, timeout=timeout, headers=headers, model=model) # type: ignore
# Use the deployment name (model) for URL construction, not the base_model from data
img_gen_api_base = self.create_azure_base_url(
azure_client_params=azure_client_params, model=model
azure_client_params=azure_client_params,
model=model,
base_model=base_model,
)
## LOGGING
@ -1309,6 +1323,21 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
data=data,
headers=headers,
)
provider_config = get_azure_image_generation_config(
data.get("model", "dall-e-2")
)
if isinstance(provider_config, AzureFoundryMAIImageGenerationConfig):
return provider_config.transform_image_generation_response(
model=data.get("model", "dall-e-2"),
raw_response=httpx_response,
model_response=model_response or ImageResponse(),
logging_obj=logging_obj,
request_data=data,
optional_params=data,
litellm_params=data,
encoding=litellm.encoding,
)
response = httpx_response.json()
## LOGGING

3
litellm/llms/azure/image_generation/__init__.py
View file

@ -1,4 +1,5 @@
from litellm._logging import verbose_logger
from litellm.llms.azure_ai.image_generation import AzureFoundryMAIImageGenerationConfig
from litellm.llms.base_llm.image_generation.transformation import (
BaseImageGenerationConfig,
)
@ -24,6 +25,8 @@ def get_azure_image_generation_config(model: str) -> BaseImageGenerationConfig:
return AzureDallE2ImageGenerationConfig()
elif "dalle3" in model:
return AzureDallE3ImageGenerationConfig()
elif AzureFoundryMAIImageGenerationConfig.is_mai_model(model):
return AzureFoundryMAIImageGenerationConfig()
else:
verbose_logger.debug(
f"Using AzureGPTImageGenerationConfig for model: {model}. This follows the gpt-image model format."

3
litellm/llms/azure_ai/anthropic/messages_transformation.py
View file

@ -21,6 +21,9 @@ class AzureAnthropicMessagesConfig(AnthropicMessagesConfig):
and Azure endpoint format.
"""
def should_strip_billing_metadata(self) -> bool:
return True
def validate_anthropic_messages_environment(
self,
headers: dict,

3
litellm/llms/azure_ai/anthropic/transformation.py
View file

@ -40,6 +40,9 @@ class AzureAnthropicConfig(AnthropicConfig):
def custom_llm_provider(self) -> Optional[str]:
return "azure_ai"
def should_strip_billing_metadata(self) -> bool:
return True
def validate_environment(
self,
headers: dict,

14
litellm/llms/azure_ai/image_edit/__init__.py
View file

@ -1,21 +1,33 @@
from litellm.llms.azure_ai.image_generation.flux_transformation import (
AzureFoundryFluxImageGenerationConfig,
)
from litellm.llms.azure_ai.image_generation.mai_transformation import (
AzureFoundryMAIImageGenerationConfig,
)
from litellm.llms.base_llm.image_edit.transformation import BaseImageEditConfig
from .flux2_transformation import AzureFoundryFlux2ImageEditConfig
from .mai_transformation import AzureFoundryMAIImageEditConfig
from .transformation import AzureFoundryFluxImageEditConfig
__all__ = ["AzureFoundryFluxImageEditConfig", "AzureFoundryFlux2ImageEditConfig"]
__all__ = [
"AzureFoundryFluxImageEditConfig",
"AzureFoundryFlux2ImageEditConfig",
"AzureFoundryMAIImageEditConfig",
]
def get_azure_ai_image_edit_config(model: str) -> BaseImageEditConfig:
"""
Get the appropriate image edit config for an Azure AI model.
- MAI models use /mai/v1/images/edits with multipart form data and size
- FLUX 2 models use JSON with base64 image
- FLUX 1 models use multipart/form-data
"""
if AzureFoundryMAIImageGenerationConfig.is_mai_model(model):
return AzureFoundryMAIImageEditConfig()
# Check if it's a FLUX 2 model
if AzureFoundryFluxImageGenerationConfig.is_flux2_model(model):
return AzureFoundryFlux2ImageEditConfig()

199
litellm/llms/azure_ai/image_edit/mai_transformation.py Normal file
View file

@ -0,0 +1,199 @@
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, cast
import httpx
from httpx._types import RequestFiles
from litellm.llms.azure_ai.common_utils import AzureFoundryModelInfo
from litellm.llms.azure_ai.image_generation.mai_transformation import (
AzureFoundryMAIImageGenerationConfig,
)
from litellm.llms.openai.common_utils import OpenAIError
from litellm.llms.openai.image_edit.transformation import OpenAIImageEditConfig
from litellm.secret_managers.main import get_secret_str
from litellm.types.images.main import ImageEditOptionalRequestParams
from litellm.types.llms.openai import FileTypes
from litellm.types.router import GenericLiteLLMParams
from litellm.types.utils import ImageResponse
from litellm.utils import convert_to_model_response_object
if TYPE_CHECKING:
from litellm.litellm_core_utils.logging import Logging as LiteLLMLoggingObj
class AzureFoundryMAIImageEditConfig(OpenAIImageEditConfig):
"""Azure AI Foundry MAI image editing (e.g. MAI-Image-2.5)."""
DEFAULT_SIZE = "1024x1024"
def get_supported_openai_params(self, model: str) -> list:
return ["prompt", "image", "model", "n", "size"]
def map_openai_params(
self,
image_edit_optional_params: ImageEditOptionalRequestParams,
model: str,
drop_params: bool,
) -> Dict:
optional_params: Dict[str, Any] = {}
supported_params = self.get_supported_openai_params(model)
for key, value in dict(image_edit_optional_params).items():
if value is None or key in optional_params:
continue
if key in supported_params:
if key == "size" and value:
size_param = cast(str, value)
self._validate_size_param(size_param)
optional_params[key] = size_param
else:
optional_params[key] = value
elif not drop_params:
raise ValueError(
f"Parameter {key} is not supported for model {model}. "
f"Supported parameters are {supported_params}. "
f"Set drop_params=True to drop unsupported parameters."
)
if "size" not in optional_params:
optional_params["size"] = self.DEFAULT_SIZE
return optional_params
def _validate_size_param(self, size: str) -> None:
known_sizes = {
"1024x1024",
"1792x1024",
"1024x1792",
"512x512",
"256x256",
}
if size in known_sizes:
return
if "x" in size:
try:
tuple(map(int, size.lower().split("x", 1)))
return
except ValueError:
raise ValueError(
f"Invalid size format: '{size}'. Expected format 'WIDTHxHEIGHT' (e.g., '1024x1024')."
)
raise ValueError(
f"Unsupported size value: '{size}'. "
f"Use a known size (e.g., '1024x1024') or a custom 'WIDTHxHEIGHT' string."
)
def validate_environment(
self,
headers: dict,
model: str,
api_key: Optional[str] = None,
litellm_params: Optional[dict] = None,
api_base: Optional[str] = None,
) -> dict:
api_key = AzureFoundryModelInfo.get_api_key(api_key)
if not api_key:
raise ValueError(
f"Azure AI API key is required for model {model}. "
"Set AZURE_AI_API_KEY environment variable or pass api_key parameter."
)
headers.update({"api-key": api_key})
return headers
def get_complete_url(
self,
model: str,
api_base: Optional[str],
litellm_params: dict,
) -> str:
api_base = AzureFoundryModelInfo.get_api_base(api_base)
if api_base is None:
raise ValueError(
"Azure AI API base is required. Set AZURE_AI_API_BASE environment variable or pass api_base parameter."
)
api_version = (
litellm_params.get("api_version")
or get_secret_str("AZURE_AI_API_VERSION")
or "preview"
)
return AzureFoundryMAIImageGenerationConfig.get_mai_image_edit_url(
api_base=api_base,
api_version=api_version,
)
def transform_image_edit_request(
self,
model: str,
prompt: Optional[str],
image: Optional[FileTypes],
image_edit_optional_request_params: Dict,
litellm_params: GenericLiteLLMParams,
headers: dict,
) -> Tuple[Dict, RequestFiles]:
request_params = {
"model": model,
**image_edit_optional_request_params,
}
if prompt is not None:
request_params["prompt"] = prompt
data_without_files = {
key: value
for key, value in request_params.items()
if key not in ["image", "mask"]
}
files_list: List[Tuple[str, Any]] = []
if image is not None:
image_list = [image] if not isinstance(image, list) else image
for _image in image_list:
if _image is not None:
self._add_image_to_files(
files_list=files_list,
image=_image,
field_name="image",
)
break
return data_without_files, files_list
def transform_image_edit_response(
self,
model: str,
raw_response: httpx.Response,
logging_obj: "LiteLLMLoggingObj",
) -> ImageResponse:
try:
response = raw_response.json()
except Exception:
raise OpenAIError(
message=raw_response.text, status_code=raw_response.status_code
)
if "usage" in response:
response["usage"] = (
AzureFoundryMAIImageGenerationConfig.normalize_mai_image_usage(
response.get("usage")
)
)
logging_obj.post_call(
input="",
api_key="",
additional_args={"complete_input_dict": {}},
original_response=response,
)
return convert_to_model_response_object(
response_object=response,
model_response_object=ImageResponse(),
response_type="image_generation",
)

4
litellm/llms/azure_ai/image_generation/__init__.py
View file

@ -7,12 +7,14 @@ from .dall_e_2_transformation import AzureFoundryDallE2ImageGenerationConfig
from .dall_e_3_transformation import AzureFoundryDallE3ImageGenerationConfig
from .flux_transformation import AzureFoundryFluxImageGenerationConfig
from .gpt_transformation import AzureFoundryGPTImageGenerationConfig
from .mai_transformation import AzureFoundryMAIImageGenerationConfig
__all__ = [
"AzureFoundryFluxImageGenerationConfig",
"AzureFoundryGPTImageGenerationConfig",
"AzureFoundryDallE2ImageGenerationConfig",
"AzureFoundryDallE3ImageGenerationConfig",
"AzureFoundryMAIImageGenerationConfig",
]
@ -24,6 +26,8 @@ def get_azure_ai_image_generation_config(model: str) -> BaseImageGenerationConfi
return AzureFoundryDallE2ImageGenerationConfig()
elif "dalle3" in model:
return AzureFoundryDallE3ImageGenerationConfig()
elif AzureFoundryMAIImageGenerationConfig.is_mai_model(model):
return AzureFoundryMAIImageGenerationConfig()
elif "flux" in model:
return AzureFoundryFluxImageGenerationConfig()
else:

26
litellm/llms/azure_ai/image_generation/cost_calculator.py
View file

@ -1,6 +1,9 @@
from typing import Any
import litellm
from litellm.litellm_core_utils.llm_cost_calc.utils import (
calculate_image_response_cost_from_usage,
)
from litellm.types.utils import ImageResponse
@ -9,19 +12,28 @@ def cost_calculator(
image_response: Any,
) -> float:
"""
Recraft image generation cost calculator
Azure AI image generation cost calculator
"""
_model_info = litellm.get_model_info(
model=model,
custom_llm_provider=litellm.LlmProviders.AZURE_AI.value,
)
output_cost_per_image: float = _model_info.get("output_cost_per_image") or 0.0
num_images: int = 0
if isinstance(image_response, ImageResponse):
token_based_cost = calculate_image_response_cost_from_usage(
model=model,
image_response=image_response,
custom_llm_provider=litellm.LlmProviders.AZURE_AI.value,
)
if token_based_cost is not None:
return token_based_cost
output_cost_per_image: float = _model_info.get("output_cost_per_image") or 0.0
num_images: int = 0
if image_response.data:
num_images = len(image_response.data)
return output_cost_per_image * num_images
else:
raise ValueError(
f"image_response must be of type ImageResponse got type={type(image_response)}"
)
raise ValueError(
f"image_response must be of type ImageResponse got type={type(image_response)}"
)

236
litellm/llms/azure_ai/image_generation/mai_transformation.py Normal file
View file

@ -0,0 +1,236 @@
from typing import TYPE_CHECKING, Any, Dict, List, Optional
import httpx
from litellm.llms.base_llm.image_generation.transformation import (
BaseImageGenerationConfig,
)
from litellm.llms.openai.common_utils import OpenAIError
from litellm.types.llms.openai import OpenAIImageGenerationOptionalParams
from litellm.types.utils import ImageResponse
from litellm.utils import convert_to_model_response_object
if TYPE_CHECKING:
from litellm.litellm_core_utils.logging import Logging as LiteLLMLoggingObj
class AzureFoundryMAIImageGenerationConfig(BaseImageGenerationConfig):
"""Azure AI Foundry MAI image generation (e.g. MAI-Image-2.5)."""
DEFAULT_WIDTH = 1024
DEFAULT_HEIGHT = 1024
@staticmethod
def get_mai_image_generation_url(
api_base: Optional[str],
api_version: Optional[str],
) -> str:
if api_base is None:
raise ValueError("api_base is required for Azure AI MAI image generation")
api_version = api_version or "preview"
path, separator, query = api_base.partition("?")
path = path.rstrip("/")
if "/mai/" in path:
prefix, _, _ = path.partition("/images/")
path = f"{prefix}/images/generations"
else:
path = f"{path}/mai/v1/images/generations"
if separator:
return f"{path}?{query}"
return f"{path}?api-version={api_version}"
@staticmethod
def get_mai_image_edit_url(
api_base: Optional[str],
api_version: Optional[str],
) -> str:
if api_base is None:
raise ValueError("api_base is required for Azure AI MAI image editing")
api_version = api_version or "preview"
path, separator, query = api_base.partition("?")
path = path.rstrip("/")
if "/mai/" in path:
prefix, _, _ = path.partition("/images/")
path = f"{prefix}/images/edits"
else:
path = f"{path}/mai/v1/images/edits"
if separator:
return f"{path}?{query}"
return f"{path}?api-version={api_version}"
@staticmethod
def is_mai_model(model: str) -> bool:
model_normalized = model.lower().replace("-", "").replace("_", "")
return "maiimage" in model_normalized
@staticmethod
def normalize_mai_image_usage(usage: Optional[Dict[str, Any]]) -> Dict[str, Any]:
"""Map Azure MAI usage fields to OpenAI ImageUsage schema."""
if usage is None:
return {
"input_tokens": 0,
"input_tokens_details": {"image_tokens": 0, "text_tokens": 0},
"output_tokens": 0,
"total_tokens": 0,
}
normalized_usage = dict(usage)
input_tokens_details = normalized_usage.get("input_tokens_details")
if not isinstance(input_tokens_details, dict):
input_tokens_details = {}
text_tokens = normalized_usage.get("num_input_text_tokens")
if text_tokens is None:
text_tokens = input_tokens_details.get("text_tokens")
if text_tokens is None:
text_tokens = normalized_usage.get("input_tokens", 0) or 0
image_tokens = normalized_usage.get("num_input_image_tokens")
if image_tokens is None:
image_tokens = input_tokens_details.get("image_tokens")
if image_tokens is None:
image_tokens = 0
output_tokens = normalized_usage.get("output_tokens")
if output_tokens is None:
output_tokens = normalized_usage.get("num_output_tokens")
if output_tokens is None:
output_tokens = normalized_usage.get("output_image_tokens")
if output_tokens is None:
output_tokens = 0
input_tokens = normalized_usage.get("input_tokens")
if input_tokens is None:
input_tokens = text_tokens + image_tokens
total_tokens = normalized_usage.get("total_tokens")
if total_tokens is None:
total_tokens = input_tokens + output_tokens
normalized_usage.update(
{
"input_tokens": input_tokens,
"input_tokens_details": {
"image_tokens": image_tokens,
"text_tokens": text_tokens,
},
"output_tokens": output_tokens,
"total_tokens": total_tokens,
}
)
return normalized_usage
def get_supported_openai_params(
self, model: str
) -> List[OpenAIImageGenerationOptionalParams]:
return ["n", "size"]
def map_openai_params(
self,
non_default_params: dict,
optional_params: dict,
model: str,
drop_params: bool,
) -> dict:
supported_params = self.get_supported_openai_params(model)
for k, v in non_default_params.items():
if k in optional_params:
continue
if k in supported_params:
if k == "size" and v:
self._map_size_param(v, optional_params)
else:
optional_params[k] = v
elif k in ("width", "height"):
optional_params[k] = v
elif not drop_params:
raise ValueError(
f"Parameter {k} is not supported for model {model}. "
f"Supported parameters are {supported_params} and width/height. "
f"Set drop_params=True to drop unsupported parameters."
)
if "width" not in optional_params:
optional_params["width"] = self.DEFAULT_WIDTH
if "height" not in optional_params:
optional_params["height"] = self.DEFAULT_HEIGHT
optional_params.pop("size", None)
return optional_params
def _map_size_param(self, size: str, optional_params: dict) -> None:
size_mapping = {
"1024x1024": (1024, 1024),
"1792x1024": (1792, 1024),
"1024x1792": (1024, 1792),
"512x512": (512, 512),
"256x256": (256, 256),
}
if size in size_mapping:
width, height = size_mapping[size]
optional_params["width"] = width
optional_params["height"] = height
elif "x" in size:
try:
width, height = map(int, size.lower().split("x"))
optional_params["width"] = width
optional_params["height"] = height
except ValueError:
raise ValueError(
f"Invalid size format: '{size}'. Expected format 'WIDTHxHEIGHT' (e.g., '1024x1024')."
)
else:
raise ValueError(
f"Unsupported size value: '{size}'. "
f"Use a known size (e.g., '1024x1024') or a custom 'WIDTHxHEIGHT' string."
)
def transform_image_generation_response(
self,
model: str,
raw_response: httpx.Response,
model_response: ImageResponse,
logging_obj: "LiteLLMLoggingObj",
request_data: dict,
optional_params: dict,
litellm_params: dict,
encoding: Any,
api_key: Optional[str] = None,
json_mode: Optional[bool] = None,
) -> ImageResponse:
try:
response = raw_response.json()
except Exception:
raise OpenAIError(
message=raw_response.text, status_code=raw_response.status_code
)
if "usage" in response:
response["usage"] = self.normalize_mai_image_usage(response.get("usage"))
logging_obj.post_call(
input=request_data.get("prompt", ""),
api_key=api_key,
additional_args={"complete_input_dict": request_data},
original_response=response,
)
image_response: ImageResponse = convert_to_model_response_object(
response_object=response,
model_response_object=model_response,
response_type="image_generation",
)
width = optional_params.get("width", self.DEFAULT_WIDTH)
height = optional_params.get("height", self.DEFAULT_HEIGHT)
image_response.size = f"{width}x{height}" # type: ignore[assignment]
return image_response

5
litellm/llms/base_llm/base_model_iterator.py
View file

@ -50,6 +50,11 @@ def convert_model_response_to_streaming(
model=model_response.model,
choices=streaming_choices,
)
# Carry usage onto the streaming chunk so fake-streamed responses
# (e.g. Vertex AI Gemma :predict) still report token counts.
usage = getattr(model_response, "usage", None)
if usage is not None:
setattr(processed_chunk, "usage", usage)
return processed_chunk
except Exception as e:
raise ValueError(

8
litellm/llms/base_llm/chat/transformation.py
View file

@ -442,6 +442,14 @@ class BaseConfig(ABC):
"""Hook for providers to post-process streaming responses. Default: pass-through."""
return stream
def apply_assembled_streaming_response_metadata(
self,
response: "ModelResponse",
chunks: List[Any],
) -> None:
"""Hook for providers to merge chunk metadata into assembled streaming responses."""
return None
def calculate_additional_costs(
self, model: str, prompt_tokens: int, completion_tokens: int
) -> Optional[dict]:

20
litellm/llms/base_llm/responses/transformation.py
View file

@ -62,6 +62,26 @@ class BaseResponsesAPIConfig(ABC):
"""
return False
def sign_request(
self,
headers: dict,
optional_params: dict,
request_data: dict,
api_base: str,
api_key: Optional[str] = None,
model: Optional[str] = None,
stream: Optional[bool] = None,
fake_stream: Optional[bool] = None,
) -> Tuple[dict, Optional[bytes]]:
"""Sign the request after the body is finalized.
Default is a no-op (returns headers unchanged, no signed body). Providers
whose endpoint requires request signing (e.g. Bedrock Mantle SigV4)
override this and return the signed body bytes so the handler sends those
exact bytes.
"""
return headers, None
@abstractmethod
def get_supported_openai_params(self, model: str) -> list:
pass

45
litellm/llms/bedrock/chat/converse_transformation.py
View file

@ -920,10 +920,15 @@ class AmazonConverseConfig(BaseConfig):
continue
value = [value]
optional_params["stopSequences"] = value
if param == "temperature":
optional_params["temperature"] = value
if param == "top_p":
optional_params["topP"] = value
if param == "temperature" or param == "top_p":
AnthropicConfig._apply_sampling_param(
optional_params=optional_params,
model=model,
param=param,
value=value,
drop_params=drop_params,
output_key="topP" if param == "top_p" else param,
)
if param == "tools" and isinstance(value, list):
self._apply_tool_call_transformation(
tools=cast(List[OpenAIChatCompletionToolParam], value),
@ -1221,7 +1226,9 @@ class AmazonConverseConfig(BaseConfig):
inference_params["topK"] = inference_params.pop("top_k")
return InferenceConfig(**inference_params)
def _handle_top_k_value(self, model: str, inference_params: dict) -> dict:
def _handle_top_k_value(
self, model: str, inference_params: dict, drop_params: bool = False
) -> dict:
base_model = BedrockModelInfo.get_base_model(model)
val_top_k = None
@ -1230,16 +1237,25 @@ class AmazonConverseConfig(BaseConfig):
elif "top_k" in inference_params:
val_top_k = inference_params.pop("top_k")
if val_top_k:
if val_top_k is not None:
if base_model.startswith("anthropic"):
return {"top_k": val_top_k}
top_k_params: dict = {}
AnthropicConfig._apply_sampling_param(
optional_params=top_k_params,
model=model,
param="top_k",
value=val_top_k,
drop_params=drop_params,
output_key="top_k",
)
return top_k_params
if base_model.startswith("amazon.nova"):
return {"inferenceConfig": {"topK": val_top_k}}
return {}
def _prepare_request_params(
self, optional_params: dict, model: str
self, optional_params: dict, model: str, drop_params: bool = False
) -> Tuple[dict, dict, dict, Optional[OutputConfigBlock]]:
"""Prepare and separate request parameters."""
# Consume the internal ``_output_config_normalized`` marker set by
@ -1338,7 +1354,7 @@ class AmazonConverseConfig(BaseConfig):
# Only set the topK value in for models that support it
additional_request_params.update(
self._handle_top_k_value(model, inference_params)
self._handle_top_k_value(model, inference_params, drop_params)
)
# Filter out internal/MCP-related parameters that shouldn't be sent to the API
@ -1572,6 +1588,7 @@ class AmazonConverseConfig(BaseConfig):
optional_params: dict,
messages: Optional[List[AllMessageValues]] = None,
headers: Optional[dict] = None,
drop_params: bool = False,
) -> CommonRequestObject:
## VALIDATE REQUEST
"""
@ -1618,7 +1635,7 @@ class AmazonConverseConfig(BaseConfig):
additional_request_params,
request_metadata,
output_config,
) = self._prepare_request_params(optional_params, model)
) = self._prepare_request_params(optional_params, model, drop_params)
original_tools = inference_params.pop("tools", [])
@ -1649,12 +1666,14 @@ class AmazonConverseConfig(BaseConfig):
bedrock_tool_config["toolChoice"] = tool_choice_values
data: CommonRequestObject = {
"additionalModelRequestFields": additional_request_params,
"system": system_content_blocks,
"inferenceConfig": self._transform_inference_params(
inference_params=inference_params
),
}
if additional_request_params:
data["additionalModelRequestFields"] = additional_request_params
if system_content_blocks:
data["system"] = system_content_blocks
# Handle all config blocks
for config_name, config_class in self.get_config_blocks().items():
@ -1699,6 +1718,7 @@ class AmazonConverseConfig(BaseConfig):
optional_params=optional_params,
messages=messages,
headers=headers,
drop_params=litellm_params.get("drop_params") is True,
)
bedrock_messages = (
@ -1756,6 +1776,7 @@ class AmazonConverseConfig(BaseConfig):
optional_params=optional_params,
messages=messages,
headers=headers,
drop_params=litellm_params.get("drop_params") is True,
)
## TRANSFORMATION ##

3
litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py
View file

@ -60,6 +60,9 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig):
def custom_llm_provider(self) -> Optional[str]:
return "bedrock"
def should_strip_billing_metadata(self) -> bool:
return True
def get_supported_openai_params(self, model: str) -> List[str]:
return AnthropicConfig.get_supported_openai_params(self, model)

3
litellm/llms/bedrock/claude_platform/transformation.py
View file

@ -17,6 +17,9 @@ class BedrockClaudePlatformConfig(BedrockClaudePlatformMixin, AnthropicConfig):
def custom_llm_provider(self) -> Optional[str]:
return "bedrock"
def should_strip_billing_metadata(self) -> bool:
return True
def validate_environment(
self,
headers: dict,

36
litellm/llms/bedrock/count_tokens/transformation.py
View file

@ -11,6 +11,11 @@ from typing import Any, Dict, List, Optional
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
from litellm.llms.bedrock.common_utils import get_bedrock_base_model
# Placeholder satisfying the Anthropic InvokeModel schema's required
# max_tokens field; CountTokens only counts input, so it has no effect
# on any generation.
DEFAULT_ANTHROPIC_INVOKE_MODEL_MAX_TOKENS = 1024
class BedrockCountTokensConfig(BaseAWSLLM):
"""
@ -32,8 +37,20 @@ class BedrockCountTokensConfig(BaseAWSLLM):
Returns:
'converse' or 'invokeModel'
"""
# If the request has messages in the expected Anthropic format, use converse
if "messages" in request_data and isinstance(request_data["messages"], list):
messages = request_data.get("messages")
if isinstance(messages, list):
# Anthropic content blocks carry a "type" key ({"type": "text", ...});
# Converse blocks don't ({"text": ...}, {"toolUse": ...}). Converse
# rejects Anthropic-shape blocks, so route those to invokeModel,
# which forwards the body verbatim.
for message in messages:
if not isinstance(message, dict):
continue
content = message.get("content")
if isinstance(content, list) and any(
isinstance(block, dict) and "type" in block for block in content
):
return "invokeModel"
return "converse"
# For raw text or other formats, use invokeModel
@ -68,7 +85,7 @@ class BedrockCountTokensConfig(BaseAWSLLM):
{
"input": {
"invokeModel": {
"body": "{...raw model input...}"
"body": "<base64-encoded raw model input>"
}
}
}
@ -168,13 +185,24 @@ class BedrockCountTokensConfig(BaseAWSLLM):
self, request_data: Dict[str, Any]
) -> Dict[str, Any]:
"""Transform to InvokeModel input format."""
import base64
import json
# For InvokeModel, we need to provide the raw body that would be sent to the model
# Remove the 'model' field from the body as it's not part of the model input
body_data = {k: v for k, v in request_data.items() if k != "model"}
return {"input": {"invokeModel": {"body": json.dumps(body_data)}}}
if "messages" in body_data:
# Bedrock validates the body against the model's InvokeModel schema;
# Anthropic Messages bodies require these fields.
body_data.setdefault("anthropic_version", "bedrock-2023-05-31")
body_data.setdefault(
"max_tokens", DEFAULT_ANTHROPIC_INVOKE_MODEL_MAX_TOKENS
)
# The CountTokens API expects invokeModel.body as a base64-encoded blob
encoded_body = base64.b64encode(json.dumps(body_data).encode()).decode()
return {"input": {"invokeModel": {"body": encoded_body}}}
def get_bedrock_count_tokens_endpoint(
self,

134
litellm/llms/bedrock_mantle/responses/transformation.py
View file

@ -1,17 +1,31 @@
"""
Amazon Bedrock Mantle - Responses API backend.
gpt-5.5 / gpt-5.4 on Mantle are exposed ONLY on the `/openai/v1/responses`
path (not the standard `/v1/responses`). Payloads and SSE follow the OpenAI
Mantle serves Responses on two upstream paths: gpt frontier models (gpt-5.5 /
gpt-5.4) on `/openai/v1/responses`, and everything else that supports Responses
(e.g. gpt-oss) on the standard `/v1/responses`. The gate picks the path per
model and injects it via `use_openai_path`. Payloads and SSE follow the OpenAI
Responses spec, so this config inherits OpenAIResponsesAPIConfig and overrides
only the endpoint URL and Bearer authentication.
only the endpoint URL and authentication.
Auth: AWS Bedrock API key as Bearer token (BEDROCK_MANTLE_API_KEY or the
standard AWS_BEARER_TOKEN_BEDROCK), NOT SigV4.
Auth: Bearer token (BEDROCK_MANTLE_API_KEY or the standard
AWS_BEARER_TOKEN_BEDROCK, or litellm_params.api_key) when present; otherwise
AWS SigV4 (service name "bedrock") using the standard credential chain (IAM
role / access key / profile / web identity), signed via the shared
BaseAWSLLM._sign_request after the request body is finalized.
"""
from typing import Optional
import re
from typing import Optional, Tuple
from botocore.exceptions import (
CredentialRetrievalError,
NoCredentialsError,
PartialCredentialsError,
ProfileNotFound,
)
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig
from litellm.secret_managers.main import get_secret_str
from litellm.types.router import GenericLiteLLMParams
@ -29,22 +43,49 @@ _BASE_SUFFIXES_TO_STRIP = (
"/v1",
)
# Standard Mantle host: https://bedrock-mantle.<region>.api.aws (group 1 = region).
_MANTLE_HOST_RE = re.compile(
r"^https?://bedrock-mantle\.([^/.]+)\.api\.aws", re.IGNORECASE
)
class BedrockMantleResponsesAPIConfig(OpenAIResponsesAPIConfig):
def __init__(
self,
aws_signer: Optional[BaseAWSLLM] = None,
use_openai_path: bool = True,
):
super().__init__()
self._aws_signer = aws_signer or BaseAWSLLM()
self.use_openai_path = use_openai_path
@property
def custom_llm_provider(self) -> LlmProviders:
return LlmProviders.BEDROCK_MANTLE
@staticmethod
def _resolve_region(params: dict) -> str:
region = params.get("aws_region_name")
if region:
return region
base = params.get("api_base") or get_secret_str("BEDROCK_MANTLE_API_BASE")
if base:
match = _MANTLE_HOST_RE.match(base.rstrip("/"))
if match:
return match.group(1)
return (
get_secret_str("BEDROCK_MANTLE_REGION")
or get_secret_str("AWS_REGION_NAME")
or get_secret_str("AWS_REGION")
or BEDROCK_MANTLE_DEFAULT_REGION
)
def get_complete_url(
self,
api_base: Optional[str],
litellm_params: dict,
) -> str:
region = (
get_secret_str("BEDROCK_MANTLE_REGION")
or get_secret_str("AWS_REGION")
or BEDROCK_MANTLE_DEFAULT_REGION
)
region = self._resolve_region({**litellm_params, "api_base": api_base})
base = (
api_base
or get_secret_str("BEDROCK_MANTLE_API_BASE")
@ -55,7 +96,13 @@ class BedrockMantleResponsesAPIConfig(OpenAIResponsesAPIConfig):
if base.endswith(suffix):
base = base[: -len(suffix)]
break
return f"{base}/openai/v1/responses"
# For the standard Mantle host (including the default-region base that
# responses/main.py auto-injects into litellm_params.api_base), pin to the
# single resolved region so aws_region_name wins; preserve custom proxy hosts.
if _MANTLE_HOST_RE.match(base):
base = f"https://bedrock-mantle.{region}.api.aws"
path = "/openai/v1/responses" if self.use_openai_path else "/v1/responses"
return f"{base}{path}"
def validate_environment(
self, headers: dict, model: str, litellm_params: Optional[GenericLiteLLMParams]
@ -66,12 +113,8 @@ class BedrockMantleResponsesAPIConfig(OpenAIResponsesAPIConfig):
or get_secret_str("BEDROCK_MANTLE_API_KEY")
or get_secret_str("AWS_BEARER_TOKEN_BEDROCK")
)
if not api_key:
raise ValueError(
"Bedrock Mantle API key is required. Set BEDROCK_MANTLE_API_KEY "
"(or AWS_BEARER_TOKEN_BEDROCK) or pass api_key."
)
headers["Authorization"] = f"Bearer {api_key}"
if api_key:
headers["Authorization"] = f"Bearer {api_key}"
return headers
def supports_native_file_search(self) -> bool:
@ -79,3 +122,58 @@ class BedrockMantleResponsesAPIConfig(OpenAIResponsesAPIConfig):
def supports_native_websocket(self) -> bool:
return False
def sign_request(
self,
headers: dict,
optional_params: dict,
request_data: dict,
api_base: str,
api_key: Optional[str] = None,
model: Optional[str] = None,
stream: Optional[bool] = None,
fake_stream: Optional[bool] = None,
) -> Tuple[dict, Optional[bytes]]:
bearer = (
api_key
or get_secret_str("BEDROCK_MANTLE_API_KEY")
or get_secret_str("AWS_BEARER_TOKEN_BEDROCK")
)
if not bearer:
# SigV4 path. Pin the credential-scope region to the region of the actual
# signing URL (api_base, already region-resolved by get_complete_url) so the
# SigV4 scope and the URL host can never disagree. Resolve from api_base first,
# then fall back to the regular precedence. Also drop any caller Authorization
# so _sign_request's restore-original-Authorization step cannot override the
# SigV4 header.
optional_params = {
**optional_params,
"aws_region_name": self._resolve_region(
{**optional_params, "api_base": api_base}
),
}
headers = {k: v for k, v in headers.items() if k.lower() != "authorization"}
try:
return self._aws_signer._sign_request(
service_name="bedrock",
headers=headers,
optional_params=optional_params,
request_data=request_data,
api_base=api_base,
api_key=bearer,
model=model,
stream=stream,
fake_stream=fake_stream,
)
except (
NoCredentialsError,
PartialCredentialsError,
ProfileNotFound,
CredentialRetrievalError,
) as e:
raise ValueError(
"Bedrock Mantle auth failed: no Bearer token and no usable AWS "
"credentials. Set BEDROCK_MANTLE_API_KEY (or AWS_BEARER_TOKEN_BEDROCK) "
"or pass api_key for Bearer auth, or provide AWS credentials "
"(IAM role / access key / profile / web identity) for SigV4."
) from e

105
litellm/llms/custom_httpx/llm_http_handler.py
View file

@ -2318,6 +2318,31 @@ class BaseLLMHTTPHandler:
# but never included in the outbound provider payload.
request_context["litellm_params"] = dict(litellm_params)
is_stream_request = bool(stream)
if is_stream_request and fake_stream is True:
stream, data = self._prepare_fake_stream_request(
stream=stream,
data=data,
fake_stream=fake_stream,
)
# Sign after the body is final (post-transform/normalize/extra_body and post
# fake-stream prep) so signed bytes match what we send. No-op for providers
# that inherit the default sign_request.
headers, signed_body = responses_api_provider_config.sign_request(
headers=headers,
optional_params=dict(litellm_params),
request_data=data,
api_base=api_base,
api_key=litellm_params.api_key,
model=model,
stream=stream,
fake_stream=fake_stream,
)
body_kwargs: Dict[str, Any] = (
{"data": signed_body} if signed_body is not None else {"json": data}
)
## LOGGING
logging_obj.pre_call(
input=input,
@ -2330,22 +2355,14 @@ class BaseLLMHTTPHandler:
)
try:
if stream:
# For streaming, use stream=True in the request
if fake_stream is True:
stream, data = self._prepare_fake_stream_request(
stream=stream,
data=data,
fake_stream=fake_stream,
)
if is_stream_request:
response = sync_httpx_client.post(
url=api_base,
headers=headers,
json=data,
timeout=timeout
or float(response_api_optional_request_params.get("timeout", 0)),
stream=stream,
**body_kwargs,
)
if fake_stream is True:
return MockResponsesAPIStreamingIterator(
@ -2370,13 +2387,12 @@ class BaseLLMHTTPHandler:
call_type=CallTypes.responses.value,
)
else:
# For non-streaming requests
response = sync_httpx_client.post(
url=api_base,
headers=headers,
json=data,
timeout=timeout
or float(response_api_optional_request_params.get("timeout", 0)),
**body_kwargs,
)
except Exception as e:
raise self._handle_error(
@ -2464,6 +2480,28 @@ class BaseLLMHTTPHandler:
# but never included in the outbound provider payload.
request_context["litellm_params"] = dict(litellm_params)
is_stream_request = bool(stream)
if is_stream_request and fake_stream is True:
stream, data = self._prepare_fake_stream_request(
stream=stream,
data=data,
fake_stream=fake_stream,
)
headers, signed_body = responses_api_provider_config.sign_request(
headers=headers,
optional_params=dict(litellm_params),
request_data=data,
api_base=api_base,
api_key=litellm_params.api_key,
model=model,
stream=stream,
fake_stream=fake_stream,
)
body_kwargs: Dict[str, Any] = (
{"data": signed_body} if signed_body is not None else {"json": data}
)
## LOGGING
logging_obj.pre_call(
input=input,
@ -2476,22 +2514,14 @@ class BaseLLMHTTPHandler:
)
try:
if stream:
# For streaming, we need to use stream=True in the request
if fake_stream is True:
stream, data = self._prepare_fake_stream_request(
stream=stream,
data=data,
fake_stream=fake_stream,
)
if is_stream_request:
response = await async_httpx_client.post(
url=api_base,
headers=headers,
json=data,
timeout=timeout
or float(response_api_optional_request_params.get("timeout", 0)),
stream=stream,
**body_kwargs,
)
if fake_stream is True:
@ -2518,13 +2548,12 @@ class BaseLLMHTTPHandler:
call_type=CallTypes.responses.value,
)
else:
# For non-streaming, proceed as before
response = await async_httpx_client.post(
url=api_base,
headers=headers,
json=data,
timeout=timeout
or float(response_api_optional_request_params.get("timeout", 0)),
**body_kwargs,
)
except Exception as e:
@ -4005,6 +4034,18 @@ class BaseLLMHTTPHandler:
)
data = BaseResponsesAPIConfig.normalize_responses_api_request_dict(data)
headers, signed_body = responses_api_provider_config.sign_request(
headers=headers,
optional_params=dict(litellm_params),
request_data=data,
api_base=url,
api_key=litellm_params.api_key,
model=model,
)
body_kwargs: Dict[str, Any] = (
{"data": signed_body} if signed_body is not None else {"json": data}
)
## LOGGING
logging_obj.pre_call(
input=input,
@ -4018,7 +4059,7 @@ class BaseLLMHTTPHandler:
try:
response = sync_httpx_client.post(
url=url, headers=headers, json=data, timeout=timeout
url=url, headers=headers, timeout=timeout, **body_kwargs
)
except Exception as e:
@ -4088,6 +4129,18 @@ class BaseLLMHTTPHandler:
)
data = BaseResponsesAPIConfig.normalize_responses_api_request_dict(data)
headers, signed_body = responses_api_provider_config.sign_request(
headers=headers,
optional_params=dict(litellm_params),
request_data=data,
api_base=url,
api_key=litellm_params.api_key,
model=model,
)
body_kwargs: Dict[str, Any] = (
{"data": signed_body} if signed_body is not None else {"json": data}
)
## LOGGING
logging_obj.pre_call(
input=input,
@ -4101,7 +4154,7 @@ class BaseLLMHTTPHandler:
try:
response = await async_httpx_client.post(
url=url, headers=headers, json=data, timeout=timeout
url=url, headers=headers, timeout=timeout, **body_kwargs
)
except Exception as e:

22
litellm/llms/databricks/streaming_utils.py
View file

@ -25,6 +25,28 @@ class ModelResponseIterator:
finish_reason = ""
usage: Optional[ChatCompletionUsageBlock] = None
# Usage-only final chunk (OpenAI ``stream_options.include_usage``)
# arrives with an empty ``choices`` list — return usage without
# indexing ``choices[0]``.
if len(processed_chunk.choices) == 0:
final_usage = getattr(processed_chunk, "usage", None)
return GenericStreamingChunk(
text="",
tool_use=None,
is_finished=False,
finish_reason="",
usage=(
ChatCompletionUsageBlock(
prompt_tokens=final_usage.prompt_tokens or 0,
completion_tokens=final_usage.completion_tokens or 0,
total_tokens=final_usage.total_tokens or 0,
)
if final_usage is not None
else None
),
index=0,
)
if processed_chunk.choices[0].delta.content is not None: # type: ignore
text = processed_chunk.choices[0].delta.content # type: ignore

3
litellm/llms/deepseek/messages/transformation.py
View file

@ -26,6 +26,9 @@ class DeepSeekAnthropicMessagesConfig(AnthropicMessagesConfig):
def custom_llm_provider(self) -> Optional[str]:
return "deepseek"
def should_strip_billing_metadata(self) -> bool:
return True
@staticmethod
def get_api_key(api_key: Optional[str] = None) -> Optional[str]:
return api_key or get_secret_str("DEEPSEEK_API_KEY") or litellm.api_key

6
litellm/llms/fal_ai/image_generation/__init__.py
View file

@ -7,6 +7,7 @@ from .flux_pro_v11_transformation import FalAIFluxProV11Config
from .flux_pro_v11_ultra_transformation import FalAIFluxProV11UltraConfig
from .flux_schnell_transformation import FalAIFluxSchnellConfig
from .imagen4_transformation import FalAIImagen4Config
from .nano_banana_transformation import FalAINanoBananaConfig
from .recraft_v3_transformation import FalAIRecraftV3Config
from .ideogram_v3_transformation import FalAIIdeogramV3Config
from .stable_diffusion_transformation import FalAIStableDiffusionConfig
@ -20,6 +21,7 @@ __all__ = [
"FalAIBaseConfig",
"FalAIImageGenerationConfig",
"FalAIImagen4Config",
"FalAINanoBananaConfig",
"FalAIRecraftV3Config",
"FalAIBriaConfig",
"FalAIFluxProV11Config",
@ -45,7 +47,9 @@ def get_fal_ai_image_generation_config(model: str) -> BaseImageGenerationConfig:
model_lower = model.lower()
# Map model names to their corresponding configuration classes
if "imagen4" in model_lower or "imagen-4" in model_lower:
if "nano-banana" in model_lower or "gemini-25-flash-image" in model_lower:
return FalAINanoBananaConfig()
elif "imagen4" in model_lower or "imagen-4" in model_lower:
return FalAIImagen4Config()
elif "recraft" in model_lower:
return FalAIRecraftV3Config()

105
litellm/llms/fal_ai/image_generation/nano_banana_transformation.py Normal file
View file

@ -0,0 +1,105 @@
from typing import List, Optional
from litellm.secret_managers.main import get_secret_str
from litellm.types.llms.openai import OpenAIImageGenerationOptionalParams
from .transformation import FalAIBaseConfig
class FalAINanoBananaConfig(FalAIBaseConfig):
"""
Configuration for Fal AI's Nano Banana / Gemini 2.5 Flash Image models.
Serves the imagen4 deprecation migration path. The same underlying model is
exposed under two endpoints that share an identical schema:
- fal-ai/nano-banana
- fal-ai/gemini-25-flash-image
Documentation: https://fal.ai/models/fal-ai/nano-banana
"""
SUPPORTED_ASPECT_RATIOS: List[str] = [
"21:9",
"16:9",
"3:2",
"4:3",
"5:4",
"1:1",
"4:5",
"3:4",
"2:3",
"9:16",
]
def get_complete_url(
self,
api_base: Optional[str],
api_key: Optional[str],
model: str,
optional_params: dict,
litellm_params: dict,
stream: Optional[bool] = None,
) -> str:
base_url: str = (
api_base or get_secret_str("FAL_AI_API_BASE") or self.DEFAULT_BASE_URL
).rstrip("/")
endpoint = model if model.startswith("fal-ai/") else f"fal-ai/{model}"
return f"{base_url}/{endpoint}"
def get_supported_openai_params(
self, model: str
) -> List[OpenAIImageGenerationOptionalParams]:
return ["n", "response_format", "size"]
def map_openai_params(
self,
non_default_params: dict,
optional_params: dict,
model: str,
drop_params: bool,
) -> dict:
supported_params = self.get_supported_openai_params(model)
for key, value in non_default_params.items():
if key == "response_format":
continue
elif key == "n":
if "num_images" not in optional_params:
optional_params["num_images"] = value
elif key == "size":
if "aspect_ratio" not in optional_params:
optional_params["aspect_ratio"] = self._map_aspect_ratio(value)
elif key not in optional_params and not drop_params:
raise ValueError(
f"Parameter {key} is not supported for model {model}. "
f"Supported parameters are {supported_params}. "
"Set drop_params=True to drop unsupported parameters."
)
return optional_params
def _map_aspect_ratio(self, size: str) -> str:
if not isinstance(size, str) or "x" not in size:
return "1:1"
try:
width, height = (int(part) for part in size.split("x"))
target = width / height
except (ValueError, ZeroDivisionError):
return "1:1"
def ratio_of(aspect_ratio: str) -> float:
w, h = (int(part) for part in aspect_ratio.split(":"))
return w / h
return min(
self.SUPPORTED_ASPECT_RATIOS,
key=lambda aspect_ratio: abs(ratio_of(aspect_ratio) - target),
)
def transform_image_generation_request(
self,
model: str,
prompt: str,
optional_params: dict,
litellm_params: dict,
headers: dict,
) -> dict:
return {"prompt": prompt, **optional_params}

45
litellm/llms/github_copilot/responses/transformation.py
View file

@ -2,7 +2,7 @@
GitHub Copilot Responses API Configuration.
This module provides the configuration for GitHub Copilot's Responses API,
which is required for models like gpt-5.1-codex that only support the /responses endpoint.
which is required for models like gpt-5.3-codex that only support the /responses endpoint.
Implementation based on analysis of the copilot-api project by caozhiyuan:
https://github.com/caozhiyuan/copilot-api
@ -12,6 +12,7 @@ from typing import TYPE_CHECKING, Any, Dict, Optional, Union
import os
import litellm
from litellm._logging import verbose_logger
from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH
from litellm.exceptions import AuthenticationError
@ -22,6 +23,7 @@ from litellm.types.llms.openai import (
)
from litellm.types.router import GenericLiteLLMParams
from litellm.types.utils import LlmProviders
from litellm.utils import _cached_get_model_info_helper
from ..authenticator import Authenticator
from ..common_utils import (
@ -38,6 +40,47 @@ else:
LiteLLMLoggingObj = Any
def github_copilot_supports_responses_api(model: str) -> bool:
"""
Gate native /v1/responses dispatch per github_copilot model.
Resolution (first match wins): mode "responses" -> True; mode "chat" ->
False (opt-out wins for dual-endpoint models); "/v1/responses" in
supported_endpoints -> True; else False. Unknown model -> False (the bridge
always works since every Copilot model supports /chat/completions).
Reads merged model info (per-deployment model_info applied via the router's
register_model, which also clears the cache used here).
"""
try:
info = _cached_get_model_info_helper(
model=model, custom_llm_provider="github_copilot"
)
except Exception as e:
verbose_logger.debug(
"github_copilot_supports_responses_api: get_model_info failed "
"for %s: %s",
model,
e,
)
return False
mode = info.get("mode")
if mode == "responses":
return True
if mode == "chat":
return False
# supported_endpoints is dropped by ModelInfoBase; read it from the raw
# model_cost entry via the resolved key.
key = info.get("key")
raw_info = litellm.model_cost.get(key) if isinstance(key, str) else None
endpoints = (
raw_info.get("supported_endpoints") if isinstance(raw_info, dict) else None
)
return isinstance(endpoints, list) and "/v1/responses" in endpoints
class GithubCopilotResponsesAPIConfig(OpenAIResponsesAPIConfig):
"""
Configuration for GitHub Copilot's Responses API.

16
litellm/llms/litellm_proxy/skills/README.md
View file

@ -18,7 +18,7 @@ flowchart TB
F[Request with container.skills] --> G[SkillsInjectionHook]
G --> H{skill_id prefix?}
H -->|"litellm:skill_abc"| I[Fetch from LiteLLM DB]
H -->|"litellm_skill_abc"| I[Fetch from LiteLLM DB]
H -->|"skill_xyz" no prefix| J[Pass to Anthropic as native skill]
I --> K{Model provider?}
@ -57,7 +57,7 @@ sequenceDiagram
Note over LiteLLM,PreHook: PRE-CALL HOOK
LiteLLM->>PreHook: Intercept request
PreHook->>PreHook: Fetch skill from DB (litellm:skill_id)
PreHook->>PreHook: Fetch skill from DB (litellm_skill_id)
PreHook->>PreHook: Extract SKILL.md from ZIP
PreHook->>PreHook: Inject SKILL.md into system prompt
PreHook->>PreHook: Add litellm_code_execution tool
@ -105,7 +105,7 @@ response = await litellm.acompletion(
model="gpt-4o-mini",
messages=[{"role": "user", "content": "Create a bouncing ball GIF"}],
container={
"skills": [{"type": "custom", "skill_id": "litellm:skill_abc123"}]
"skills": [{"type": "custom", "skill_id": "litellm_skill_abc123"}]
},
)
@ -261,7 +261,7 @@ response = litellm.completion(
messages=[{"role": "user", "content": "Analyze this data..."}],
container={
"skills": [
{"type": "custom", "skill_id": "litellm:skill_abc123"} # litellm: prefix
{"type": "custom", "skill_id": "litellm_skill_abc123"} # litellm_skill_ prefix
]
}
)
@ -277,7 +277,7 @@ response = litellm.completion(
"messages": [{"role": "user", "content": "Help me analyze data"}],
"container": {
"skills": [
{"type": "custom", "skill_id": "litellm:skill_abc123"}
{"type": "custom", "skill_id": "litellm_skill_abc123"}
]
}
}
@ -287,7 +287,7 @@ response = litellm.completion(
The hook (`litellm/proxy/hooks/litellm_skills/main.py`) intercepts the request:
1. **Detects `litellm:` prefix** → Fetches skill from database
1. **Detects `litellm_skill_` prefix** → Fetches skill from database
2. **Checks model provider** → Bedrock is not Anthropic
3. **Extracts SKILL.md** from stored ZIP file
4. **Converts skill to tool** + **Injects content into system prompt**
@ -361,8 +361,8 @@ model LiteLLM_SkillsTable {
| Create skill on Anthropic | `anthropic` | N/A | Forward to Anthropic API |
| Create skill in LiteLLM DB | `litellm_proxy` | N/A | Store in database |
| Use Anthropic native skill | N/A | `skill_xyz` | Pass to Anthropic container.skills |
| Use LiteLLM skill on Anthropic | N/A | `litellm:skill_abc` | Convert to tools |
| Use LiteLLM skill on Bedrock/OpenAI | N/A | `litellm:skill_abc` | Convert to tools + inject SKILL.md |
| Use LiteLLM skill on Anthropic | N/A | `litellm_skill_abc` | Convert to tools |
| Use LiteLLM skill on Bedrock/OpenAI | N/A | `litellm_skill_abc` | Convert to tools + inject SKILL.md |
## Testing

4
litellm/llms/litellm_proxy/skills/constants.py
View file

@ -4,6 +4,10 @@ Constants for LiteLLM Skills
Centralized constants for skills processing, code execution, and sandbox configuration.
"""
LITELLM_SKILL_ID_PREFIX: str = "litellm_skill_"
"""Prefix for DB-backed skill IDs. The model-facing tool name is the skill ID
with hyphens/spaces replaced by underscores, which leaves this prefix intact."""
# Code execution loop settings
DEFAULT_MAX_ITERATIONS: int = 10
"""Maximum number of iterations for the automatic code execution loop."""

12
litellm/llms/litellm_proxy/skills/handler.py
View file

@ -10,6 +10,7 @@ from typing import Any, Dict, List, Optional
from litellm._logging import verbose_logger
from litellm.caching.in_memory_cache import InMemoryCache
from litellm.llms.litellm_proxy.skills.constants import LITELLM_SKILL_ID_PREFIX
from litellm.proxy._types import LiteLLM_SkillsTable, NewSkillRequest, UserAPIKeyAuth
from litellm.proxy.common_utils.resource_ownership import (
get_primary_resource_owner_scope,
@ -17,6 +18,7 @@ from litellm.proxy.common_utils.resource_ownership import (
is_proxy_admin,
user_can_access_resource_owner,
)
from litellm.repositories.table_repositories import SkillsRepository
# Skills are looked up on every chat completion that has skills enabled
# (`SkillsInjectionHook` calls ``fetch_skill_from_db``). 60s LRU/TTL cache
@ -67,7 +69,7 @@ class LiteLLMSkillsHandler:
) -> LiteLLM_SkillsTable:
prisma_client = await LiteLLMSkillsHandler._get_prisma_client()
skill_id = f"litellm_skill_{uuid.uuid4()}"
skill_id = f"{LITELLM_SKILL_ID_PREFIX}{uuid.uuid4()}"
owner = get_primary_resource_owner_scope(user_api_key_dict) or user_id
if owner is None:
# Identity-less callers (no user_id / team_id / org_id /
@ -107,7 +109,7 @@ class LiteLLMSkillsHandler:
f"LiteLLMSkillsHandler: Creating skill {skill_id} with title={data.display_title}"
)
new_skill = await prisma_client.db.litellm_skillstable.create(data=skill_data)
new_skill = await SkillsRepository(prisma_client).table.create(data=skill_data)
return _prisma_skill_to_litellm(new_skill)
@staticmethod
@ -133,7 +135,7 @@ class LiteLLMSkillsHandler:
return []
find_many_kwargs["where"] = {"created_by": {"in": owner_scopes}}
skills = await prisma_client.db.litellm_skillstable.find_many(
skills = await SkillsRepository(prisma_client).table.find_many(
**find_many_kwargs
)
return [_prisma_skill_to_litellm(s) for s in skills]
@ -150,7 +152,7 @@ class LiteLLMSkillsHandler:
return cached
prisma_client = await LiteLLMSkillsHandler._get_prisma_client()
skill = await prisma_client.db.litellm_skillstable.find_unique(
skill = await SkillsRepository(prisma_client).table.find_unique(
where={"skill_id": skill_id}
)
_SKILL_CACHE.set_cache(
@ -189,7 +191,7 @@ class LiteLLMSkillsHandler:
):
raise ValueError(f"Skill not found: {skill_id}")
await prisma_client.db.litellm_skillstable.delete(where={"skill_id": skill_id})
await SkillsRepository(prisma_client).table.delete(where={"skill_id": skill_id})
_SKILL_CACHE.set_cache(skill_id, _NEGATIVE_SKILL_SENTINEL)
return {"id": skill_id, "type": "skill_deleted"}

3
litellm/llms/minimax/messages/transformation.py
View file

@ -28,6 +28,9 @@ class MinimaxMessagesConfig(AnthropicMessagesConfig):
def custom_llm_provider(self) -> Optional[str]:
return "minimax"
def should_strip_billing_metadata(self) -> bool:
return True
@staticmethod
def get_api_key(api_key: Optional[str] = None) -> Optional[str]:
"""

2
litellm/llms/openai/completion/handler.py
View file

@ -49,6 +49,8 @@ class OpenAITextCompletion(BaseLLM):
headers: Optional[dict] = None,
):
try:
if headers:
optional_params = {**optional_params, "extra_headers": headers}
if headers is None:
headers = self.validate_environment(api_key=api_key)
if model is None or messages is None:

19
litellm/llms/openai_like/dynamic_config.py
View file

@ -187,6 +187,7 @@ def create_responses_config_class(provider: SimpleProviderConfig):
from litellm.llms.openai_like.responses.transformation import (
OpenAILikeResponsesConfig,
)
from litellm.types.llms.openai import ResponseInputParam
from litellm.types.router import GenericLiteLLMParams
class JSONProviderResponsesConfig(OpenAILikeResponsesConfig):
@ -223,5 +224,23 @@ def create_responses_config_class(provider: SimpleProviderConfig):
api_base = api_base.rstrip("/")
return f"{api_base}/responses"
def transform_responses_api_request(
self,
model: str,
input: Union[str, ResponseInputParam],
response_api_optional_request_params: dict,
litellm_params: GenericLiteLLMParams,
headers: dict,
) -> dict:
if provider.special_handling.get("force_store_false"):
response_api_optional_request_params["store"] = False
return super().transform_responses_api_request(
model=model,
input=input,
response_api_optional_request_params=response_api_optional_request_params,
litellm_params=litellm_params,
headers=headers,
)
_responses_config_cache[provider.slug] = JSONProviderResponsesConfig
return JSONProviderResponsesConfig

9
litellm/llms/openai_like/providers.json
View file

@ -132,5 +132,14 @@
"param_mappings": {
"max_completion_tokens": "max_tokens"
}
},
"parasail": {
"base_url": "https://api.parasail.io/v1",
"api_key_env": "PARASAIL_API_KEY",
"api_base_env": "PARASAIL_API_BASE",
"supported_endpoints": ["/v1/chat/completions", "/v1/responses"],
"special_handling": {
"force_store_false": true
}
}
}

47
litellm/llms/vertex_ai/common_utils.py
View file

@ -12,7 +12,11 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import unpack_defs
from litellm.llms.base_llm.base_utils import BaseLLMModelInfo, BaseTokenCounter
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.types.llms.openai import AllMessageValues
from litellm.types.llms.vertex_ai import PartType, Schema
from litellm.types.llms.vertex_ai import (
VERTEX_AI_PROVIDER_METADATA_FIELDS,
PartType,
Schema,
)
from litellm.types.utils import TokenCountResponse
from litellm.utils import supports_response_schema, supports_system_messages
@ -27,6 +31,47 @@ class VertexAIError(BaseLLMException):
super().__init__(message=message, status_code=status_code, headers=headers)
def redact_vertex_ai_metadata_from_logged_object(obj: Any) -> None:
if isinstance(obj, dict):
for field in VERTEX_AI_PROVIDER_METADATA_FIELDS:
if field in obj:
obj[field] = []
hidden_params = obj.get("_hidden_params")
if isinstance(hidden_params, dict):
for field in VERTEX_AI_PROVIDER_METADATA_FIELDS:
hidden_params.pop(field, None)
return
for field in VERTEX_AI_PROVIDER_METADATA_FIELDS:
if hasattr(obj, field):
setattr(obj, field, [])
hidden_params = getattr(obj, "_hidden_params", None)
if isinstance(hidden_params, dict):
for field in VERTEX_AI_PROVIDER_METADATA_FIELDS:
hidden_params.pop(field, None)
def redact_vertex_ai_metadata_from_litellm_params(model_call_details: dict) -> None:
"""
success_handler() merges response._hidden_params into
litellm_params.metadata['hidden_params'] before redaction runs, so the Vertex
metadata must be scrubbed from that copy too.
"""
litellm_params = model_call_details.get("litellm_params")
if not isinstance(litellm_params, dict):
return
for metadata_key in ("metadata", "litellm_metadata"):
metadata = litellm_params.get(metadata_key)
if not isinstance(metadata, dict):
continue
hidden_params = metadata.get("hidden_params")
if not isinstance(hidden_params, dict):
continue
for field in VERTEX_AI_PROVIDER_METADATA_FIELDS:
hidden_params.pop(field, None)
def vertex_request_labels_from_litellm_params(
litellm_params: Optional[dict],
) -> Optional[Dict[str, str]]:

14
litellm/llms/vertex_ai/context_caching/vertex_ai_context_caching.py
View file

@ -19,7 +19,7 @@ from litellm.types.llms.vertex_ai import (
VertexAICachedContentResponseObject,
)
from ..common_utils import VertexAIError
from ..common_utils import VertexAIError, get_vertex_base_url
from ..vertex_llm_base import VertexBase
from .transformation import (
separate_cached_messages,
@ -69,17 +69,13 @@ class ContextCachingEndpoints(VertexBase):
elif custom_llm_provider == "vertex_ai":
auth_header = vertex_auth_header
endpoint = "cachedContents"
if vertex_location == "global":
url = f"https://aiplatform.googleapis.com/v1/projects/{vertex_project}/locations/{vertex_location}/{endpoint}"
else:
url = f"https://{vertex_location}-aiplatform.googleapis.com/v1/projects/{vertex_project}/locations/{vertex_location}/{endpoint}"
base_url = get_vertex_base_url(vertex_location)
url = f"{base_url}/v1/projects/{vertex_project}/locations/{vertex_location}/{endpoint}"
else:
auth_header = vertex_auth_header
endpoint = "cachedContents"
if vertex_location == "global":
url = f"https://aiplatform.googleapis.com/v1beta1/projects/{vertex_project}/locations/{vertex_location}/{endpoint}"
else:
url = f"https://{vertex_location}-aiplatform.googleapis.com/v1beta1/projects/{vertex_project}/locations/{vertex_location}/{endpoint}"
base_url = get_vertex_base_url(vertex_location)
url = f"{base_url}/v1beta1/projects/{vertex_project}/locations/{vertex_location}/{endpoint}"
return self._check_custom_proxy(
api_base=api_base,

84
litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py
View file

@ -63,6 +63,7 @@ from litellm.types.llms.openai import (
OpenAIChatCompletionFinishReason,
)
from litellm.types.llms.vertex_ai import (
VERTEX_AI_PROVIDER_METADATA_FIELDS,
VERTEX_CREDENTIALS_TYPES,
Candidates,
ContentType,
@ -1111,6 +1112,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
{
"voice": "alloy",
"format": "mp3",
"language_code": "en-US",
}
Expected output:
@ -1119,7 +1121,8 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
prebuiltVoiceConfig: {
voiceName: "alloy",
}
}
},
languageCode: "en-US",
}
"""
from litellm.types.llms.vertex_ai import (
@ -1145,6 +1148,9 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
voice_config: VoiceConfig = {"prebuiltVoiceConfig": prebuilt_voice_config}
speech_config["voiceConfig"] = voice_config
if "language_code" in value:
speech_config["languageCode"] = value["language_code"]
return cast(dict, speech_config)
@staticmethod
@ -2253,6 +2259,71 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
citation_metadata,
)
@staticmethod
def _get_stream_chunk_attr(chunk: Any, field_name: str) -> Any:
if isinstance(chunk, dict):
value = chunk.get(field_name)
if value is not None:
return value
model_extra = chunk.get("model_extra")
if isinstance(model_extra, dict):
value = model_extra.get(field_name)
if value is not None:
return value
hidden_params = chunk.get("_hidden_params")
if isinstance(hidden_params, dict):
return hidden_params.get(field_name)
return None
return getattr(chunk, field_name, None)
@staticmethod
def _set_stream_metadata_on_response(
model_response: Any,
grounding_metadata: List[dict],
url_context_metadata: List[dict],
safety_ratings: List[dict],
citation_metadata: List[dict],
) -> None:
setattr(model_response, "vertex_ai_grounding_metadata", grounding_metadata) # type: ignore
if grounding_metadata:
model_response._hidden_params["vertex_ai_grounding_metadata"] = (
grounding_metadata
)
setattr(model_response, "vertex_ai_url_context_metadata", url_context_metadata) # type: ignore
if url_context_metadata:
model_response._hidden_params["vertex_ai_url_context_metadata"] = (
url_context_metadata
)
setattr(model_response, "vertex_ai_safety_ratings", safety_ratings) # type: ignore
setattr(model_response, "vertex_ai_safety_results", safety_ratings) # type: ignore
if safety_ratings:
model_response._hidden_params["vertex_ai_safety_ratings"] = safety_ratings
model_response._hidden_params["vertex_ai_safety_results"] = safety_ratings
setattr(model_response, "vertex_ai_citation_metadata", citation_metadata) # type: ignore
if citation_metadata:
model_response._hidden_params["vertex_ai_citation_metadata"] = (
citation_metadata
)
def apply_assembled_streaming_response_metadata(
self,
response: ModelResponse,
chunks: List[Any],
) -> None:
for field_name in VERTEX_AI_PROVIDER_METADATA_FIELDS:
merged: List[Any] = []
for chunk in chunks:
value = VertexGeminiConfig._get_stream_chunk_attr(chunk, field_name)
if not value:
continue
if isinstance(value, list):
merged.extend(value)
else:
merged.append(value)
if merged:
setattr(response, field_name, merged)
response._hidden_params[field_name] = merged
@staticmethod
def _convert_grounding_metadata_to_annotations(
grounding_metadata: List[dict],
@ -3385,10 +3456,13 @@ class ModelResponseIterator:
if choice.finish_reason == "stop":
choice.finish_reason = "tool_calls"
setattr(model_response, "vertex_ai_grounding_metadata", grounding_metadata) # type: ignore
setattr(model_response, "vertex_ai_url_context_metadata", url_context_metadata) # type: ignore
setattr(model_response, "vertex_ai_safety_ratings", safety_ratings) # type: ignore
setattr(model_response, "vertex_ai_citation_metadata", citation_metadata) # type: ignore
VertexGeminiConfig._set_stream_metadata_on_response(
model_response,
grounding_metadata,
url_context_metadata,
safety_ratings,
citation_metadata,
)
return (
grounding_metadata,

3
litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/transformation.py
View file

@ -17,6 +17,9 @@ from ..output_params_utils import sanitize_vertex_anthropic_output_params
class VertexAIPartnerModelsAnthropicMessagesConfig(AnthropicMessagesConfig, VertexBase):
def should_strip_billing_metadata(self) -> bool:
return True
def validate_anthropic_messages_environment(
self,
headers: dict,

3
litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py
View file

@ -52,6 +52,9 @@ class VertexAIAnthropicConfig(AnthropicConfig):
def custom_llm_provider(self) -> Optional[str]:
return "vertex_ai"
def should_strip_billing_metadata(self) -> bool:
return True
def _add_context_management_beta_headers(
self, beta_set: set, context_management: dict
) -> None:

67
litellm/llms/xai/chat/transformation.py
View file

@ -5,6 +5,7 @@ import httpx
import litellm
from litellm._logging import verbose_logger
from litellm.constants import XAI_API_BASE
from litellm.exceptions import AuthenticationError
from litellm.litellm_core_utils.prompt_templates.common_utils import (
filter_value_from_dict,
strip_name_from_messages,
@ -39,6 +40,72 @@ class XAIChatConfig(OpenAIGPTConfig):
dynamic_api_key = XAIModelInfo.get_api_key(api_key)
return api_base, dynamic_api_key
def validate_environment(
self,
headers: dict,
model: str,
messages: List[AllMessageValues],
optional_params: dict,
litellm_params: dict,
api_key: Optional[str] = None,
api_base: Optional[str] = None,
) -> dict:
from litellm.llms.xai.oauth import (
XAIOAuthAuthenticator,
XAIOAuthError,
should_use_xai_oauth,
)
dynamic_api_key = XAIModelInfo.get_api_key(api_key)
if should_use_xai_oauth(litellm_params) and not dynamic_api_key:
try:
headers["Authorization"] = (
f"Bearer {XAIOAuthAuthenticator().get_access_token()}"
)
except XAIOAuthError as exc:
raise AuthenticationError(
model=model,
llm_provider=self.custom_llm_provider or "xai",
message=str(exc),
) from exc
if "content-type" not in headers and "Content-Type" not in headers:
headers["Content-Type"] = "application/json"
return headers
return super().validate_environment(
headers=headers,
model=model,
messages=messages,
optional_params=optional_params,
litellm_params=litellm_params,
api_key=dynamic_api_key,
api_base=api_base,
)
def get_complete_url(
self,
api_base: Optional[str],
api_key: Optional[str],
model: str,
optional_params: dict,
litellm_params: dict,
stream: Optional[bool] = None,
) -> str:
from litellm.llms.xai.oauth import XAIOAuthAuthenticator, should_use_xai_oauth
dynamic_api_key = XAIModelInfo.get_api_key(api_key)
if should_use_xai_oauth(litellm_params) and not dynamic_api_key:
api_base = XAIOAuthAuthenticator().get_api_base()
return super().get_complete_url(
api_base=api_base,
api_key=dynamic_api_key,
model=model,
optional_params=optional_params,
litellm_params=litellm_params,
stream=stream,
)
def get_supported_openai_params(self, model: str) -> list:
base_openai_params = [
"logit_bias",

421
litellm/llms/xai/oauth.py Normal file
View file

@ -0,0 +1,421 @@
import base64
import hashlib
import json
import os
import secrets
import sys
import threading
import time
import uuid
import webbrowser
from http.server import BaseHTTPRequestHandler, HTTPServer
from typing import Any, Dict, Optional, Tuple, Union
from urllib.parse import parse_qs, urlencode, urlparse
import httpx
from litellm._logging import verbose_logger
from litellm.constants import XAI_API_BASE
from litellm.llms.custom_httpx.http_handler import HTTPHandler, _get_httpx_client
from litellm.secret_managers.main import get_secret_str
XAI_OAUTH_ISSUER = "https://auth.x.ai"
XAI_OAUTH_DISCOVERY_URL = f"{XAI_OAUTH_ISSUER}/.well-known/openid-configuration"
XAI_OAUTH_CLIENT_ID = "b1a00492-073a-47ea-816f-4c329264a828"
XAI_OAUTH_SCOPE = "openid profile email offline_access grok-cli:access api:access"
XAI_OAUTH_REDIRECT_HOST = "127.0.0.1"
XAI_OAUTH_REDIRECT_PORT = 56121
XAI_OAUTH_REDIRECT_PATH = "/callback"
XAI_OAUTH_EXPIRY_SKEW_SECONDS = 120
XAI_OAUTH_CALLBACK_TIMEOUT_SECONDS = 180
_XAI_OAUTH_REFRESH_LOCK = threading.Lock()
class XAIOAuthError(Exception):
pass
class XAIOAuthLoginRequiredError(XAIOAuthError):
pass
class _CallbackHandler(BaseHTTPRequestHandler):
server: "_CallbackServer"
def do_GET(self) -> None:
parsed = urlparse(self.path)
if parsed.path != XAI_OAUTH_REDIRECT_PATH:
self.send_response(404)
self.end_headers()
return
params = parse_qs(parsed.query)
result = {
"code": params.get("code", [None])[0],
"state": params.get("state", [None])[0],
"error": params.get("error", [None])[0],
"error_description": params.get("error_description", [None])[0],
}
self.server.callback_result = result
if result["state"] != self.server.expected_state:
self.send_response(400)
self.send_header("Content-Type", "text/html; charset=utf-8")
self.end_headers()
self.wfile.write(
b"<html><body><h1>xAI authorization state mismatch.</h1></body></html>"
)
return
self.send_response(200)
self.send_header("Content-Type", "text/html; charset=utf-8")
self.end_headers()
body = (
b"<html><body><h1>xAI authorization failed.</h1>You can close this tab.</body></html>"
if result["error"]
else b"<html><body><h1>xAI authorization received.</h1>You can close this tab.</body></html>"
)
self.wfile.write(body)
def log_message(self, format: str, *args: Any) -> None:
return
class _CallbackServer(HTTPServer):
expected_state: str
callback_result: Optional[Dict[str, Optional[str]]]
class XAIOAuthAuthenticator:
def __init__(
self, http_client: Optional[Union[httpx.Client, HTTPHandler]] = None
) -> None:
self.token_dir = get_secret_str("XAI_OAUTH_TOKEN_DIR") or os.path.expanduser(
"~/.config/litellm/xai_oauth"
)
self.auth_file = os.path.join(
self.token_dir, get_secret_str("XAI_OAUTH_AUTH_FILE") or "auth.json"
)
self.http_client = http_client
def get_api_base(self) -> str:
return (
get_secret_str("XAI_OAUTH_API_BASE")
or get_secret_str("XAI_API_BASE")
or XAI_API_BASE
)
def get_access_token(self) -> str:
auth_data = self._read_auth_file()
if not auth_data:
raise XAIOAuthLoginRequiredError(
"xAI OAuth login required. Run `litellm xai-oauth login`."
)
access_token = auth_data.get("access_token")
if access_token and not self._is_expired(auth_data):
return access_token
refresh_token = auth_data.get("refresh_token")
if not refresh_token:
raise XAIOAuthLoginRequiredError(
"xAI OAuth refresh token missing. Run `litellm xai-oauth login`."
)
with _XAI_OAUTH_REFRESH_LOCK:
locked_auth_data = self._read_auth_file() or auth_data
access_token = locked_auth_data.get("access_token")
if access_token and not self._is_expired(locked_auth_data):
return access_token
refreshed = self._refresh_tokens(locked_auth_data)
return refreshed["access_token"]
def login(self, force: bool = False, no_browser: bool = False) -> Dict[str, Any]:
existing = self._read_auth_file()
if existing and not force and existing.get("access_token"):
if not self._is_expired(existing):
return existing
if existing.get("refresh_token"):
try:
return self._refresh_tokens(existing)
except XAIOAuthError:
pass
discovery = self._discover()
verifier, challenge = self._pkce_pair()
state = uuid.uuid4().hex
nonce = uuid.uuid4().hex
server, redirect_uri = self._start_callback_server(state)
authorize_url = self._build_authorize_url(
authorization_endpoint=discovery["authorization_endpoint"],
redirect_uri=redirect_uri,
challenge=challenge,
state=state,
nonce=nonce,
)
if no_browser or not webbrowser.open(authorize_url):
sys.stdout.write(
f"Open this URL to authenticate with xAI:\n{authorize_url}\n"
)
sys.stdout.flush()
result = self._wait_for_callback(server)
if result.get("state") != state:
raise XAIOAuthError("xAI OAuth state mismatch")
if result.get("error"):
description = result.get("error_description") or result["error"]
raise XAIOAuthError(f"xAI authorization failed: {description}")
code = result.get("code")
if not code:
raise XAIOAuthError("xAI authorization failed: no code returned")
token_payload = self._exchange_token(
discovery["token_endpoint"],
{
"grant_type": "authorization_code",
"code": code,
"redirect_uri": redirect_uri,
"client_id": XAI_OAUTH_CLIENT_ID,
"code_verifier": verifier,
},
)
auth_data = self._build_auth_record(token_payload, discovery["token_endpoint"])
self._write_auth_file(auth_data)
return auth_data
def _client(self) -> Union[httpx.Client, HTTPHandler]:
return self.http_client or _get_httpx_client()
def _ensure_token_dir(self) -> None:
os.makedirs(self.token_dir, mode=0o700, exist_ok=True)
try:
os.chmod(self.token_dir, 0o700)
except OSError:
verbose_logger.debug("Could not chmod xAI OAuth token directory")
def _read_auth_file(self) -> Optional[Dict[str, Any]]:
try:
with open(self.auth_file, "r") as f:
data = json.load(f)
return data if isinstance(data, dict) else None
except (IOError, json.JSONDecodeError):
return None
def _write_auth_file(self, data: Dict[str, Any]) -> None:
self._ensure_token_dir()
tmp_file = os.path.join(
self.token_dir,
f".{os.path.basename(self.auth_file)}.{uuid.uuid4().hex}.tmp",
)
flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL
if hasattr(os, "O_NOFOLLOW"):
flags |= os.O_NOFOLLOW
fd = os.open(tmp_file, flags, 0o600)
try:
with os.fdopen(fd, "w") as f:
json.dump(data, f)
f.flush()
os.fsync(f.fileno())
os.replace(tmp_file, self.auth_file)
try:
os.chmod(self.auth_file, 0o600)
except OSError:
verbose_logger.debug("Could not chmod xAI OAuth auth file")
except Exception:
try:
os.close(fd)
except OSError:
pass
try:
os.unlink(tmp_file)
except OSError:
pass
raise
def _is_expired(self, auth_data: Dict[str, Any]) -> bool:
expires_at = auth_data.get("expires_at")
if expires_at is None:
return True
try:
return time.time() >= float(expires_at) - XAI_OAUTH_EXPIRY_SKEW_SECONDS
except (TypeError, ValueError):
return True
def _discover(self) -> Dict[str, str]:
try:
response = self._client().get(
XAI_OAUTH_DISCOVERY_URL, headers={"Accept": "application/json"}
)
response.raise_for_status()
except httpx.HTTPStatusError as exc:
raise XAIOAuthError(
f"xAI OAuth discovery request failed: {exc.response.status_code} {exc.response.text}"
) from exc
try:
data = response.json()
except ValueError as exc:
raise XAIOAuthError(
"xAI OAuth discovery response was not valid JSON"
) from exc
authorization_endpoint = data.get("authorization_endpoint")
token_endpoint = data.get("token_endpoint")
if not authorization_endpoint or not token_endpoint:
raise XAIOAuthError("xAI OAuth discovery missing endpoints")
return {
"authorization_endpoint": self._validate_xai_endpoint(
authorization_endpoint
),
"token_endpoint": self._validate_xai_endpoint(token_endpoint),
}
def _validate_xai_endpoint(self, url: str) -> str:
parsed = urlparse(url)
host = (parsed.hostname or "").lower()
if parsed.scheme != "https" or (host != "x.ai" and not host.endswith(".x.ai")):
raise XAIOAuthError(
f"xAI OAuth discovery returned unexpected endpoint: {url}"
)
return url
def _pkce_pair(self) -> Tuple[str, str]:
verifier = (
base64.urlsafe_b64encode(secrets.token_bytes(32)).rstrip(b"=").decode()
)
challenge = (
base64.urlsafe_b64encode(hashlib.sha256(verifier.encode()).digest())
.rstrip(b"=")
.decode()
)
return verifier, challenge
def _start_callback_server(self, state: str) -> Tuple[_CallbackServer, str]:
last_error: Optional[OSError] = None
for port in (XAI_OAUTH_REDIRECT_PORT, 0):
try:
server = _CallbackServer(
(XAI_OAUTH_REDIRECT_HOST, port), _CallbackHandler
)
server.expected_state = state
server.callback_result = None
actual_port = server.server_address[1]
redirect_uri = f"http://{XAI_OAUTH_REDIRECT_HOST}:{actual_port}{XAI_OAUTH_REDIRECT_PATH}"
return server, redirect_uri
except OSError as exc:
last_error = exc
raise XAIOAuthError(f"Could not start xAI OAuth callback server: {last_error}")
def _build_authorize_url(
self,
authorization_endpoint: str,
redirect_uri: str,
challenge: str,
state: str,
nonce: str,
) -> str:
params = {
"response_type": "code",
"client_id": XAI_OAUTH_CLIENT_ID,
"redirect_uri": redirect_uri,
"scope": XAI_OAUTH_SCOPE,
"code_challenge": challenge,
"code_challenge_method": "S256",
"state": state,
"nonce": nonce,
}
return f"{authorization_endpoint}?{urlencode(params)}"
def _wait_for_callback(self, server: _CallbackServer) -> Dict[str, Optional[str]]:
server.timeout = 1
deadline = time.time() + XAI_OAUTH_CALLBACK_TIMEOUT_SECONDS
try:
while time.time() < deadline:
server.handle_request()
if server.callback_result is not None:
return server.callback_result
finally:
server.server_close()
raise XAIOAuthError("Timed out waiting for xAI OAuth callback")
def _exchange_token(
self, token_endpoint: str, data: Dict[str, str]
) -> Dict[str, Any]:
try:
response = self._client().post(
token_endpoint,
headers={
"Accept": "application/json",
"Content-Type": "application/x-www-form-urlencoded",
},
data=data,
)
response.raise_for_status()
except httpx.HTTPStatusError as exc:
raise XAIOAuthError(
f"xAI OAuth token request failed: {exc.response.status_code} {exc.response.text}"
) from exc
try:
body = response.json()
except ValueError as exc:
raise XAIOAuthError("xAI OAuth token response was not valid JSON") from exc
if not isinstance(body, dict):
raise XAIOAuthError("xAI OAuth token response was not an object")
return body
def _build_auth_record(
self,
token_payload: Dict[str, Any],
token_endpoint: str,
fallback_refresh_token: Optional[str] = None,
) -> Dict[str, Any]:
access_token = token_payload.get("access_token")
refresh_token = token_payload.get("refresh_token") or fallback_refresh_token
if not access_token:
raise XAIOAuthError("xAI OAuth token response missing access_token")
if not refresh_token:
raise XAIOAuthError("xAI OAuth token response missing refresh_token")
expires_in = token_payload.get("expires_in") or 3600
try:
expires_at = int(time.time() + int(expires_in))
except (TypeError, ValueError):
expires_at = int(time.time() + 3600)
return {
"access_token": access_token,
"refresh_token": refresh_token,
"id_token": token_payload.get("id_token"),
"token_type": token_payload.get("token_type") or "Bearer",
"token_endpoint": token_endpoint,
"expires_at": expires_at,
}
def _refresh_tokens(self, auth_data: Dict[str, Any]) -> Dict[str, Any]:
token_endpoint = auth_data.get("token_endpoint")
if not token_endpoint:
token_endpoint = self._discover()["token_endpoint"]
token_endpoint = self._validate_xai_endpoint(token_endpoint)
refresh_token = auth_data.get("refresh_token")
if not refresh_token:
raise XAIOAuthLoginRequiredError(
"xAI OAuth refresh token missing. Run `litellm xai-oauth login`."
)
token_payload = self._exchange_token(
token_endpoint,
{
"grant_type": "refresh_token",
"refresh_token": refresh_token,
"client_id": XAI_OAUTH_CLIENT_ID,
},
)
refreshed = self._build_auth_record(
token_payload,
token_endpoint,
fallback_refresh_token=refresh_token,
)
self._write_auth_file(refreshed)
return refreshed
def should_use_xai_oauth(litellm_params: Optional[Dict[str, Any]]) -> bool:
return bool((litellm_params or {}).get("use_xai_oauth"))

38
litellm/llms/xai/responses/transformation.py
View file

@ -3,6 +3,7 @@ from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union
import litellm
from litellm._logging import verbose_logger
from litellm.constants import XAI_API_BASE
from litellm.exceptions import AuthenticationError
from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig
from litellm.llms.xai.common_utils import XAIModelInfo
from litellm.secret_managers.main import get_secret_str
@ -220,10 +221,27 @@ class XAIResponsesAPIConfig(OpenAIResponsesAPIConfig):
litellm_params.api_key, legacy_generic_before_env=True
)
if not api_key:
from litellm.llms.xai.oauth import (
XAIOAuthAuthenticator,
XAIOAuthError,
should_use_xai_oauth,
)
if should_use_xai_oauth(litellm_params.model_dump()):
try:
api_key = XAIOAuthAuthenticator().get_access_token()
except XAIOAuthError as exc:
raise AuthenticationError(
model=model,
llm_provider=self.custom_llm_provider.value,
message=str(exc),
) from exc
if not api_key:
raise ValueError(
"XAI API key is required. Set api_key, litellm.xai_key, "
"litellm.api_key, or XAI_API_KEY."
"litellm.api_key, XAI_API_KEY, or use_xai_oauth=True."
)
headers.update(
@ -244,12 +262,20 @@ class XAIResponsesAPIConfig(OpenAIResponsesAPIConfig):
Returns:
str: The full URL for the XAI /responses endpoint
"""
api_base = (
api_base
or litellm.api_base
or get_secret_str("XAI_API_BASE")
or XAI_API_BASE
from litellm.llms.xai.oauth import XAIOAuthAuthenticator, should_use_xai_oauth
api_key = XAIModelInfo.get_api_key(
litellm_params.get("api_key"), legacy_generic_before_env=True
)
if should_use_xai_oauth(litellm_params) and not api_key:
api_base = XAIOAuthAuthenticator().get_api_base()
else:
api_base = (
api_base
or litellm.api_base
or get_secret_str("XAI_API_BASE")
or XAI_API_BASE
)
# Remove trailing slashes
api_base = api_base.rstrip("/")

11
litellm/main.py
View file

@ -1638,6 +1638,7 @@ def completion( # type: ignore # noqa: PLR0915
litellm_request_debug=kwargs.get("litellm_request_debug", False),
tpm=kwargs.get("tpm"),
rpm=kwargs.get("rpm"),
use_xai_oauth=kwargs.get("use_xai_oauth", False),
)
cast(LiteLLMLoggingObj, logging).update_environment_variables(
model=model,
@ -2134,9 +2135,6 @@ def completion( # type: ignore # noqa: PLR0915
headers = headers or litellm.headers
if extra_headers is not None:
optional_params["extra_headers"] = extra_headers
## LOAD CONFIG - if set
config = litellm.OpenAITextCompletionConfig.get_config()
for k, v in config.items():
@ -2162,6 +2160,7 @@ def completion( # type: ignore # noqa: PLR0915
_response = openai_text_completions.completion(
model=model,
messages=messages,
headers=headers,
model_response=model_response,
print_verbose=print_verbose,
api_key=api_key,
@ -7761,6 +7760,9 @@ def stream_chunk_builder( # noqa: PLR0915
"cost",
logging_obj._response_cost_calculator(result=response),
)
processor.apply_provider_assembled_streaming_metadata(
response, chunks, logging_obj
)
return response
tool_call_chunks = [
@ -7940,6 +7942,9 @@ def stream_chunk_builder( # noqa: PLR0915
usage, "cost", logging_obj._response_cost_calculator(result=response)
)
processor.apply_provider_assembled_streaming_metadata(
response, chunks, logging_obj
)
return response
except Exception as e:
verbose_logger.exception(

513
litellm/model_prices_and_context_window_backup.json
View file

@ -1156,6 +1156,7 @@
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_sampling_params": false,
"supports_tool_choice": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
@ -1202,6 +1203,7 @@
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_sampling_params": false,
"supports_tool_choice": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
@ -1233,6 +1235,7 @@
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_sampling_params": false,
"supports_tool_choice": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
@ -1264,6 +1267,7 @@
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_sampling_params": false,
"supports_tool_choice": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
@ -1295,6 +1299,139 @@
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_sampling_params": false,
"supports_tool_choice": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
"supports_native_structured_output": true,
"supports_max_reasoning_effort": true,
"supports_output_config": true,
"bedrock_output_config_effort_ceiling": "xhigh"
},
"anthropic.claude-fable-5": {
"cache_creation_input_token_cost": 1.25e-05,
"cache_creation_input_token_cost_above_1hr": 2e-05,
"cache_read_input_token_cost": 1e-06,
"input_cost_per_token": 1e-05,
"litellm_provider": "bedrock_converse",
"max_input_tokens": 1000000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 5e-05,
"search_context_cost_per_query": {
"search_context_size_high": 0.01,
"search_context_size_low": 0.01,
"search_context_size_medium": 0.01
},
"supports_adaptive_thinking": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_sampling_params": false,
"supports_tool_choice": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
"supports_native_structured_output": true,
"supports_max_reasoning_effort": true,
"supports_output_config": true,
"bedrock_output_config_effort_ceiling": "xhigh"
},
"global.anthropic.claude-fable-5": {
"cache_creation_input_token_cost": 1.25e-05,
"cache_creation_input_token_cost_above_1hr": 2e-05,
"cache_read_input_token_cost": 1e-06,
"input_cost_per_token": 1e-05,
"litellm_provider": "bedrock_converse",
"max_input_tokens": 1000000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 5e-05,
"search_context_cost_per_query": {
"search_context_size_high": 0.01,
"search_context_size_low": 0.01,
"search_context_size_medium": 0.01
},
"supports_adaptive_thinking": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_sampling_params": false,
"supports_tool_choice": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
"supports_native_structured_output": true,
"supports_max_reasoning_effort": true,
"supports_output_config": true,
"bedrock_output_config_effort_ceiling": "xhigh"
},
"us.anthropic.claude-fable-5": {
"cache_creation_input_token_cost": 1.375e-05,
"cache_creation_input_token_cost_above_1hr": 2.2e-05,
"cache_read_input_token_cost": 1.1e-06,
"input_cost_per_token": 1.1e-05,
"litellm_provider": "bedrock_converse",
"max_input_tokens": 1000000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 5.5e-05,
"search_context_cost_per_query": {
"search_context_size_high": 0.01,
"search_context_size_low": 0.01,
"search_context_size_medium": 0.01
},
"supports_adaptive_thinking": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_sampling_params": false,
"supports_tool_choice": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
"supports_native_structured_output": true,
"supports_max_reasoning_effort": true,
"supports_output_config": true,
"bedrock_output_config_effort_ceiling": "xhigh"
},
"eu.anthropic.claude-fable-5": {
"cache_creation_input_token_cost": 1.375e-05,
"cache_creation_input_token_cost_above_1hr": 2.2e-05,
"cache_read_input_token_cost": 1.1e-06,
"input_cost_per_token": 1.1e-05,
"litellm_provider": "bedrock_converse",
"max_input_tokens": 1000000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 5.5e-05,
"search_context_cost_per_query": {
"search_context_size_high": 0.01,
"search_context_size_low": 0.01,
"search_context_size_medium": 0.01
},
"supports_adaptive_thinking": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_sampling_params": false,
"supports_tool_choice": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
@ -1327,6 +1464,7 @@
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_sampling_params": false,
"supports_tool_choice": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
@ -1359,6 +1497,7 @@
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_sampling_params": false,
"supports_tool_choice": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
@ -1391,6 +1530,7 @@
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_sampling_params": false,
"supports_tool_choice": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
@ -1423,6 +1563,7 @@
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_sampling_params": false,
"supports_tool_choice": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
@ -1455,6 +1596,7 @@
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_sampling_params": false,
"supports_tool_choice": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
@ -1485,6 +1627,7 @@
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_sampling_params": false,
"supports_tool_choice": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
@ -2208,6 +2351,37 @@
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_sampling_params": false,
"supports_tool_choice": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
"supports_max_reasoning_effort": true
},
"azure_ai/claude-fable-5": {
"input_cost_per_token": 1e-05,
"output_cost_per_token": 5e-05,
"litellm_provider": "azure_ai",
"max_input_tokens": 1000000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"search_context_cost_per_query": {
"search_context_size_high": 0.01,
"search_context_size_low": 0.01,
"search_context_size_medium": 0.01
},
"cache_creation_input_token_cost": 1.25e-05,
"cache_creation_input_token_cost_above_1hr": 2e-05,
"cache_read_input_token_cost": 1e-06,
"supports_adaptive_thinking": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_sampling_params": false,
"supports_tool_choice": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
@ -2237,6 +2411,7 @@
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_sampling_params": false,
"supports_tool_choice": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
@ -6889,6 +7064,43 @@
"/v1/images/generations"
]
},
"azure_ai/MAI-Image-2.5": {
"input_cost_per_image_token": 8e-06,
"input_cost_per_token": 5e-06,
"litellm_provider": "azure_ai",
"mode": "image_generation",
"output_cost_per_image": 0.05,
"output_cost_per_image_token": 4.7e-05,
"source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/new-mai-models-in-microsoft-foundry-across-text-image-voice-and-speech/4524632",
"supported_endpoints": [
"/v1/images/generations",
"/v1/images/edits"
]
},
"azure_ai/MAI-Image-2.5-Flash": {
"input_cost_per_image_token": 1.75e-06,
"input_cost_per_token": 1.75e-06,
"litellm_provider": "azure_ai",
"mode": "image_generation",
"output_cost_per_image": 0.0338,
"output_cost_per_image_token": 3.3e-05,
"source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/new-mai-models-in-microsoft-foundry-across-text-image-voice-and-speech/4524632",
"supported_endpoints": [
"/v1/images/generations",
"/v1/images/edits"
]
},
"azure_ai/MAI-Image-2e": {
"input_cost_per_token": 5e-06,
"litellm_provider": "azure_ai",
"mode": "image_generation",
"output_cost_per_image": 0.02,
"output_cost_per_image_token": 1.95e-05,
"source": "https://aka.ms/mai-image-2e-foundryblog",
"supported_endpoints": [
"/v1/images/generations"
]
},
"azure_ai/Llama-3.2-11B-Vision-Instruct": {
"input_cost_per_token": 3.7e-07,
"litellm_provider": "azure_ai",
@ -10133,6 +10345,7 @@
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_sampling_params": false,
"supports_tool_choice": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
@ -10167,6 +10380,7 @@
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_sampling_params": false,
"supports_tool_choice": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
@ -10177,6 +10391,40 @@
},
"supports_output_config": true
},
"claude-fable-5": {
"cache_creation_input_token_cost": 1.25e-05,
"cache_creation_input_token_cost_above_1hr": 2e-05,
"cache_read_input_token_cost": 1e-06,
"input_cost_per_token": 1e-05,
"litellm_provider": "anthropic",
"max_input_tokens": 1000000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 5e-05,
"search_context_cost_per_query": {
"search_context_size_high": 0.01,
"search_context_size_low": 0.01,
"search_context_size_medium": 0.01
},
"supports_adaptive_thinking": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_sampling_params": false,
"supports_tool_choice": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
"supports_max_reasoning_effort": true,
"provider_specific_entry": {
"us": 1.1
},
"supports_output_config": true
},
"claude-opus-4-8": {
"cache_creation_input_token_cost": 6.25e-06,
"cache_creation_input_token_cost_above_1hr": 1e-05,
@ -10201,6 +10449,7 @@
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_sampling_params": false,
"supports_tool_choice": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
@ -14014,6 +14263,22 @@
"/v1/images/generations"
]
},
"fal_ai/fal-ai/nano-banana": {
"litellm_provider": "fal_ai",
"mode": "image_generation",
"output_cost_per_image": 0.039,
"supported_endpoints": [
"/v1/images/generations"
]
},
"fal_ai/fal-ai/gemini-25-flash-image": {
"litellm_provider": "fal_ai",
"mode": "image_generation",
"output_cost_per_image": 0.039,
"supported_endpoints": [
"/v1/images/generations"
]
},
"featherless_ai/featherless-ai/Qwerky-72B": {
"litellm_provider": "featherless_ai",
"max_input_tokens": 32768,
@ -14270,10 +14535,10 @@
"mode": "chat",
"output_cost_per_token": 4.4e-06,
"source": "https://fireworks.ai/models/fireworks/glm-5p1",
"supports_function_calling": false,
"supports_function_calling": true,
"supports_reasoning": true,
"supports_response_schema": false,
"supports_tool_choice": false
"supports_response_schema": true,
"supports_tool_choice": true
},
"fireworks_ai/accounts/fireworks/models/gpt-oss-120b": {
"input_cost_per_token": 1.5e-07,
@ -14551,10 +14816,10 @@
"mode": "chat",
"output_cost_per_token": 4.4e-06,
"source": "https://fireworks.ai/models/fireworks/glm-5p1",
"supports_function_calling": false,
"supports_function_calling": true,
"supports_reasoning": true,
"supports_response_schema": false,
"supports_tool_choice": false
"supports_response_schema": true,
"supports_tool_choice": true
},
"fireworks_ai/kimi-k2p5": {
"cache_read_input_token_cost": 1e-07,
@ -24127,9 +24392,12 @@
"max_output_tokens": 8192
},
"minimax/MiniMax-M3": {
"input_cost_per_token": 6e-07,
"output_cost_per_token": 2.4e-06,
"cache_read_input_token_cost": 1.2e-07,
"input_cost_per_token": 3e-07,
"input_cost_per_token_above_512k_tokens": 6e-07,
"output_cost_per_token": 1.2e-06,
"output_cost_per_token_above_512k_tokens": 2.4e-06,
"cache_read_input_token_cost": 6e-08,
"cache_read_input_token_cost_above_512k_tokens": 1.2e-07,
"litellm_provider": "minimax",
"mode": "chat",
"supports_function_calling": true,
@ -24138,7 +24406,7 @@
"supports_reasoning": true,
"supports_system_messages": true,
"supports_vision": true,
"max_input_tokens": 512000,
"max_input_tokens": 1000000,
"max_output_tokens": 128000
},
"mistral.devstral-2-123b": {
@ -33951,6 +34219,7 @@
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_sampling_params": false,
"supports_tool_choice": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
@ -33979,6 +34248,67 @@
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_sampling_params": false,
"supports_tool_choice": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
"supports_max_reasoning_effort": true
},
"vertex_ai/claude-fable-5": {
"cache_creation_input_token_cost": 1.25e-05,
"cache_creation_input_token_cost_above_1hr": 2e-05,
"cache_read_input_token_cost": 1e-06,
"input_cost_per_token": 1e-05,
"litellm_provider": "vertex_ai-anthropic_models",
"max_input_tokens": 1000000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 5e-05,
"search_context_cost_per_query": {
"search_context_size_high": 0.01,
"search_context_size_low": 0.01,
"search_context_size_medium": 0.01
},
"supports_adaptive_thinking": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_sampling_params": false,
"supports_tool_choice": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
"supports_max_reasoning_effort": true
},
"vertex_ai/claude-fable-5@default": {
"cache_creation_input_token_cost": 1.25e-05,
"cache_creation_input_token_cost_above_1hr": 2e-05,
"cache_read_input_token_cost": 1e-06,
"input_cost_per_token": 1e-05,
"litellm_provider": "vertex_ai-anthropic_models",
"max_input_tokens": 1000000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 5e-05,
"search_context_cost_per_query": {
"search_context_size_high": 0.01,
"search_context_size_low": 0.01,
"search_context_size_medium": 0.01
},
"supports_adaptive_thinking": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_sampling_params": false,
"supports_tool_choice": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
@ -34008,6 +34338,7 @@
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_sampling_params": false,
"supports_tool_choice": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
@ -34037,6 +34368,7 @@
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_sampling_params": false,
"supports_tool_choice": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
@ -41317,6 +41649,7 @@
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "responses",
"use_openai_responses_path": true,
"supported_endpoints": ["/v1/responses"],
"supported_modalities": ["text", "image"],
"supported_output_modalities": ["text"],
@ -41336,6 +41669,7 @@
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "responses",
"use_openai_responses_path": true,
"supported_endpoints": ["/v1/responses"],
"supported_modalities": ["text", "image"],
"supported_output_modalities": ["text"],
@ -41637,5 +41971,164 @@
"/v1/audio/transcriptions"
],
"supports_audio_input": true
},
"tensormesh/Qwen/Qwen3.5-397B-A17B-FP8": {
"litellm_provider": "tensormesh",
"mode": "chat",
"input_cost_per_token": 6e-07,
"output_cost_per_token": 3.6e-06,
"cache_read_input_token_cost": 0,
"max_input_tokens": 262144,
"max_output_tokens": 262144,
"supports_function_calling": true,
"supports_tool_choice": true,
"supports_response_schema": true,
"supports_prompt_caching": true,
"supports_system_messages": true,
"supports_reasoning": true,
"source": "https://serverless.tensormesh.ai/v1/models/openrouter"
},
"tensormesh/Qwen/Qwen3-Coder-480B-A35B-Instruct-FP8": {
"litellm_provider": "tensormesh",
"mode": "chat",
"input_cost_per_token": 4.5e-07,
"output_cost_per_token": 1.8e-06,
"cache_read_input_token_cost": 0,
"max_input_tokens": 262144,
"max_output_tokens": 262144,
"supports_function_calling": true,
"supports_tool_choice": true,
"supports_response_schema": true,
"supports_prompt_caching": true,
"supports_system_messages": true,
"source": "https://serverless.tensormesh.ai/v1/models/openrouter"
},
"tensormesh/Qwen/Qwen3.6-27B-FP8": {
"litellm_provider": "tensormesh",
"mode": "chat",
"input_cost_per_token": 3.2e-07,
"output_cost_per_token": 3.2e-06,
"cache_read_input_token_cost": 0,
"max_input_tokens": 262144,
"max_output_tokens": 262144,
"supports_function_calling": true,
"supports_tool_choice": true,
"supports_response_schema": true,
"supports_prompt_caching": true,
"supports_system_messages": true,
"supports_reasoning": true,
"source": "https://serverless.tensormesh.ai/v1/models/openrouter"
},
"tensormesh/lukealonso/GLM-5.1-NVFP4-MTP": {
"litellm_provider": "tensormesh",
"mode": "chat",
"input_cost_per_token": 1.4e-06,
"output_cost_per_token": 4.4e-06,
"cache_read_input_token_cost": 0,
"max_input_tokens": 202752,
"max_output_tokens": 202752,
"supports_function_calling": true,
"supports_tool_choice": true,
"supports_response_schema": true,
"supports_prompt_caching": true,
"supports_system_messages": true,
"supports_reasoning": true,
"source": "https://serverless.tensormesh.ai/v1/models/openrouter"
},
"tensormesh/deepseek-ai/DeepSeek-V4-Flash": {
"litellm_provider": "tensormesh",
"mode": "chat",
"input_cost_per_token": 1.4e-07,
"output_cost_per_token": 2.8e-07,
"cache_read_input_token_cost": 0,
"max_input_tokens": 32768,
"max_output_tokens": 32768,
"supports_function_calling": true,
"supports_tool_choice": true,
"supports_response_schema": true,
"supports_prompt_caching": true,
"supports_system_messages": true,
"supports_reasoning": true,
"source": "https://serverless.tensormesh.ai/v1/models/openrouter"
},
"tensormesh/moonshotai/Kimi-K2.6": {
"litellm_provider": "tensormesh",
"mode": "chat",
"input_cost_per_token": 9.6e-07,
"output_cost_per_token": 4e-06,
"cache_read_input_token_cost": 0,
"max_input_tokens": 32768,
"max_output_tokens": 32768,
"supports_function_calling": true,
"supports_tool_choice": true,
"supports_response_schema": true,
"supports_prompt_caching": true,
"supports_system_messages": true,
"supports_reasoning": true,
"source": "https://serverless.tensormesh.ai/v1/models/openrouter"
},
"tensormesh/MiniMaxAI/MiniMax-M2.5": {
"litellm_provider": "tensormesh",
"mode": "chat",
"input_cost_per_token": 3e-07,
"output_cost_per_token": 1.2e-06,
"cache_read_input_token_cost": 0,
"max_input_tokens": 196608,
"max_output_tokens": 196608,
"supports_function_calling": true,
"supports_tool_choice": true,
"supports_response_schema": true,
"supports_prompt_caching": true,
"supports_system_messages": true,
"supports_reasoning": true,
"source": "https://serverless.tensormesh.ai/v1/models/openrouter"
},
"tensormesh/google/gemma-4-31B-it": {
"litellm_provider": "tensormesh",
"mode": "chat",
"input_cost_per_token": 1.4e-07,
"output_cost_per_token": 5.6e-07,
"cache_read_input_token_cost": 0,
"max_input_tokens": 32768,
"max_output_tokens": 32768,
"supports_function_calling": true,
"supports_tool_choice": true,
"supports_response_schema": true,
"supports_prompt_caching": true,
"supports_system_messages": true,
"supports_reasoning": true,
"source": "https://serverless.tensormesh.ai/v1/models/openrouter"
},
"tensormesh/openai/gpt-oss-120b": {
"litellm_provider": "tensormesh",
"mode": "chat",
"input_cost_per_token": 1.5e-07,
"output_cost_per_token": 6e-07,
"cache_read_input_token_cost": 0,
"max_input_tokens": 131072,
"max_output_tokens": 131072,
"supports_function_calling": true,
"supports_tool_choice": true,
"supports_response_schema": true,
"supports_prompt_caching": true,
"supports_system_messages": true,
"supports_reasoning": true,
"source": "https://serverless.tensormesh.ai/v1/models/openrouter"
},
"tensormesh/openai/gpt-oss-20b": {
"litellm_provider": "tensormesh",
"mode": "chat",
"input_cost_per_token": 7e-08,
"output_cost_per_token": 2.8e-07,
"cache_read_input_token_cost": 0,
"max_input_tokens": 131072,
"max_output_tokens": 131072,
"supports_function_calling": true,
"supports_tool_choice": true,
"supports_response_schema": true,
"supports_prompt_caching": true,
"supports_system_messages": true,
"supports_reasoning": true,
"source": "https://serverless.tensormesh.ai/v1/models/openrouter"
}
}

66
litellm/models/__init__.py Normal file
View file

@ -0,0 +1,66 @@
"""
Domain models for LiteLLM backend.
"""
from litellm.models.access_group import LiteLLM_AccessGroupTable
from litellm.models.budget import (
LiteLLM_BudgetTable,
LiteLLM_BudgetTableFull,
LiteLLM_TeamMemberTable,
)
from litellm.models.config import LiteLLM_Config
from litellm.models.credentials import (
CreateCredentialItem,
CredentialBase,
CredentialItem,
)
from litellm.models.end_user import LiteLLM_EndUserTable
from litellm.models.managed_files import (
LiteLLM_ManagedFileTable,
LiteLLM_ManagedObjectTable,
LiteLLM_ManagedVectorStoresTable,
LiteLLM_ManagedVectorStoreTable,
)
from litellm.models.mcp_server import LiteLLM_MCPServerTable
from litellm.models.model import LiteLLM_ProxyModelTable
from litellm.models.object_permission import LiteLLM_ObjectPermissionTable
from litellm.models.organization import LiteLLM_OrganizationTable
from litellm.models.organization_membership import LiteLLM_OrganizationMembershipTable
from litellm.models.project import LiteLLM_ProjectTable
from litellm.models.skills import LiteLLM_SkillsTable
from litellm.models.spend_logs import LiteLLM_ErrorLogs, LiteLLM_SpendLogs
from litellm.models.tag import LiteLLM_TagTable
from litellm.models.team import LiteLLM_TeamTable
from litellm.models.team_membership import LiteLLM_TeamMembership
from litellm.models.user import LiteLLM_UserTable
from litellm.models.verification_token import LiteLLM_VerificationToken
__all__ = [
"LiteLLM_AccessGroupTable",
"LiteLLM_BudgetTable",
"LiteLLM_BudgetTableFull",
"LiteLLM_TeamMemberTable",
"LiteLLM_Config",
"CredentialBase",
"CredentialItem",
"CreateCredentialItem",
"LiteLLM_EndUserTable",
"LiteLLM_ManagedFileTable",
"LiteLLM_ManagedObjectTable",
"LiteLLM_ManagedVectorStoreTable",
"LiteLLM_ManagedVectorStoresTable",
"LiteLLM_MCPServerTable",
"LiteLLM_ProxyModelTable",
"LiteLLM_ObjectPermissionTable",
"LiteLLM_OrganizationTable",
"LiteLLM_OrganizationMembershipTable",
"LiteLLM_ProjectTable",
"LiteLLM_SkillsTable",
"LiteLLM_ErrorLogs",
"LiteLLM_SpendLogs",
"LiteLLM_TagTable",
"LiteLLM_TeamTable",
"LiteLLM_TeamMembership",
"LiteLLM_UserTable",
"LiteLLM_VerificationToken",
]

26
litellm/models/access_group.py Normal file
View file

@ -0,0 +1,26 @@
"""
Access group table model.
Canonical definition for ``litellm_accessgrouptable``. Re-exported from
``litellm.proxy._types`` for backwards compatibility.
"""
from datetime import datetime
from typing import List, Optional
from litellm.types.llms.base import LiteLLMPydanticObjectBase
class LiteLLM_AccessGroupTable(LiteLLMPydanticObjectBase):
access_group_id: str
access_group_name: str
description: Optional[str] = None
access_model_names: List[str] = []
access_mcp_server_ids: List[str] = []
access_agent_ids: List[str] = []
assigned_team_ids: List[str] = []
assigned_key_ids: List[str] = []
created_at: Optional[datetime] = None
created_by: Optional[str] = None
updated_at: Optional[datetime] = None
updated_by: Optional[str] = None

38
litellm/models/base.py Normal file
View file

@ -0,0 +1,38 @@
"""
Base model class for domain models.
"""
from datetime import datetime
from typing import Any, Dict, Optional
from pydantic import BaseModel, ConfigDict
class DomainModel(BaseModel):
"""Base class for all domain models."""
model_config = ConfigDict(
from_attributes=True,
protected_namespaces=(),
extra="ignore",
)
created_at: Optional[datetime] = None
updated_at: Optional[datetime] = None
@classmethod
def from_db_record(cls, record: Any) -> "DomainModel":
"""Create a domain model from a database record."""
if record is None:
raise ValueError("Cannot create domain model from None record")
if isinstance(record, dict):
return cls(**record)
if hasattr(record, "model_dump") and callable(record.model_dump):
return cls(**record.model_dump())
if hasattr(record, "dict") and callable(record.dict):
return cls(**record.dict())
return cls(**dict(record))
def to_db_dict(self, exclude_unset: bool = False) -> Dict[str, Any]:
"""Convert domain model to a dictionary for database operations."""
return self.model_dump(exclude_none=True, exclude_unset=exclude_unset)

56
litellm/models/budget.py Normal file
View file

@ -0,0 +1,56 @@
"""
Budget table model.
Canonical definition for ``litellm_budgettable``. Re-exported from
``litellm.proxy._types`` for backwards compatibility.
"""
from datetime import datetime
from typing import List, Optional
from pydantic import ConfigDict
from litellm.types.llms.base import LiteLLMPydanticObjectBase
class LiteLLM_BudgetTable(LiteLLMPydanticObjectBase):
"""Represents user-controllable params for a LiteLLM_BudgetTable record.
Budget-write paths use `model_fields.keys()` on this class as an allowlist
for user input. Keep server-managed fields (e.g. `budget_reset_at`) on
`LiteLLM_BudgetTableFull` so they aren't user-settable.
"""
budget_id: Optional[str] = None
soft_budget: Optional[float] = None
max_budget: Optional[float] = None
max_parallel_requests: Optional[int] = None
tpm_limit: Optional[int] = None
rpm_limit: Optional[int] = None
model_max_budget: Optional[dict] = None
budget_duration: Optional[str] = None
allowed_models: Optional[List[str]] = (
None # per-member model scope; empty = inherit team models
)
model_config = ConfigDict(protected_namespaces=())
class LiteLLM_BudgetTableFull(LiteLLM_BudgetTable):
"""LiteLLM_BudgetTable + server-managed fields returned on API responses."""
budget_reset_at: Optional[datetime] = None
created_at: datetime
class LiteLLM_TeamMemberTable(LiteLLM_BudgetTable):
"""
Used to track spend of a user_id within a team_id
"""
spend: Optional[float] = None
user_id: Optional[str] = None
team_id: Optional[str] = None
budget_id: Optional[str] = None
model_config = ConfigDict(protected_namespaces=())

15
litellm/models/config.py Normal file
View file

@ -0,0 +1,15 @@
"""
Config table model.
Canonical definition for ``litellm_config``. Re-exported from
``litellm.proxy._types`` for backwards compatibility.
"""
from typing import Dict
from litellm.types.llms.base import LiteLLMPydanticObjectBase
class LiteLLM_Config(LiteLLMPydanticObjectBase):
param_name: str
param_value: Dict

31
litellm/models/credentials.py Normal file
View file

@ -0,0 +1,31 @@
"""
Credential table models.
These are the canonical credential types for the proxy. They live in the model
layer; ``litellm.types.utils`` re-exports them for backwards compatibility.
"""
from typing import Optional
from pydantic import BaseModel, model_validator
class CredentialBase(BaseModel):
credential_name: str
credential_info: dict
class CredentialItem(CredentialBase):
credential_values: dict
class CreateCredentialItem(CredentialBase):
credential_values: Optional[dict] = None
model_id: Optional[str] = None
@model_validator(mode="before")
@classmethod
def check_credential_params(cls, values):
if not values.get("credential_values") and not values.get("model_id"):
raise ValueError("Either credential_values or model_id must be set")
return values

35
litellm/models/end_user.py Normal file
View file

@ -0,0 +1,35 @@
"""
End-user table model.
Canonical definition for ``litellm_endusertable``. Re-exported from
``litellm.proxy._types`` for backwards compatibility.
"""
from typing import Literal, Optional
from pydantic import ConfigDict, model_validator
from litellm.models.budget import LiteLLM_BudgetTable
from litellm.models.object_permission import LiteLLM_ObjectPermissionTable
from litellm.types.llms.base import LiteLLMPydanticObjectBase
class LiteLLM_EndUserTable(LiteLLMPydanticObjectBase):
user_id: str
blocked: bool
alias: Optional[str] = None
spend: float = 0.0
allowed_model_region: Optional[Literal["eu", "us"]] = None
default_model: Optional[str] = None
litellm_budget_table: Optional[LiteLLM_BudgetTable] = None
object_permission_id: Optional[str] = None
object_permission: Optional[LiteLLM_ObjectPermissionTable] = None
@model_validator(mode="before")
@classmethod
def set_model_info(cls, values):
if values.get("spend") is None:
values.update({"spend": 0.0})
return values
model_config = ConfigDict(protected_namespaces=())

62
litellm/models/managed_files.py Normal file
View file

@ -0,0 +1,62 @@
"""
Managed file, object, and vector store table models.
Canonical definitions for the ``litellm_managed*`` tables. Re-exported from
``litellm.proxy._types`` for backwards compatibility.
"""
from datetime import datetime
from typing import Any, Dict, List, Literal, Optional, Union
from litellm.types.llms.base import LiteLLMPydanticObjectBase
from litellm.types.llms.openai import OpenAIFileObject, ResponsesAPIResponse
from litellm.types.utils import LiteLLMBatch, LiteLLMFineTuningJob
class LiteLLM_ManagedFileTable(LiteLLMPydanticObjectBase):
unified_file_id: str
file_object: Optional[OpenAIFileObject] = None
model_mappings: Dict[str, str]
flat_model_file_ids: List[str]
created_by: Optional[str] = None
team_id: Optional[str] = None
updated_by: Optional[str] = None
storage_backend: Optional[str] = None
storage_url: Optional[str] = None
class LiteLLM_ManagedObjectTable(LiteLLMPydanticObjectBase):
unified_object_id: str
model_object_id: str
file_purpose: Literal["batch", "fine-tune", "response", "container"]
file_object: Union[LiteLLMBatch, LiteLLMFineTuningJob, ResponsesAPIResponse]
created_by: Optional[str] = None
team_id: Optional[str] = None
class LiteLLM_ManagedVectorStoreTable(LiteLLMPydanticObjectBase):
"""Table for managing vector stores with target_model_names support."""
unified_resource_id: str
resource_object: Optional[Any] = None
model_mappings: Dict[str, str]
flat_model_resource_ids: List[str]
created_by: Optional[str] = None
team_id: Optional[str] = None
updated_by: Optional[str] = None
storage_backend: Optional[str] = None
storage_url: Optional[str] = None
class LiteLLM_ManagedVectorStoresTable(LiteLLMPydanticObjectBase):
vector_store_id: str
custom_llm_provider: str
vector_store_name: Optional[str]
vector_store_description: Optional[str]
vector_store_metadata: Optional[Dict[str, Any]]
created_at: Optional[datetime]
updated_at: Optional[datetime]
litellm_credential_name: Optional[str]
litellm_params: Optional[Dict[str, Any]]
team_id: Optional[str]
user_id: Optional[str]

103
litellm/models/mcp_server.py Normal file
View file

@ -0,0 +1,103 @@
"""
MCP server table model.
Canonical definition for ``litellm_mcpservertable``. Re-exported from
``litellm.proxy._types`` for backwards compatibility.
"""
import enum
from datetime import datetime
from typing import Dict, List, Literal, Optional
from pydantic import Field
from litellm.types.llms.base import LiteLLMPydanticObjectBase
from litellm.types.mcp import MCPAuthType, MCPCredentials, MCPTransportType
from litellm.types.mcp_server.mcp_server_manager import MCPInfo
class MCPEnvVarScope(str, enum.Enum):
"""Scope for an MCP server environment variable.
- ``global``: value is provided by the admin and used for all users.
- ``user``: each user must provide their own value via the per-user
env-var endpoint. The admin-supplied ``value`` is treated as a
placeholder/hint and is not used at request time.
"""
global_ = "global"
user = "user"
class MCPEnvVar(LiteLLMPydanticObjectBase):
"""One environment variable for an MCP server.
Variables can be interpolated into ``static_headers`` using ``${NAME}``
syntax. ``scope=global`` values are stored on the server. ``scope=user``
values are stored per-user in ``LiteLLM_MCPUserEnvVars`` and supplied by
each user.
"""
name: str
value: str = ""
scope: MCPEnvVarScope = MCPEnvVarScope.global_
description: Optional[str] = None
class LiteLLM_MCPServerTable(LiteLLMPydanticObjectBase):
"""Represents a LiteLLM_MCPServerTable record"""
server_id: str
server_name: Optional[str] = None
alias: Optional[str] = None
description: Optional[str] = None
url: Optional[str] = None
spec_path: Optional[str] = None
transport: MCPTransportType
auth_type: Optional[MCPAuthType] = None
credentials: Optional[MCPCredentials] = None
instructions: Optional[str] = None
created_at: Optional[datetime] = None
created_by: Optional[str] = None
updated_at: Optional[datetime] = None
updated_by: Optional[str] = None
teams: List[Dict[str, Optional[str]]] = Field(default_factory=list)
mcp_access_groups: List[str] = Field(default_factory=list)
allowed_tools: List[str] = Field(default_factory=list)
tool_name_to_display_name: Optional[Dict[str, str]] = None
tool_name_to_description: Optional[Dict[str, str]] = None
extra_headers: List[str] = Field(default_factory=list)
mcp_info: Optional[MCPInfo] = None
static_headers: Optional[Dict[str, str]] = None
env_vars: Optional[List[MCPEnvVar]] = None
status: Optional[Literal["healthy", "unhealthy", "unknown"]] = Field(
default="unknown",
description="Health status: 'healthy', 'unhealthy', 'unknown'",
)
last_health_check: Optional[datetime] = None
health_check_error: Optional[str] = None
command: Optional[str] = None
args: List[str] = Field(default_factory=list)
env: Dict[str, str] = Field(default_factory=dict)
authorization_url: Optional[str] = None
token_url: Optional[str] = None
registration_url: Optional[str] = None
oauth2_flow: Optional[Literal["client_credentials", "authorization_code"]] = None
allow_all_keys: bool = False
available_on_public_internet: bool = True
delegate_auth_to_upstream: bool = False
oauth_passthrough: bool = False
is_byok: bool = False
byok_description: List[str] = Field(default_factory=list)
byok_api_key_help_url: Optional[str] = None
has_user_credential: Optional[bool] = None
source_url: Optional[str] = None
timeout: Optional[float] = None
approval_status: Optional[str] = Field(
default="active",
description="Approval status: 'pending_review', 'active', 'rejected'",
)
submitted_by: Optional[str] = None
submitted_at: Optional[datetime] = None
reviewed_at: Optional[datetime] = None
review_notes: Optional[str] = None

59
litellm/models/model.py Normal file
View file

@ -0,0 +1,59 @@
"""
Proxy model table model.
Canonical definition for ``litellm_proxymodeltable``. Re-exported from
``litellm.proxy._types`` for backwards compatibility.
"""
import json
from datetime import datetime
from typing import Optional
from pydantic import ConfigDict, model_validator
from litellm.types.llms.base import LiteLLMPydanticObjectBase
class LiteLLM_ProxyModelTable(LiteLLMPydanticObjectBase):
model_id: str
model_name: str
litellm_params: dict
model_info: Optional[dict] = None
blocked: bool = False
created_at: Optional[datetime] = None
created_by: Optional[str] = None
updated_at: Optional[datetime] = None
updated_by: Optional[str] = None
model_config = ConfigDict(protected_namespaces=())
@model_validator(mode="before")
@classmethod
def check_potential_json_str(cls, values):
if isinstance(values.get("litellm_params"), str):
try:
values["litellm_params"] = json.loads(values["litellm_params"])
except json.JSONDecodeError:
pass
if isinstance(values.get("model_info"), str):
try:
values["model_info"] = json.loads(values["model_info"])
except json.JSONDecodeError:
pass
return values
@property
def is_blocked(self) -> bool:
return self.blocked
@property
def team_id(self) -> Optional[str]:
if self.model_info:
return self.model_info.get("team_id")
return None
@property
def team_public_model_name(self) -> Optional[str]:
if self.model_info:
return self.model_info.get("team_public_model_name")
return None

26
litellm/models/object_permission.py Normal file
View file

@ -0,0 +1,26 @@
"""
Object permission table model.
Canonical definition for ``litellm_objectpermissiontable``. Re-exported from
``litellm.proxy._types`` for backwards compatibility.
"""
from typing import Dict, List, Optional
from litellm.types.llms.base import LiteLLMPydanticObjectBase
class LiteLLM_ObjectPermissionTable(LiteLLMPydanticObjectBase):
"""Represents a LiteLLM_ObjectPermissionTable record"""
object_permission_id: str
mcp_servers: Optional[List[str]] = []
mcp_access_groups: Optional[List[str]] = []
mcp_tool_permissions: Optional[Dict[str, List[str]]] = None
vector_stores: Optional[List[str]] = []
agents: Optional[List[str]] = []
agent_access_groups: Optional[List[str]] = []
models: Optional[List[str]] = []
mcp_toolsets: Optional[List[str]] = None
blocked_tools: Optional[List[str]] = []
search_tools: Optional[List[str]] = []

31
litellm/models/organization.py Normal file
View file

@ -0,0 +1,31 @@
"""
Organization table model.
Canonical definition for ``litellm_organizationtable``. Re-exported from
``litellm.proxy._types`` for backwards compatibility.
"""
from typing import List, Optional
from litellm.models.budget import LiteLLM_BudgetTable
from litellm.models.object_permission import LiteLLM_ObjectPermissionTable
from litellm.models.user import LiteLLM_UserTable
from litellm.types.llms.base import LiteLLMPydanticObjectBase
class LiteLLM_OrganizationTable(LiteLLMPydanticObjectBase):
"""Represents user-controllable params for a LiteLLM_OrganizationTable record"""
organization_id: Optional[str] = None
organization_alias: Optional[str] = None
budget_id: str
spend: float = 0.0
metadata: Optional[dict] = None
models: List[str] = []
model_spend: Optional[dict] = {}
created_by: str
updated_by: str
users: Optional[List[LiteLLM_UserTable]] = None
litellm_budget_table: Optional[LiteLLM_BudgetTable] = None
object_permission: Optional[LiteLLM_ObjectPermissionTable] = None
object_permission_id: Optional[str] = None

40
litellm/models/organization_membership.py Normal file
View file

@ -0,0 +1,40 @@
"""
Organization membership table model.
Canonical definition for ``litellm_organizationmembership``. Re-exported from
``litellm.proxy._types`` for backwards compatibility.
"""
from datetime import datetime
from typing import Any, Optional
from pydantic import ConfigDict, model_validator
from litellm.models.budget import LiteLLM_BudgetTable
from litellm.types.llms.base import LiteLLMPydanticObjectBase
class LiteLLM_OrganizationMembershipTable(LiteLLMPydanticObjectBase):
"""Tracks which organizations a user belongs to and their spend within it."""
user_id: str
organization_id: str
user_role: Optional[str] = None
spend: float = 0.0
budget_id: Optional[str] = None
created_at: datetime
updated_at: datetime
user: Optional[Any] = None
litellm_budget_table: Optional[LiteLLM_BudgetTable] = None
user_email: Optional[str] = None
model_config = ConfigDict(protected_namespaces=())
@model_validator(mode="after")
def populate_user_email(self) -> "LiteLLM_OrganizationMembershipTable":
if self.user_email is None and self.user is not None:
if isinstance(self.user, dict):
self.user_email = self.user.get("user_email")
else:
self.user_email = getattr(self.user, "user_email", None)
return self

41
litellm/models/project.py Normal file
View file

@ -0,0 +1,41 @@
"""
Project table model.
Canonical definition for ``litellm_projecttable``. Re-exported from
``litellm.proxy._types`` for backwards compatibility.
"""
from datetime import datetime
from typing import List, Optional
from litellm.models.budget import LiteLLM_BudgetTable
from litellm.models.object_permission import LiteLLM_ObjectPermissionTable
from litellm.types.llms.base import LiteLLMPydanticObjectBase
class LiteLLM_ProjectTable(LiteLLMPydanticObjectBase):
"""Database model representation for project"""
project_id: str
project_alias: Optional[str] = None
description: Optional[str] = None
team_id: Optional[str] = None
budget_id: Optional[str] = None
metadata: Optional[dict] = None
models: List[str] = []
spend: float = 0.0
model_spend: Optional[dict] = None
model_rpm_limit: Optional[dict] = None
model_tpm_limit: Optional[dict] = None
blocked: bool = False
object_permission_id: Optional[str] = None
created_by: Optional[str] = None
updated_by: Optional[str] = None
created_at: Optional[datetime] = None
updated_at: Optional[datetime] = None
litellm_budget_table: Optional[LiteLLM_BudgetTable] = None
object_permission: Optional[LiteLLM_ObjectPermissionTable] = None
@property
def is_blocked(self) -> bool:
return self.blocked

30
litellm/models/skills.py Normal file
View file

@ -0,0 +1,30 @@
"""
Skills table model.
Canonical definition for ``litellm_skillstable``. Re-exported from
``litellm.proxy._types`` for backwards compatibility.
"""
from datetime import datetime
from typing import Any, Dict, Optional
from litellm.types.llms.base import LiteLLMPydanticObjectBase
class LiteLLM_SkillsTable(LiteLLMPydanticObjectBase):
"""Represents a LiteLLM_SkillsTable record"""
skill_id: str
display_title: Optional[str] = None
description: Optional[str] = None
instructions: Optional[str] = None
source: str = "custom"
latest_version: Optional[str] = None
file_content: Optional[bytes] = None
file_name: Optional[str] = None
file_type: Optional[str] = None
metadata: Optional[Dict[str, Any]] = None
created_at: Optional[datetime] = None
created_by: Optional[str] = None
updated_at: Optional[datetime] = None
updated_by: Optional[str] = None

50
litellm/models/spend_logs.py Normal file
View file

@ -0,0 +1,50 @@
"""
Spend and error log table models.
Canonical definitions for ``litellm_spendlogs`` and ``litellm_errorlogs``.
Re-exported from ``litellm.proxy._types`` for backwards compatibility.
"""
from datetime import datetime
from typing import Optional, Union
from pydantic import Json
from litellm._uuid import uuid
from litellm.types.llms.base import LiteLLMPydanticObjectBase
class LiteLLM_SpendLogs(LiteLLMPydanticObjectBase):
request_id: str
api_key: str
model: Optional[str] = ""
api_base: Optional[str] = ""
call_type: str
spend: Optional[float] = 0.0
total_tokens: Optional[int] = 0
prompt_tokens: Optional[int] = 0
completion_tokens: Optional[int] = 0
startTime: Union[str, datetime, None]
endTime: Union[str, datetime, None]
user: Optional[str] = ""
metadata: Optional[Json] = {}
cache_hit: Optional[str] = "False"
cache_key: Optional[str] = None
request_tags: Optional[Json] = None
requester_ip_address: Optional[str] = None
messages: Optional[Union[str, list, dict]]
response: Optional[Union[str, list, dict]]
class LiteLLM_ErrorLogs(LiteLLMPydanticObjectBase):
request_id: Optional[str] = str(uuid.uuid4())
api_base: Optional[str] = ""
model_group: Optional[str] = ""
litellm_model_name: Optional[str] = ""
model_id: Optional[str] = ""
request_kwargs: Optional[dict] = {}
exception_type: Optional[str] = ""
status_code: Optional[str] = ""
exception_string: Optional[str] = ""
startTime: Union[str, datetime, None]
endTime: Union[str, datetime, None]

36
litellm/models/tag.py Normal file
View file

@ -0,0 +1,36 @@
"""
Tag table model.
Canonical definition for ``litellm_tagtable``. Re-exported from
``litellm.proxy._types`` for backwards compatibility.
"""
from datetime import datetime
from typing import List, Optional
from pydantic import model_validator
from litellm.models.budget import LiteLLM_BudgetTable
from litellm.types.llms.base import LiteLLMPydanticObjectBase
class LiteLLM_TagTable(LiteLLMPydanticObjectBase):
tag_name: str
description: Optional[str] = None
models: List[str] = []
model_info: Optional[dict] = None
spend: float = 0.0
budget_id: Optional[str] = None
litellm_budget_table: Optional[LiteLLM_BudgetTable] = None
created_at: Optional[datetime] = None
created_by: Optional[str] = None
updated_at: Optional[datetime] = None
@model_validator(mode="before")
@classmethod
def set_model_info(cls, values):
if values.get("spend") is None:
values.update({"spend": 0.0})
if values.get("models") is None:
values.update({"models": []})
return values

154
litellm/models/team.py Normal file
View file

@ -0,0 +1,154 @@
"""
Team table models.
Canonical definitions for ``litellm_teamtable`` (plus the shared Member and
budget-window value types and the team-model alias table). Re-exported from
``litellm.proxy._types`` for backwards compatibility.
"""
import json
from datetime import datetime
from typing import List, Literal, Optional, Union
from pydantic import BaseModel, ConfigDict, Field, model_validator
from litellm.models.object_permission import LiteLLM_ObjectPermissionTable
from litellm.types.llms.base import LiteLLMPydanticObjectBase
class MemberBase(LiteLLMPydanticObjectBase):
user_id: Optional[str] = Field(
default=None,
description="The unique ID of the user to add. Either user_id or user_email must be provided",
)
user_email: Optional[str] = Field(
default=None,
description="The email address of the user to add. Either user_id or user_email must be provided",
)
@model_validator(mode="before")
@classmethod
def check_user_info(cls, values):
if not isinstance(values, dict):
raise ValueError("input needs to be a dictionary")
if values.get("user_id") is None and values.get("user_email") is None:
raise ValueError("Either user id or user email must be provided")
return values
class Member(MemberBase):
role: Literal["admin", "user"] = Field(
description="The role of the user within the team. 'admin' users can manage team settings and members, 'user' is a regular team member"
)
class BudgetLimitEntry(LiteLLMPydanticObjectBase):
"""A single budget window with its own limit and independent reset schedule."""
budget_duration: str
max_budget: float
reset_at: Optional[datetime] = None
class LiteLLM_ModelTable(LiteLLMPydanticObjectBase):
id: Optional[int] = None
model_aliases: Optional[Union[str, dict]] = None
created_by: str
updated_by: str
team: Optional["LiteLLM_TeamTable"] = None
model_config = ConfigDict(protected_namespaces=())
class TeamBase(LiteLLMPydanticObjectBase):
team_alias: Optional[str] = None
team_id: Optional[str] = None
organization_id: Optional[str] = None
admins: list = []
members: list = []
members_with_roles: List[Member] = []
team_member_permissions: Optional[List[str]] = None
metadata: Optional[dict] = None
tpm_limit: Optional[int] = None
rpm_limit: Optional[int] = None
max_budget: Optional[float] = None
soft_budget: Optional[float] = None
budget_duration: Optional[str] = None
budget_limits: Optional[List[BudgetLimitEntry]] = None
models: list = []
blocked: bool = False
router_settings: Optional[dict] = None
access_group_ids: Optional[List[str]] = None
default_team_member_models: Optional[List[str]] = None
class LiteLLM_TeamTable(TeamBase):
team_id: str # type: ignore
spend: Optional[float] = None
max_parallel_requests: Optional[int] = None
budget_duration: Optional[str] = None
budget_reset_at: Optional[datetime] = None
model_id: Optional[int] = None
model_spend: Optional[dict] = {}
model_max_budget: Optional[dict] = {}
policies: Optional[List[str]] = None
allow_team_guardrail_config: Optional[bool] = False
litellm_model_table: Optional[LiteLLM_ModelTable] = None
object_permission: Optional[LiteLLM_ObjectPermissionTable] = None
object_permission_id: Optional[str] = None
updated_at: Optional[datetime] = None
created_at: Optional[datetime] = None
model_config = ConfigDict(protected_namespaces=())
@model_validator(mode="before")
@classmethod
def set_model_info(cls, values):
dict_fields = [
"metadata",
"aliases",
"config",
"permissions",
"model_max_budget",
"model_aliases",
"router_settings",
"budget_limits",
]
if isinstance(values, BaseModel):
values = values.model_dump()
if (
isinstance(values.get("members_with_roles"), dict)
and not values["members_with_roles"]
):
values["members_with_roles"] = []
for field in dict_fields:
value = values.get(field)
if value is not None and isinstance(value, str):
try:
values[field] = json.loads(value)
except json.JSONDecodeError:
raise ValueError(f"Field {field} should be a valid dictionary")
return values
class LiteLLM_TeamTableCachedObj(LiteLLM_TeamTable):
last_refreshed_at: Optional[float] = None
class LiteLLM_DeletedTeamTable(LiteLLM_TeamTable):
"""Audit record for deleted teams; mirrors the team plus deletion metadata."""
id: Optional[str] = None
deleted_at: Optional[datetime] = None
deleted_by: Optional[str] = None
deleted_by_api_key: Optional[str] = None
litellm_changed_by: Optional[str] = None
model_config = ConfigDict(protected_namespaces=())
LiteLLM_ModelTable.model_rebuild()

32
litellm/models/team_membership.py Normal file
View file

@ -0,0 +1,32 @@
"""
Team membership table model.
Canonical definition for ``litellm_teammembership``. Re-exported from
``litellm.proxy._types`` for backwards compatibility.
"""
from typing import Optional, Union
from litellm.models.budget import LiteLLM_BudgetTable, LiteLLM_BudgetTableFull
from litellm.types.llms.base import LiteLLMPydanticObjectBase
class LiteLLM_TeamMembership(LiteLLMPydanticObjectBase):
user_id: str
team_id: str
budget_id: Optional[str] = None
spend: Optional[float] = 0.0
total_spend: Optional[float] = 0.0
litellm_budget_table: Optional[
Union[LiteLLM_BudgetTableFull, LiteLLM_BudgetTable]
] = None
def safe_get_team_member_rpm_limit(self) -> Optional[int]:
if self.litellm_budget_table is not None:
return self.litellm_budget_table.rpm_limit
return None
def safe_get_team_member_tpm_limit(self) -> Optional[int]:
if self.litellm_budget_table is not None:
return self.litellm_budget_table.tpm_limit
return None

70
litellm/models/user.py Normal file
View file

@ -0,0 +1,70 @@
"""
User table model.
Canonical definition for ``litellm_usertable``. Re-exported from
``litellm.proxy._types`` for backwards compatibility.
"""
from datetime import datetime
from typing import Dict, List, Optional
from pydantic import ConfigDict, Field, model_validator
from litellm.models.object_permission import LiteLLM_ObjectPermissionTable
from litellm.models.organization_membership import (
LiteLLM_OrganizationMembershipTable,
)
from litellm.types.llms.base import LiteLLMPydanticObjectBase
class LiteLLM_UserTable(LiteLLMPydanticObjectBase):
user_id: str
user_alias: Optional[str] = None
team_id: Optional[str] = None
sso_user_id: Optional[str] = None
organization_id: Optional[str] = None
object_permission_id: Optional[str] = None
password: Optional[str] = Field(default=None, exclude=True)
teams: List[str] = []
user_role: Optional[str] = None
max_budget: Optional[float] = None
spend: float = 0.0
user_email: Optional[str] = None
models: list = []
metadata: Optional[dict] = None
max_parallel_requests: Optional[int] = None
tpm_limit: Optional[int] = None
rpm_limit: Optional[int] = None
budget_duration: Optional[str] = None
budget_reset_at: Optional[datetime] = None
allowed_cache_controls: List[str] = []
policies: List[str] = []
model_spend: Optional[Dict] = {}
model_max_budget: Optional[Dict] = {}
created_at: Optional[datetime] = None
updated_at: Optional[datetime] = None
organization_memberships: Optional[List[LiteLLM_OrganizationMembershipTable]] = None
object_permission: Optional[LiteLLM_ObjectPermissionTable] = None
model_config = ConfigDict(protected_namespaces=())
@model_validator(mode="before")
@classmethod
def set_model_info(cls, values):
if values.get("spend") is None:
values.update({"spend": 0.0})
if values.get("models") is None:
values.update({"models": []})
if values.get("teams") is None:
values.update({"teams": []})
return values
def is_over_budget(self) -> bool:
if self.max_budget is None:
return False
return self.spend >= self.max_budget
def has_model_access(self, model_name: str) -> bool:
if not self.models:
return True
return model_name in self.models

74
litellm/models/verification_token.py Normal file
View file

@ -0,0 +1,74 @@
"""
Verification token table model.
Canonical definition for ``litellm_verificationtoken``. Re-exported from
``litellm.proxy._types`` for backwards compatibility.
"""
from datetime import datetime
from typing import Dict, List, Optional, Union
from pydantic import ConfigDict
from litellm.models.object_permission import LiteLLM_ObjectPermissionTable
from litellm.types.llms.base import LiteLLMPydanticObjectBase
class LiteLLM_VerificationToken(LiteLLMPydanticObjectBase):
token: Optional[str] = None
key_name: Optional[str] = None
key_alias: Optional[str] = None
spend: float = 0.0
max_budget: Optional[float] = None
expires: Optional[Union[str, datetime]] = None
models: List = []
aliases: Dict = {}
config: Dict = {}
user_id: Optional[str] = None
team_id: Optional[str] = None
agent_id: Optional[str] = None
project_id: Optional[str] = None
max_parallel_requests: Optional[int] = None
metadata: Dict = {}
tpm_limit: Optional[int] = None
rpm_limit: Optional[int] = None
budget_duration: Optional[str] = None
budget_reset_at: Optional[datetime] = None
allowed_cache_controls: Optional[list] = []
allowed_routes: Optional[list] = []
permissions: Dict = {}
model_spend: Dict = {}
model_max_budget: Dict = {}
soft_budget_cooldown: bool = False
blocked: Optional[bool] = None
litellm_budget_table: Optional[dict] = None
budget_id: Optional[str] = None
org_id: Optional[str] = None # org id for a given key
created_at: Optional[datetime] = None
created_by: Optional[str] = None
updated_at: Optional[datetime] = None
updated_by: Optional[str] = None
last_active: Optional[datetime] = None
object_permission_id: Optional[str] = None
object_permission: Optional[LiteLLM_ObjectPermissionTable] = None
access_group_ids: Optional[List[str]] = None
rotation_count: Optional[int] = 0
auto_rotate: Optional[bool] = False
rotation_interval: Optional[str] = None
last_rotation_at: Optional[datetime] = None
key_rotation_at: Optional[datetime] = None
router_settings: Optional[dict] = None
budget_limits: Optional[List[dict]] = None
model_config = ConfigDict(protected_namespaces=())
class LiteLLM_DeletedVerificationToken(LiteLLM_VerificationToken):
"""Audit record for deleted keys; mirrors the token plus deletion metadata."""
id: Optional[str] = None
deleted_at: Optional[datetime] = None
deleted_by: Optional[str] = None
deleted_by_api_key: Optional[str] = None
litellm_changed_by: Optional[str] = None
model_config = ConfigDict(protected_namespaces=())

Some files were not shown because too many files have changed in this diff Show more