diff --git a/.github/PULL_REQUEST_TEMPLATE/general.md b/.github/PULL_REQUEST_TEMPLATE/general.md
index 4f1a18e925e..f815601a29c 100644
--- a/.github/PULL_REQUEST_TEMPLATE/general.md
+++ b/.github/PULL_REQUEST_TEMPLATE/general.md
@@ -6,19 +6,12 @@
## Handwritten 1-3 sentence description
-
-``
-
-``
+[insert handwritten description here]
Details
@@ -100,6 +93,7 @@ If you're seeing a delay in your PR being merged, ping the LiteLLM Team on [Slac
Otherwise proof must be fully e2e, no mocks, with real provider calls where applicable. Pytest alone is insufficient. The following format applies to live proof
Show only the latest proof. Before uses the merge base and After the tested tip. Rerun affected QA after behavior changes. Verified neutral diffs may carry proof with the tested SHA, current tip, equivalence reason and focused checks, following AGENTS.md and /qa
Use matching Before/After case headings with numbered commands and observed output. Put shared setup above them. For one case, omit case headings
+ A reviewer must be able to rerun every case from the body alone, so the shared setup gives the proxy start command, the config.yaml, env var names (never values) and the client code or curl used, all exactly as run. Wrap any block longer than about 20 lines in a collapsed with a one-line summary
### Before ()
diff --git a/.github/workflows/image-scan.yml b/.github/workflows/image-scan.yml
index 3e2bca8779d..d769b6ab8af 100644
--- a/.github/workflows/image-scan.yml
+++ b/.github/workflows/image-scan.yml
@@ -53,7 +53,7 @@ jobs:
if: >-
github.event_name != 'pull_request' ||
github.event.pull_request.head.repo.full_name == github.repository
- timeout-minutes: 30
+ timeout-minutes: 60
permissions:
contents: read
steps:
diff --git a/.github/workflows/interactions-spec-canary.yml b/.github/workflows/interactions-spec-canary.yml
index a2e4646dce5..89ebdd87a7c 100644
--- a/.github/workflows/interactions-spec-canary.yml
+++ b/.github/workflows/interactions-spec-canary.yml
@@ -49,9 +49,9 @@ jobs:
id: compliance
run: |
uv run --no-sync pytest tests/unit/interactions/test_openapi_compliance.py \
- --interactions-spec-url=https://ai.google.dev/static/api/interactions.openapi.json \
+ --interactions-spec-url=https://aistudio.google.com/api-reference/interactions.openapi.json \
--tb=short -vv -p no:rerunfailures -p no:pytest-retry \
- --allow-hosts=127.0.0.1,::1,localhost,ai.google.dev
+ --allow-hosts=127.0.0.1,::1,localhost,ai.google.dev,aistudio.google.com
tracking-issue:
name: tracking-issue
diff --git a/.github/workflows/test-linting.yml b/.github/workflows/test-linting.yml
index cbeea71ad0f..fd1d2973951 100644
--- a/.github/workflows/test-linting.yml
+++ b/.github/workflows/test-linting.yml
@@ -21,7 +21,7 @@ jobs:
pull-requests: read
actions: read
runs-on: ubuntu-latest
- timeout-minutes: 20
+ timeout-minutes: 30
services:
redis:
image: redis@sha256:e2debfb7956fa12c7ddc79d7e645c8cf26b30c99a6e9161ea9bf4171e1668a5f
diff --git a/.github/workflows/test-unit.yml b/.github/workflows/test-unit.yml
index f1456b1fde8..fb29a92e306 100644
--- a/.github/workflows/test-unit.yml
+++ b/.github/workflows/test-unit.yml
@@ -45,7 +45,7 @@ jobs:
id: rust-bridge-key
if: steps.changes.outputs.decision != 'skip'
env:
- RUST_BRIDGE_CACHE_KEY: ${{ runner.os }}-rust-bridge-editable-${{ hashFiles('litellm-rust/Cargo.lock', 'litellm-rust/Cargo.toml', 'litellm-rust/crates/**', 'model_prices_and_context_window.json', 'pyproject.toml') }}
+ RUST_BRIDGE_CACHE_KEY: ${{ runner.os }}-rust-bridge-editable-${{ hashFiles('litellm-rust/Cargo.lock', 'litellm-rust/Cargo.toml', 'litellm-rust/crates/**', 'model_prices_and_context_window.json', 'pyproject.toml', 'packaging/litellm-proxy/pyproject.toml') }}
run: echo "key=$RUST_BRIDGE_CACHE_KEY" >> "$GITHUB_OUTPUT"
- name: Set up Python
@@ -98,6 +98,18 @@ jobs:
mkdir -p rust-bridge-dist/litellm/rust_bridge
cp litellm/rust_bridge/_native.abi3.so rust-bridge-dist/litellm/rust_bridge/_native.abi3.so
+ - name: Build the gateway wheel once for test shards
+ if: steps.changes.outputs.decision != 'skip' && steps.rust-bridge-cache.outputs.cache-hit != 'true'
+ timeout-minutes: 15
+ working-directory: packaging/litellm-proxy
+ env:
+ RUSTC_WRAPPER: sccache
+ SCCACHE_GHA_ENABLED: "true"
+ SCCACHE_GHA_VERSION: litellm-ci-v1
+ SCCACHE_GHA_RW_MODE: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' && 'READ_WRITE' || 'READ_ONLY' }}
+ CARGO_INCREMENTAL: "0"
+ run: uv run --no-project --with maturin==1.15.0 maturin build --profile dev --strip --out ../../rust-bridge-dist
+
- name: Upload Rust bridge artifact
if: steps.changes.outputs.decision != 'skip'
timeout-minutes: 10
@@ -170,6 +182,7 @@ jobs:
tests/unit/google_genai
tests/unit/router_strategy
tests/unit/router_utils
+ tests/unit/redis
tests/unit/proxy/common_utils/test_cache_aware_routing.py
tests/unit/enterprise/enterprise_callbacks/send_emails
tests/unit/enterprise/proxy/test_afile_retrieve_returns_unified_id.py
@@ -1022,9 +1035,9 @@ jobs:
run: |
diff -u model_prices_and_context_window.json litellm/model_prices_and_context_window_backup.json
if [ -z "$RUST_BRIDGE_ARTIFACT" ]; then
- .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra extra_proxy --extra proxy-runtime --extra utils
+ .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra switchyard --extra saml --extra extra_proxy --extra proxy-runtime --extra utils
else
- .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra extra_proxy --extra proxy-runtime --extra utils --no-install-project
+ .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra switchyard --extra saml --extra extra_proxy --extra proxy-runtime --extra utils --no-install-project --no-install-package litellm-proxy
uv pip install --no-deps --python .venv/bin/python rust-bridge-dist/*.whl
cp rust-bridge-dist/litellm/rust_bridge/_native.abi3.so litellm/rust_bridge/_native.abi3.so
uv run --no-sync python -c "import importlib.metadata; import litellm.rust_bridge._native; print(importlib.metadata.version('litellm'))"
@@ -1281,7 +1294,7 @@ jobs:
- name: Install dependencies
if: steps.changes.outputs.decision != 'skip'
run: |
- .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router
+ .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra switchyard
- name: Cache Prisma binaries
if: steps.changes.outputs.decision != 'skip'
@@ -1484,9 +1497,9 @@ jobs:
RUST_BRIDGE_ARTIFACT: ${{ needs.rust-bridge.outputs.artifact }}
run: |
if [ -z "$RUST_BRIDGE_ARTIFACT" ]; then
- .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra extra_proxy --extra proxy-runtime --extra utils
+ .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra switchyard --extra saml --extra extra_proxy --extra proxy-runtime --extra utils
else
- .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra extra_proxy --extra proxy-runtime --extra utils --no-install-project
+ .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra switchyard --extra saml --extra extra_proxy --extra proxy-runtime --extra utils --no-install-project --no-install-package litellm-proxy
uv pip install --no-deps --python .venv/bin/python rust-bridge-dist/*.whl
cp rust-bridge-dist/litellm/rust_bridge/_native.abi3.so litellm/rust_bridge/_native.abi3.so
fi
diff --git a/AGENTS.md b/AGENTS.md
index 9aa9ab32528..d8de447be90 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -45,14 +45,13 @@ Run QA proxies with `python litellm/proxy/proxy_cli.py --config litellm/proxy/de
Simple PRs may skip before/after e2e when the full diff is mechanical and focused checks completely prove it, such as #49030 restoring an existing helper name. Trace affected callers and run relevant tests/static checks at the tip. For a bug fix, establish that the regression catches the original defect. Size alone never qualifies. New behavior, API/provider semantics, auth, billing, persistence, concurrency or uncertain integration risk need live e2e. In Screenshots / Proof of Fix, record `E2E exception at `, scope, rationale, commands and results. Tests count as proof only for this exception. Reassess changed diffs, retain CI/bot gates, and honor explicit e2e requests (2026-10-28, unnecessary live QA for #49030)
-For other PRs, proof must show the end user's real flow before and after, with commands and output against a live proxy and real provider calls where applicable. Pytest alone is insufficient. For UI or interactive coding-tool flows, drive that surface and embed before/after screenshots, with URLs, clicks and input fields a reviewer can reproduce
+For other PRs, proof must show the end user's real flow before and after, with commands and output against a live proxy and real provider calls where applicable. The body alone must let a reviewer rerun it, with the proxy command, config, env var names and client code exactly as run, and long blocks collapsed in . Pytest alone is insufficient. For UI or interactive coding-tool flows, drive that surface and embed before/after screenshots, with URLs, clicks and input fields a reviewer can reproduce
If you ever write any human-facing text (pull requests, issues, commit messages, discussion posts, github comments, release notes, docs, etc.), always follow these guidelines to sound less AI-y:
- don't use emojis
- don't use "—". Don't use ";" or ":" unless it would be nonsensical not to. Instead, reach for ",", ".", conjunction words, etc. in descending order of preference: vary among them, weighted toward the front of the list, and skip "," where it would cause a comma splice or the sentence is getting long. Overusing any one of them also feels AI-y. A word cap does not penalize you for adding more sentences
- don't use the pattern "It's not X, it's Y", "You're not X, you're Y", etc.
- unless explicitly asked, don't use bulleted or numbered lists unless it would be nonsensical not to. Instead, prefer prose
-- don't add a trailing "." at the end of paragraphs (just like this file). That means every paragraph, not just the last one (of the markdown file, PR description, GitHub comment, etc.). Rule of thumb: if you're adding new line(s) before the next sentence, don't add a ".". Exception is that docs (i.e., litellm-docs) should have trailing "."
- don't use →. Instead, prefer not to use arrows, and if need be, use -> instead
- use plain, simple, everyday engineering language: the common phrase engineers actually say over rare compact phrasing, in grammatically complete sentences. When explicitly asked to use bullets or ordered lists and structure legitimately helps the reader, prefer nested bullets (any depth is fine) over dense lines in a flat structure
diff --git a/Dockerfile b/Dockerfile
index 4cef63c8d71..94557bcf68c 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -76,6 +76,7 @@ ENV UV_PROJECT_ENVIRONMENT=/app/.venv \
COPY pyproject.toml uv.lock ./
COPY enterprise/pyproject.toml enterprise/
COPY litellm-proxy-extras/pyproject.toml litellm-proxy-extras/
+COPY packaging/litellm-proxy/pyproject.toml packaging/litellm-proxy/
# Install third-party dependencies (cached unless pyproject.toml/uv.lock change)
RUN uv sync --frozen --no-install-project --no-install-workspace --no-default-groups --no-editable \
@@ -101,7 +102,8 @@ COPY --from=ui-builder /ui/out/. litellm/proxy/_experimental/out/
RUN sed -i 's/\r$//' docker/build_admin_ui.sh && chmod +x docker/build_admin_ui.sh && ./docker/build_admin_ui.sh
# Install project and workspace packages (fast - deps already cached)
-RUN uv sync --frozen --no-default-groups --no-editable \
+RUN uv build --package litellm-proxy --wheel --out-dir /wheels
+RUN uv sync --frozen --no-install-package litellm-proxy --no-default-groups --no-editable \
--extra proxy \
--extra proxy-runtime \
--group admin-mcp \
@@ -111,6 +113,8 @@ RUN uv sync --frozen --no-default-groups --no-editable \
--extra bedrock-realtime \
--python python3.13
+RUN uv pip install --python /app/.venv/bin/python --no-deps /wheels/litellm_proxy-*.whl
+
RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \
npm_config_cache=/root/.npm \
prisma generate --schema=./schema.prisma
diff --git a/cookbook/ai_coding_tool_guides/chatgpt_subscription_passthrough.md b/cookbook/ai_coding_tool_guides/chatgpt_subscription_passthrough.md
index e1f606a2eb7..538a5f8a404 100644
--- a/cookbook/ai_coding_tool_guides/chatgpt_subscription_passthrough.md
+++ b/cookbook/ai_coding_tool_guides/chatgpt_subscription_passthrough.md
@@ -46,7 +46,43 @@ For a managed workspace rollout, use OpenAI's [gateway configuration guide](http
## Native compatibility and Pi
-For a native Codex model catalog and request format, change Codex's `base_url` to `https://litellm.example.com/backend-api/codex`. This route uses the same credentials but does not need a model deployment or resolve model aliases. Restricted keys must allow the upstream model name.
+For a native Codex model catalog and request format, change Codex's `base_url` to `https://litellm.example.com/backend-api/codex`. This route uses the same credentials. A ChatGPT model keeps its upstream name and needs no deployment of its own.
+
+### Mix ChatGPT and LiteLLM models
+
+On the native route, Codex's model picker shows OpenAI's catalog first and then the other chat and responses models the virtual key can use. Pick one of those and Codex's requests go through LiteLLM's `/v1/responses` with the key's usual routing, budgets, guardrails, and logging. The proxy removes the ChatGPT bearer and account header from that request, so the other provider and your callbacks never receive them. A name is routed this way when another provider serves it as an exact `model_name`, through a router `model_group_alias`, or as a team's public model name. A ChatGPT model, a name that only a wildcard deployment matches, or a name OpenAI's catalog lists for the caller's ChatGPT account keeps the passthrough, so a model OpenAI adds later still uses the subscription even when a deployment of another provider has the same name. The proxy learns the account's list from the catalog Codex fetches, or asks OpenAI with the caller's login. When it cannot get the list, for example because OpenAI is down, the request times out, or the client is not Codex, every request keeps the passthrough for the next minute, as it did before LiteLLM models were added to the picker. When the key can use many models, the picker lists the ones that fit in Codex's catalog size limit, and the proxy logs a warning with how many it left out.
+
+Codex's picker sends a `none` reasoning effort for a LiteLLM model whose catalog entry lists no effort levels. With `litellm_settings: drop_params: true`, the proxy drops it for a model that does not support `none`, and the provider applies its own default. A model whose cost map entry or `model_info` sets `supports_none_reasoning_effort: true` keeps it. Without `drop_params`, the effort reaches the provider unchanged.
+
+Codex also sends its ChatGPT app tools with every turn, which on a LiteLLM model adds a few hundred thousand prompt tokens to each turn. Set `apps = false` under `[features]` in Codex's `config.toml` to turn them off.
+
+Only a key that may call `/v1/responses` gets these models. A key whose `allowed_routes` covers only the native routes sees OpenAI's catalog unchanged, and every request it sends uses the subscription.
+
+Codex sends earlier reasoning items back with each turn. The proxy tags the reasoning that a LiteLLM model returns on this route, OpenAI and Azure deployments included. When a session switches to a ChatGPT model, the proxy drops those items before forwarding the request to OpenAI, which would reject them. Messages, tool calls, and ChatGPT's own reasoning are kept, and a LiteLLM model still gets its own reasoning back.
+
+When the catalog adds LiteLLM models for a key, its `ETag` is a digest of OpenAI's ETag and those models, and that key's responses carry the same value in `X-Models-Etag`. Codex keeps its cached catalog while the two match and fetches it again when OpenAI's catalog or the key's models change. A key whose catalog adds no LiteLLM models keeps OpenAI's ETags.
+
+### Restrict native models with an access group
+
+A key with a model allowlist needs a rule for ChatGPT names that OpenAI adds over time. Put the subscription wildcards in an access group under `chatgpt/` names:
+
+```yaml
+model_list:
+ - model_name: chatgpt/gpt-*
+ litellm_params:
+ model: chatgpt/gpt-*
+ model_info:
+ access_groups: ["chatgpt-native"]
+ - model_name: chatgpt/codex-*
+ litellm_params:
+ model: chatgpt/codex-*
+ model_info:
+ access_groups: ["chatgpt-native"]
+```
+
+Then add `chatgpt-native` to the key's models, next to any other model names it should reach. On the native route, a key that holds only the `chatgpt/` form of a name is checked as that form, so a request for `gpt-5.6-sol` is allowed when the key may call `chatgpt/gpt-5.6-sol`. The key can call every ChatGPT model that matches the wildcards, including ones the proxy configuration does not name.
+
+On `/v1` these wildcards match only a request that asks for a `chatgpt/` name, so a request for a plain `gpt-` or `codex-` name still goes where it went before, such as a `*` catch-all deployment. A `/v1` request for a `chatgpt/` name without a ChatGPT login falls back to the proxy's own saved login if it has one, so keep that login unset as described above.
### Configure Pi
@@ -71,7 +107,7 @@ Keep Pi's subscription login as the provider credential. The header value names
Send a prompt and check the completed request in LiteLLM's Logs page. Its gateway user/key and token counts should match the caller. Give each person their own LiteLLM virtual key for separate attribution. The gateway credential and ChatGPT login are independently authenticated; LiteLLM does not establish that both identities belong to the same person.
-The native routes are `GET /backend-api/codex/models` and `POST /backend-api/codex/responses`. Model requests use the upstream model name, so a restricted LiteLLM key must allow that name. OpenAI's model catalog is returned in its native format; seeing a model there does not override gateway permissions.
+The native routes are `GET /backend-api/codex/models` and `POST /backend-api/codex/responses`. Model requests use the upstream model name, so a restricted LiteLLM key needs the `chatgpt-native` access group above or that exact name. OpenAI's model catalog is returned in its native format; seeing a model there does not override gateway permissions.
The proxy forwards the ChatGPT bearer and account header to the fixed OpenAI Codex endpoint. It removes gateway credentials and cookies from the upstream request and redacts credential headers in logging callbacks. The client remains responsible for refresh and reauthentication. Native passthrough returns authentication and quota errors without switching subscriptions. Standard model routes retain the caller login across ChatGPT retries; explicitly configured cross-provider fallbacks use their own credentials. An explicit workspace header is preserved for OpenAI to validate.
diff --git a/docker/Dockerfile.database b/docker/Dockerfile.database
index 20f810686a3..2ebffc50b5e 100644
--- a/docker/Dockerfile.database
+++ b/docker/Dockerfile.database
@@ -74,6 +74,7 @@ ENV UV_PROJECT_ENVIRONMENT=/app/.venv \
COPY pyproject.toml uv.lock ./
COPY enterprise/pyproject.toml enterprise/
COPY litellm-proxy-extras/pyproject.toml litellm-proxy-extras/
+COPY packaging/litellm-proxy/pyproject.toml packaging/litellm-proxy/
# Install third-party dependencies (cached unless pyproject.toml/uv.lock change)
RUN uv sync --frozen --no-install-project --no-install-workspace --no-default-groups --no-editable \
@@ -99,7 +100,8 @@ COPY --from=ui-builder /ui/out/. litellm/proxy/_experimental/out/
RUN sed -i 's/\r$//' docker/build_admin_ui.sh && chmod +x docker/build_admin_ui.sh && ./docker/build_admin_ui.sh
# Install project and workspace packages (fast - deps already cached)
-RUN uv sync --frozen --no-default-groups --no-editable \
+RUN uv build --package litellm-proxy --wheel --out-dir /wheels
+RUN uv sync --frozen --no-install-package litellm-proxy --no-default-groups --no-editable \
--extra proxy \
--extra proxy-runtime \
--group admin-mcp \
@@ -109,6 +111,8 @@ RUN uv sync --frozen --no-default-groups --no-editable \
--extra bedrock-realtime \
--python python3.13
+RUN uv pip install --python /app/.venv/bin/python --no-deps /wheels/litellm_proxy-*.whl
+
RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \
npm_config_cache=/root/.npm \
prisma generate --schema=./schema.prisma
diff --git a/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py b/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py
index a31e926256a..4cfc6d4e081 100644
--- a/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py
+++ b/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py
@@ -8,7 +8,6 @@ import re
from collections.abc import Callable, Mapping, Sequence
from dataclasses import replace as dataclasses_replace
from datetime import datetime, timedelta
-from types import MappingProxyType
from typing import TYPE_CHECKING, Final, List, Literal, Optional, Protocol, Tuple, cast
from pydantic import TypeAdapter, ValidationError
@@ -766,7 +765,6 @@ class CheckBatchCost:
credentials: Final = self.llm_router.registry().credentials(model_id, form="with_provider") or {}
_file_content = await afile_content(
file_id=cast(str, raw_output_file_id),
- _litellm_internal_model_credentials=MappingProxyType(dict(credentials)),
**cast(FileContentCallOptions, credentials),
)
@@ -867,10 +865,7 @@ class CheckBatchCost:
error_file_failed_requests: Final = await count_error_file_failed_requests(
response,
custom_llm_provider=batch_file_provider,
- litellm_params={
- **credentials,
- "_litellm_internal_model_credentials": MappingProxyType(dict(credentials)),
- },
+ litellm_params=credentials,
)
batch_result: Final = (
output_file_result
@@ -1029,11 +1024,7 @@ class CheckBatchCost:
custom_llm_provider=deployment_info.litellm_params.custom_llm_provider,
)
cost_provider: Final = cast(BatchFileProvider, llm_provider)
- credentials: Final = self.llm_router.registry().credentials(model_id, form="with_provider") or {}
- file_access: Final = {
- **credentials,
- "_litellm_internal_model_credentials": MappingProxyType(dict(credentials)),
- }
+ file_access: Final = self.llm_router.registry().credentials(model_id, form="with_provider") or {}
output_content: Final = (
await fetch_batch_managed_file_content(
response.output_file_id, custom_llm_provider=cost_provider, litellm_params=file_access
diff --git a/enterprise/litellm_enterprise/proxy/hooks/managed_files.py b/enterprise/litellm_enterprise/proxy/hooks/managed_files.py
index 91c5e2c5f39..aac4469be20 100644
--- a/enterprise/litellm_enterprise/proxy/hooks/managed_files.py
+++ b/enterprise/litellm_enterprise/proxy/hooks/managed_files.py
@@ -127,7 +127,6 @@ class _ManagedFileRetrieve(Protocol):
self,
*,
file_id: str,
- _litellm_internal_model_credentials: Mapping[str, object] | None = None,
**kwargs: Unpack[FileRetrieveCallOptions],
) -> OpenAIFileObject: ...
@@ -400,7 +399,7 @@ class _RouterFileCallKwargs(TypedDict, total=False):
_MODEL_FILE_ID_MAPPING: Final = TypeAdapter(dict[str, str])
_FILE_UPLOAD_ONLY_FIELDS: Final = frozenset({"file", "purpose", "expires_after", "extra_body"})
-_FILE_DELETE_RESERVED_FIELDS: Final = frozenset({"model", "file_id", "_litellm_internal_model_credentials"})
+_FILE_DELETE_RESERVED_FIELDS: Final = frozenset({"model", "file_id"})
@dataclass(frozen=True, slots=True)
@@ -459,15 +458,7 @@ async def _delete_provider_file(
provider_file_id: str,
request_data: Mapping[str, object],
) -> None:
- credentials: Final = llm_router.registry().credentials(model_id, form="with_provider")
- delete_data: Final = {
- **{k: v for k, v in request_data.items() if k not in _FILE_DELETE_RESERVED_FIELDS},
- **(
- {"_litellm_internal_model_credentials": MappingProxyType(dict(credentials))}
- if credentials is not None
- else {}
- ),
- }
+ delete_data: Final = {k: v for k, v in request_data.items() if k not in _FILE_DELETE_RESERVED_FIELDS}
await llm_router.afile_delete(
model=model_id,
file_id=provider_file_id,
@@ -1160,7 +1151,6 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
*,
provider_file_id: str,
call_options: Mapping[str, object],
- internal_model_credentials: Mapping[str, object] | None = None,
) -> OpenAIFileObject:
"""Retrieve provider file details with transient retries and SDK retries disabled."""
retrieve_file: Final = cast(_ManagedFileRetrieve, litellm.afile_retrieve)
@@ -1172,14 +1162,8 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
if attempt > 0:
await self._sleep(_PROVIDER_FILE_RETRIEVE_RETRY_DELAYS_SECONDS[attempt - 1])
try:
- if internal_model_credentials is None:
- return await retrieve_file(
- file_id=provider_file_id,
- **retrieve_options,
- )
return await retrieve_file(
file_id=provider_file_id,
- _litellm_internal_model_credentials=MappingProxyType(dict(internal_model_credentials)),
**retrieve_options,
)
except Exception as error:
@@ -1221,7 +1205,6 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
self._afile_retrieve_with_retries(
provider_file_id=provider_file_id,
call_options=router_credentials,
- internal_model_credentials=router_credentials,
),
timeout=BATCH_OUTPUT_FILE_LOOKUP_TIMEOUT_SECONDS,
)
@@ -2866,15 +2849,6 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
exception_dict = {}
for model_id, provider_file_id in specific_model_file_id_mapping.items():
try:
- # Cloud-storage providers (e.g. Bedrock S3) validate file ids
- # against the deployment's configured bucket, which they only
- # trust from this immutable server-side snapshot, never from
- # request params.
- credentials = llm_router.registry().credentials(model_id, form="with_provider")
- if credentials is not None:
- data["_litellm_internal_model_credentials"] = cast(Dict, MappingProxyType(dict(credentials)))
- else:
- data.pop("_litellm_internal_model_credentials", None)
return cast(
HttpxBinaryResponseContent,
await llm_router.afile_content(
diff --git a/enterprise/pyproject.toml b/enterprise/pyproject.toml
index 7db2de1f874..64fe03769b8 100644
--- a/enterprise/pyproject.toml
+++ b/enterprise/pyproject.toml
@@ -1,6 +1,6 @@
[project]
name = "litellm-enterprise"
-version = "0.1.81"
+version = "0.1.83"
description = "Package for LiteLLM Enterprise features"
readme = "README.md"
requires-python = ">=3.9"
@@ -26,7 +26,7 @@ required-version = ">=0.10.9"
module-root = ""
[tool.commitizen]
-version = "0.1.81"
+version = "0.1.83"
version_files = [
"pyproject.toml:^version",
"../pyproject.toml:litellm-enterprise==",
diff --git a/gateway/Dockerfile b/gateway/Dockerfile
index 658bed831ba..1e1c7d73091 100644
--- a/gateway/Dockerfile
+++ b/gateway/Dockerfile
@@ -41,10 +41,14 @@ COPY pyproject.toml README.md rust-toolchain.toml ./
COPY .cargo ./.cargo
COPY litellm-rust ./litellm-rust
COPY model_prices_and_context_window.json ./
+COPY packaging/litellm-proxy/pyproject.toml packaging/litellm-proxy/
RUN --mount=type=cache,target=/root/.cache/uv \
mkdir -p litellm/rust_bridge /wheels && \
touch litellm/__init__.py litellm/rust_bridge/__init__.py && \
+ uvx --python python3.13 --from maturin==1.15.0 --with puccinialin==0.1.14 \
+ python -c "import maturin; maturin.build_wheel('/wheels')" && \
+ cd packaging/litellm-proxy && \
uvx --python python3.13 --from maturin==1.15.0 --with puccinialin==0.1.14 \
python -c "import maturin; maturin.build_wheel('/wheels')"
@@ -85,6 +89,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \
--mount=type=bind,source=uv.lock,target=uv.lock \
--mount=type=bind,source=enterprise/pyproject.toml,target=enterprise/pyproject.toml \
--mount=type=bind,source=litellm-proxy-extras/pyproject.toml,target=litellm-proxy-extras/pyproject.toml \
+ --mount=type=bind,source=packaging/litellm-proxy/pyproject.toml,target=packaging/litellm-proxy/pyproject.toml \
uv sync --frozen --no-install-project --no-install-workspace --no-default-groups --no-editable \
--extra proxy \
--extra proxy-runtime \
@@ -98,7 +103,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \
COPY . .
RUN --mount=type=cache,target=/root/.cache/uv \
- uv sync --frozen --no-install-project --no-default-groups --no-editable \
+ uv sync --frozen --no-install-project --no-install-package litellm-proxy --no-default-groups --no-editable \
--extra proxy \
--extra proxy-runtime \
--group admin-mcp \
@@ -108,7 +113,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \
--python python3.13
RUN --mount=type=bind,from=rust-native,source=/wheels,target=/wheels \
- uv pip install --python /app/.venv/bin/python --no-deps /wheels/litellm-*.whl
+ uv pip install --python /app/.venv/bin/python --no-deps /wheels/litellm-*.whl /wheels/litellm_proxy-*.whl
# PYTHONPATH=/app makes the source tree shadow the installed package, so the
# compiled Rust extension must live next to the source or it is never imported.
diff --git a/litellm-proxy-extras/pyproject.toml b/litellm-proxy-extras/pyproject.toml
index a12e1e27cfc..1c0c708ae7a 100644
--- a/litellm-proxy-extras/pyproject.toml
+++ b/litellm-proxy-extras/pyproject.toml
@@ -1,6 +1,6 @@
[project]
name = "litellm-proxy-extras"
-version = "0.4.112"
+version = "0.4.114"
description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
readme = "README.md"
requires-python = ">=3.9"
@@ -30,7 +30,7 @@ required-version = ">=0.10.9"
module-root = ""
[tool.commitizen]
-version = "0.4.112"
+version = "0.4.114"
version_files = [
"pyproject.toml:^version",
"../pyproject.toml:litellm-proxy-extras==",
diff --git a/litellm-rust/.agents/skills/rust-gateway/SKILL.md b/litellm-rust/.agents/skills/rust-gateway/SKILL.md
index 4fee0b54a68..923c9bf5ab6 100644
--- a/litellm-rust/.agents/skills/rust-gateway/SKILL.md
+++ b/litellm-rust/.agents/skills/rust-gateway/SKILL.md
@@ -7,7 +7,7 @@ metadata:
# Rust gateway
-`crates/gateway` mounts endpoint routers and serves the listener. `crates/gateway-inference`, `gateway-mcp` and `gateway-ui` own their endpoints, `gateway-auth` is the inbound authentication middleware and `gateway-management` holds management logic that has no router yet. A crate's `AGENTS.md`, where it has one, says what belongs in it. Read the reference for the part you are touching
+`crates/gateway` mounts endpoint routers and serves the listener. `crates/gateway-inference`, `gateway-mcp` and `gateway-ui` own their endpoints, `gateway-auth` is the inbound authentication middleware and `gateway-management` holds management logic that has no router yet. `gateway-forward` is the reverse proxy that relays every request to the Python proxy and has no routes of its own. A crate's `AGENTS.md`, where it has one, says what belongs in it. Read the reference for the part you are touching
- [surfaces](references/surfaces.md)
- Each API surface mirrors the envelope the Python proxy returns there, so the shapes differ on purpose
diff --git a/litellm-rust/.agents/skills/rust-python-boundary/SKILL.md b/litellm-rust/.agents/skills/rust-python-boundary/SKILL.md
new file mode 100644
index 00000000000..5ead8f287ab
--- /dev/null
+++ b/litellm-rust/.agents/skills/rust-python-boundary/SKILL.md
@@ -0,0 +1,53 @@
+---
+name: rust-python-boundary
+description: Decide where Python-shaped input becomes typed Rust in litellm-rust. Use when reading config.yaml values, os.environ/ references, REDIS_* or other environment layering, get_secret fallbacks, or kwargs and objects handed over from Python, when coercing text to a bool, int or float the way Python does, or when adding a dependency on litellm-python-compat
+metadata:
+ internal: true
+---
+
+# Python boundary
+
+Python-shaped input becomes typed Rust in exactly two crates. Every crate past them handles typed values, so no crate re-guesses how Python would have read a value
+
+## The two boundaries
+
+- `crates/config` reads config.yaml, which is Python-shaped: `os.environ/` references, scalars that Helm or the environment turned into text, raw kwargs maps. It also owns the environment layering and `get_secret` fallbacks Python applies while it loads that config. It has no Python runtime, so it reproduces Python value semantics with `litellm-python-compat`
+- `crates/python-bridge` receives live Python objects, such as `RedisCache.redis_kwargs`. It reads them through PyO3 with the real protocol (`is_truthy()`, `str()`, `extract`), so it never imitates Python
+- Both produce the same typed input, defined in the data crate the consumer already depends on (`redis-types` for Redis), so a downstream crate cannot tell which host it runs under
+
+## Python value semantics
+
+Reading a value the way Python itself reads it. This lives only at the two boundaries
+
+- Truthiness and `x or fallback`, `str(x)`, `int(text)` and `float(text)`
+- Each surface's true and false text: `_str_to_bool`, pydantic's lax bool, `get_secret`'s `true` and `false`
+- `os.environ/` references and `REDIS_` style environment layering
+
+A boundary knows which surface a value came from. `general_settings.coordination_redis` is a pydantic model and `cache_params` goes through `_str_to_bool`, so the boundary applies that surface's rule exactly instead of refusing the text two surfaces disagree on
+
+Each typed field keeps where its value came from (the key, `REDIS_` or the fallback name), so an error or warning downstream still names it
+
+## What stays downstream
+
+A crate that stands in for a Python library keeps that library's semantics, because they are its compatibility contract and not value coercion. `redis-resolve` stands in for redis-py, so it keeps
+
+- Which target wins, and the defaults redis-py applies
+- How `parse_url` reads a url, including `int()` on `?db=` and redis-py's `to_bool` on the TLS query parameters
+- Which kwargs redis-py accepts, and whether the Rust client reads, warns about or refuses each one
+
+Its input is typed. A rule it applies reads typed fields, never a `serde_json::Value` it has to coerce first. A value it only carries back to Python untouched, like `passthrough`, may stay untyped
+
+## litellm-python-compat
+
+The rule is about where config and kwargs get coerced, not about who may depend on the crate
+
+- Its data formats (`pickle`, `json.dumps`, `repr`) are fine anywhere Rust reads or writes state Python also reads or writes, such as cache values in a shared Redis
+- Its coercions (`truthy`, `number`) belong at the two boundaries, and in a crate that stands in for a Python library when that library calls them itself
+
+## Existing code
+
+Some code still coerces Python-shaped values past the boundary, for example the `REDIS_*` merge in `gateway/src/redis/layers.rs` and the flag and number readers in `redis-resolve/src/options.rs`. Do not add more. When you change one, move the coercion to its boundary instead of extending it in place
+
+## Tests
+
+Pin a boundary's coercion with named cases next to the Python function it mirrors, in the Rust test and in its Python twin, like the Redis contract cases in `redis-resolve/tests/interpret.rs` and `tests/unit/test_redis.py`. A downstream crate's tests build its typed input directly and never feed it raw text to coerce
diff --git a/litellm-rust/AGENTS.md b/litellm-rust/AGENTS.md
index 5eaf876e3b2..5fcf05f61f7 100644
--- a/litellm-rust/AGENTS.md
+++ b/litellm-rust/AGENTS.md
@@ -8,6 +8,10 @@ For string-valued enums and their Serde conversions, follow [.agents/skills/rust
For error types, error variants and conversions between crate errors, follow [.agents/skills/rust-error/SKILL.md](.agents/skills/rust-error/SKILL.md)
+For where config.yaml values, environment layering and kwargs from Python become typed Rust, follow [.agents/skills/rust-python-boundary/SKILL.md](.agents/skills/rust-python-boundary/SKILL.md)
+
+The process-wide rustls `CryptoProvider` (ring) is installed once by the host. rust-proxy does it at the top of `main` through `litellm_gateway::install_crypto_provider`, and python-bridge must do the same in its module init before it builds any rustls config. Library crates never install it. The AWS SDK links aws-lc-rs next to ring, so without an installed default the first rustls config a library builds, such as redis-rs TLS, panics
+
For fieldless enums, derive `strum::VariantArray` and use `VARIANTS` instead of a hand-listed `ALL` array. Derive Strum string conversions instead of hand-written variant-to-string matches
Use the derived conversions directly (`<&'static str>::from(x)` / `.into()`, `str::parse`) with no `as_str`/`parse` wrapper that only delegates, and spell each variant with explicit `#[strum(serialize = "...")]` instead of `serialize_all`
diff --git a/litellm-rust/Cargo.lock b/litellm-rust/Cargo.lock
index 23ebfc7d3cf..941887b249c 100644
--- a/litellm-rust/Cargo.lock
+++ b/litellm-rust/Cargo.lock
@@ -8,6 +8,17 @@ version = "2.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa"
+[[package]]
+name = "aes"
+version = "0.8.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0"
+dependencies = [
+ "cfg-if",
+ "cipher",
+ "cpufeatures 0.2.17",
+]
+
[[package]]
name = "ahash"
version = "0.8.12"
@@ -547,7 +558,7 @@ dependencies = [
"http 1.4.2",
"http-body 0.4.6",
"hyper 0.14.32",
- "hyper 1.10.1",
+ "hyper 1.12.0",
"hyper-rustls 0.24.2",
"hyper-rustls 0.27.9",
"hyper-util",
@@ -736,7 +747,7 @@ dependencies = [
"http 1.4.2",
"http-body 1.1.0",
"http-body-util",
- "hyper 1.10.1",
+ "hyper 1.12.0",
"hyper-util",
"itoa",
"matchit",
@@ -982,6 +993,15 @@ dependencies = [
"hybrid-array",
]
+[[package]]
+name = "block-padding"
+version = "0.3.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a8894febbff9f758034a5b8e12d87918f56dfc64a8e1fe757d65e29041538d93"
+dependencies = [
+ "generic-array",
+]
+
[[package]]
name = "bollard"
version = "0.20.2"
@@ -1000,7 +1020,7 @@ dependencies = [
"home",
"http 1.4.2",
"http-body-util",
- "hyper 1.10.1",
+ "hyper 1.12.0",
"hyper-named-pipe",
"hyper-rustls 0.27.9",
"hyper-util",
@@ -1134,6 +1154,15 @@ dependencies = [
"rustversion",
]
+[[package]]
+name = "cbc"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "26b52a9543ae338f279b96b0b9fed9c8093744685043739079ce85cd58f289a6"
+dependencies = [
+ "cipher",
+]
+
[[package]]
name = "cc"
version = "1.3.0"
@@ -1210,6 +1239,16 @@ dependencies = [
"half",
]
+[[package]]
+name = "cipher"
+version = "0.4.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad"
+dependencies = [
+ "crypto-common 0.1.7",
+ "inout",
+]
+
[[package]]
name = "clap"
version = "4.6.7"
@@ -1774,6 +1813,15 @@ dependencies = [
"syn 2.0.119",
]
+[[package]]
+name = "des"
+version = "0.8.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ffdd80ce8ce993de27e9f063a444a4d53ce8e8db4c1f00cc03af5ad5a9867a1e"
+dependencies = [
+ "cipher",
+]
+
[[package]]
name = "digest"
version = "0.10.7"
@@ -2255,7 +2303,7 @@ dependencies = [
"chrono",
"http 1.4.2",
"http-body-util",
- "hyper 1.10.1",
+ "hyper 1.12.0",
"hyper-rustls 0.27.9",
"hyper-util",
"ring",
@@ -2391,7 +2439,7 @@ dependencies = [
"http 1.4.2",
"http-body 1.1.0",
"http-body-util",
- "hyper 1.10.1",
+ "hyper 1.12.0",
"lazy_static",
"opentelemetry",
"opentelemetry-semantic-conventions",
@@ -2833,9 +2881,9 @@ dependencies = [
[[package]]
name = "hyper"
-version = "1.10.1"
+version = "1.12.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "55281c53a1894c864990125767da440a4e630446785086f52523b20033b74498"
+checksum = "2c3e324da4c95177d6291d4c8730197c0d1822f8a9766814a4a44fa5ab797c9c"
dependencies = [
"atomic-waker",
"bytes",
@@ -2860,7 +2908,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fab3637d6b04a8037af8a266fdf6cf92ea957e8c53981a2bf6136572531025bf"
dependencies = [
"hex",
- "hyper 1.10.1",
+ "hyper 1.12.0",
"hyper-util",
"pin-project-lite",
"tokio",
@@ -2889,7 +2937,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f"
dependencies = [
"http 1.4.2",
- "hyper 1.10.1",
+ "hyper 1.12.0",
"hyper-util",
"rustls 0.23.42",
"rustls-native-certs",
@@ -2904,7 +2952,7 @@ version = "0.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2b90d566bffbce6a75bd8b09a05aa8c2cb1fabb6cb348f8840c9e4c90a0d83b0"
dependencies = [
- "hyper 1.10.1",
+ "hyper 1.12.0",
"hyper-util",
"pin-project-lite",
"tokio",
@@ -2923,7 +2971,7 @@ dependencies = [
"futures-util",
"http 1.4.2",
"http-body 1.1.0",
- "hyper 1.10.1",
+ "hyper 1.12.0",
"ipnet",
"libc",
"percent-encoding",
@@ -2942,7 +2990,7 @@ checksum = "986c5ce3b994526b3cd75578e62554abd09f0899d6206de48b3e96ab34ccc8c7"
dependencies = [
"hex",
"http-body-util",
- "hyper 1.10.1",
+ "hyper 1.12.0",
"hyper-util",
"pin-project-lite",
"tokio",
@@ -3105,6 +3153,16 @@ dependencies = [
"serde_core",
]
+[[package]]
+name = "inout"
+version = "0.1.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01"
+dependencies = [
+ "block-padding",
+ "generic-array",
+]
+
[[package]]
name = "ipnet"
version = "2.12.0"
@@ -3663,8 +3721,7 @@ dependencies = [
"strum",
"tempfile",
"thiserror 2.0.19",
- "tracing",
- "tracing-subscriber",
+ "veil",
]
[[package]]
@@ -3767,8 +3824,10 @@ dependencies = [
"http-body-util",
"litellm-circuit-breaker",
"litellm-config",
+ "litellm-core-utils",
"litellm-error",
"litellm-gateway-auth",
+ "litellm-gateway-forward",
"litellm-gateway-inference",
"litellm-gateway-mcp",
"litellm-gateway-ui",
@@ -3776,11 +3835,15 @@ dependencies = [
"litellm-inference",
"litellm-llms",
"litellm-redis-client",
+ "litellm-redis-connect",
+ "litellm-redis-resolve",
"litellm-redis-testing",
+ "litellm-redis-types",
"litellm-secret-value",
"litellm-secrets",
"litellm-tracing",
"rstest",
+ "rustls 0.23.42",
"serde",
"serde_json",
"tempfile",
@@ -3791,6 +3854,7 @@ dependencies = [
"tower-sessions-moka-store",
"tracing",
"uuid",
+ "veil",
]
[[package]]
@@ -3815,6 +3879,29 @@ dependencies = [
"veil",
]
+[[package]]
+name = "litellm-gateway-forward"
+version = "0.1.0"
+dependencies = [
+ "axum",
+ "bytes",
+ "envy",
+ "futures-util",
+ "http-body-util",
+ "httparse",
+ "hyper 1.12.0",
+ "hyper-util",
+ "proptest",
+ "rstest",
+ "serde",
+ "thiserror 2.0.19",
+ "tokio",
+ "tokio-tungstenite",
+ "tokio-util",
+ "tower-service",
+ "tracing",
+]
+
[[package]]
name = "litellm-gateway-inference"
version = "0.1.0"
@@ -4294,11 +4381,13 @@ dependencies = [
"criterion",
"fancy-regex 0.19.2",
"futures-util",
+ "indexmap 2.14.0",
"litellm-auth",
"litellm-auth-aws",
"litellm-cache",
"litellm-cache-response",
"litellm-callbacks-legacy-python",
+ "litellm-circuit-breaker",
"litellm-core-utils",
"litellm-error",
"litellm-host",
@@ -4313,6 +4402,12 @@ dependencies = [
"litellm-llms",
"litellm-llms-types",
"litellm-model-catalog",
+ "litellm-python-compat",
+ "litellm-redis-client",
+ "litellm-redis-connect",
+ "litellm-redis-resolve",
+ "litellm-redis-store",
+ "litellm-redis-types",
"litellm-router-core",
"litellm-router-types",
"litellm-secret-value",
@@ -4363,23 +4458,108 @@ version = "0.1.0"
dependencies = [
"futures-util",
"litellm-circuit-breaker",
+ "litellm-python-compat",
"litellm-redis-testing",
+ "litellm-redis-types",
"litellm-secret-value",
+ "pem",
+ "pkcs8",
"redis",
"rstest",
+ "rustls 0.23.42",
+ "rustls-native-certs",
+ "socket2 0.6.5",
"thiserror 2.0.19",
"tokio",
+ "tracing",
+]
+
+[[package]]
+name = "litellm-redis-connect"
+version = "0.1.0"
+dependencies = [
+ "futures-util",
+ "litellm-auth-aws",
+ "litellm-auth-azure",
+ "litellm-auth-gcp",
+ "litellm-auth-types",
+ "litellm-circuit-breaker",
+ "litellm-core-utils",
+ "litellm-error",
+ "litellm-http",
+ "litellm-redis-client",
+ "litellm-redis-resolve",
+ "litellm-redis-testing",
+ "litellm-redis-types",
+ "litellm-secret-value",
+ "rstest",
+ "rustls 0.23.42",
+ "serde",
+ "serde_json",
+ "strum",
+ "thiserror 2.0.19",
+ "time",
+ "tokio",
+ "tracing",
+ "url",
+ "veil",
+]
+
+[[package]]
+name = "litellm-redis-resolve"
+version = "0.1.0"
+dependencies = [
+ "litellm-python-compat",
+ "litellm-redis-types",
+ "litellm-secret-value",
+ "rstest",
+ "serde",
+ "serde_json",
+ "strum",
+ "thiserror 2.0.19",
+ "tracing",
+ "url",
+]
+
+[[package]]
+name = "litellm-redis-store"
+version = "0.1.0"
+dependencies = [
+ "futures-util",
+ "litellm-cache",
+ "litellm-cache-memory",
+ "litellm-circuit-breaker",
+ "litellm-python-compat",
+ "litellm-redis-client",
+ "litellm-redis-testing",
+ "litellm-secret-value",
+ "litellm-tracing",
+ "rstest",
+ "strum",
+ "thiserror 2.0.19",
+ "tokio",
+ "tracing",
]
[[package]]
name = "litellm-redis-testing"
version = "0.1.0"
dependencies = [
+ "pkcs8",
+ "rcgen",
+ "tempfile",
"testcontainers-modules",
"thiserror 2.0.19",
"tokio",
]
+[[package]]
+name = "litellm-redis-types"
+version = "0.1.0"
+dependencies = [
+ "litellm-secret-value",
+]
+
[[package]]
name = "litellm-router"
version = "0.1.0"
@@ -5262,6 +5442,16 @@ version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2ee67f1008b1ba2321834326597b8e186293b049a023cdef258527550b9935b4"
+[[package]]
+name = "pbkdf2"
+version = "0.12.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f8ed6a7761f76e3b9f92dfb0a60a6a6477c61024b775147ff0973a02653abaf2"
+dependencies = [
+ "digest 0.10.7",
+ "hmac 0.12.1",
+]
+
[[package]]
name = "pem"
version = "4.0.0"
@@ -5372,6 +5562,22 @@ dependencies = [
"spki",
]
+[[package]]
+name = "pkcs5"
+version = "0.7.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e847e2c91a18bfa887dd028ec33f2fe6f25db77db3619024764914affe8b69a6"
+dependencies = [
+ "aes",
+ "cbc",
+ "der",
+ "des",
+ "pbkdf2",
+ "scrypt",
+ "sha2 0.10.9",
+ "spki",
+]
+
[[package]]
name = "pkcs8"
version = "0.10.2"
@@ -5379,6 +5585,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7"
dependencies = [
"der",
+ "pkcs5",
+ "rand_core 0.6.4",
"spki",
]
@@ -5566,6 +5774,7 @@ version = "0.29.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4688ddedf473e32662b9b067670129a8afb8c18e351482c70d62ba4a88171e8b"
dependencies = [
+ "indexmap 2.14.0",
"libc",
"once_cell",
"portable-atomic",
@@ -6031,7 +6240,7 @@ dependencies = [
"http 1.4.2",
"http-body 1.1.0",
"http-body-util",
- "hyper 1.10.1",
+ "hyper 1.12.0",
"hyper-rustls 0.27.9",
"hyper-util",
"js-sys",
@@ -6387,6 +6596,15 @@ version = "1.0.23"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f"
+[[package]]
+name = "salsa20"
+version = "0.10.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "97a22f5af31f73a954c10289c93e8a50cc23d971e80ee446f1f6f7137a088213"
+dependencies = [
+ "cipher",
+]
+
[[package]]
name = "same-file"
version = "1.0.6"
@@ -6450,6 +6668,17 @@ version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
+[[package]]
+name = "scrypt"
+version = "0.11.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0516a385866c09368f0b5bcd1caff3366aace790fcd46e2bb032697bb172fd1f"
+dependencies = [
+ "pbkdf2",
+ "salsa20",
+ "sha2 0.10.9",
+]
+
[[package]]
name = "sct"
version = "0.7.1"
@@ -7512,6 +7741,7 @@ dependencies = [
"bytes",
"futures-core",
"futures-sink",
+ "futures-util",
"pin-project-lite",
"tokio",
]
@@ -7591,7 +7821,7 @@ dependencies = [
"http 1.4.2",
"http-body 1.1.0",
"http-body-util",
- "hyper 1.10.1",
+ "hyper 1.12.0",
"hyper-timeout",
"hyper-util",
"percent-encoding",
@@ -8167,9 +8397,9 @@ dependencies = [
[[package]]
name = "want"
-version = "0.3.1"
+version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e"
+checksum = "ec4cdd0dd910afe868b7ef477227d8d538b46b3075031afee8a9f2acb0a2ed0b"
dependencies = [
"try-lock",
]
@@ -8551,7 +8781,7 @@ dependencies = [
"futures",
"http 1.4.2",
"http-body-util",
- "hyper 1.10.1",
+ "hyper 1.12.0",
"hyper-util",
"log",
"once_cell",
diff --git a/litellm-rust/Cargo.toml b/litellm-rust/Cargo.toml
index 71914a0bffe..72d6694ba79 100644
--- a/litellm-rust/Cargo.toml
+++ b/litellm-rust/Cargo.toml
@@ -26,6 +26,7 @@ litellm-inference-ocr = { path = "crates/inference-ocr" }
litellm-inference-testing = { path = "crates/inference-testing" }
litellm-gateway-mcp = { path = "crates/gateway-mcp" }
litellm-gateway = { path = "crates/gateway" }
+litellm-gateway-forward = { path = "crates/gateway-forward" }
litellm-gateway-inference = { path = "crates/gateway-inference" }
litellm-gateway-auth = { path = "crates/gateway-auth" }
litellm-gateway-management = { path = "crates/gateway-management" }
@@ -72,7 +73,11 @@ litellm-host-python = { path = "crates/host-python" }
litellm-python-compat = { path = "crates/python-compat" }
litellm-circuit-breaker = { path = "crates/circuit-breaker" }
litellm-redis-client = { path = "crates/redis-client" }
+litellm-redis-connect = { path = "crates/redis-connect" }
+litellm-redis-store = { path = "crates/redis-store" }
litellm-redis-testing = { path = "crates/redis-testing" }
+litellm-redis-resolve = { path = "crates/redis-resolve" }
+litellm-redis-types = { path = "crates/redis-types" }
litellm-cost = { path = "crates/cost" }
litellm-model-catalog = { path = "crates/model-catalog" }
litellm-model-catalog-refresh = { path = "crates/model-catalog-refresh" }
diff --git a/litellm-rust/crates/cache-memory/src/cache.rs b/litellm-rust/crates/cache-memory/src/cache.rs
index 73fbf14b07a..809ffe3044e 100644
--- a/litellm-rust/crates/cache-memory/src/cache.rs
+++ b/litellm-rust/crates/cache-memory/src/cache.rs
@@ -2,7 +2,10 @@ use std::{
cmp::Reverse,
collections::{BinaryHeap, HashMap, HashSet},
hash::Hash,
- sync::{Arc, Mutex, MutexGuard},
+ sync::{
+ Arc, Mutex, MutexGuard,
+ atomic::{AtomicUsize, Ordering},
+ },
time::{Duration, SystemTime, UNIX_EPOCH},
};
@@ -31,7 +34,7 @@ struct CacheState {
pub struct InMemoryCache {
state: Mutex>,
- max_size_in_memory: usize,
+ max_size_in_memory: AtomicUsize,
default_ttl: Duration,
max_entry_bytes: Option,
measure_value: Option>,
@@ -74,7 +77,9 @@ impl InMemoryCache {
expirations: HashMap::new(),
expiration_heap: BinaryHeap::new(),
}),
- max_size_in_memory: max_size_in_memory.unwrap_or(DEFAULT_MAX_SIZE_IN_MEMORY),
+ max_size_in_memory: AtomicUsize::new(
+ max_size_in_memory.unwrap_or(DEFAULT_MAX_SIZE_IN_MEMORY),
+ ),
default_ttl: default_ttl
.filter(|ttl| !ttl.is_zero())
.unwrap_or(DEFAULT_TTL),
@@ -94,7 +99,7 @@ impl InMemoryCache {
value: V,
ttl: Option,
) -> Result {
- if self.max_size_in_memory == 0 {
+ if self.max_size_in_memory() == 0 {
return Ok(CacheWrite::Disabled);
}
let now = (self.now)();
@@ -129,7 +134,7 @@ impl InMemoryCache {
pub fn evict_cache(&self) -> Result<(), Error> {
let now = (self.now)();
let mut state = self.lock()?;
- Self::evict(&mut state, self.max_size_in_memory, now, None);
+ Self::evict(&mut state, self.max_size_in_memory(), now, None);
Ok(())
}
@@ -170,7 +175,16 @@ impl InMemoryCache {
}
pub fn max_size_in_memory(&self) -> usize {
- self.max_size_in_memory
+ self.max_size_in_memory.load(Ordering::Relaxed)
+ }
+
+ /// `DualCache.update_in_memory_max_size`: `None` restores the default. Entries already held stay
+ /// until the next write evicts down to the new size.
+ pub fn update_max_size_in_memory(&self, max_size_in_memory: Option) {
+ self.max_size_in_memory.store(
+ max_size_in_memory.unwrap_or(DEFAULT_MAX_SIZE_IN_MEMORY),
+ Ordering::Relaxed,
+ );
}
pub fn max_entry_bytes(&self) -> Option {
@@ -200,6 +214,23 @@ impl InMemoryCache {
.collect())
}
+ /// Every live entry, for callers that inspect the whole tier.
+ pub fn entries(&self) -> Result, Error> {
+ let now = (self.now)();
+ let state = self.lock()?;
+ Ok(state
+ .values
+ .iter()
+ .filter(|(key, _)| {
+ state
+ .expirations
+ .get(*key)
+ .is_none_or(|expiration| *expiration >= now)
+ })
+ .map(|(key, value)| (key.clone(), value.clone()))
+ .collect())
+ }
+
pub fn delete_cache(&self, key: &str) -> Result<(), Error> {
let mut state = self.lock()?;
Self::remove(&mut state, key);
@@ -276,7 +307,7 @@ impl InMemoryCache {
ttl: Option,
now: Duration,
) -> Result {
- Self::evict(state, self.max_size_in_memory, now, None);
+ Self::evict(state, self.max_size_in_memory(), now, None);
if !self.check_value_size(&value)? {
return Ok(CacheWrite::TooLarge);
}
@@ -300,12 +331,12 @@ where
eligible: &[V],
context: ExactCacheContext,
) -> Result {
- if self.max_size_in_memory == 0 {
+ if self.max_size_in_memory() == 0 {
return Ok(candidate);
}
let now = (self.now)();
let mut state = self.lock()?;
- Self::evict(&mut state, self.max_size_in_memory, now, Some(key));
+ Self::evict(&mut state, self.max_size_in_memory(), now, Some(key));
let existing = state
.values
.get(key)
@@ -335,7 +366,7 @@ impl CounterCache for InMemoryCache {
amount: f64,
context: ExactCacheContext,
) -> Result {
- if self.max_size_in_memory == 0 {
+ if self.max_size_in_memory() == 0 {
return Ok(amount);
}
let now = (self.now)();
@@ -408,7 +439,7 @@ where
values: Vec,
ttl: Option,
) -> Result {
- if self.max_size_in_memory == 0 {
+ if self.max_size_in_memory() == 0 {
return Ok(values);
}
let now = (self.now)();
diff --git a/litellm-rust/crates/config/AGENTS.md b/litellm-rust/crates/config/AGENTS.md
new file mode 100644
index 00000000000..9f6f1db9a67
--- /dev/null
+++ b/litellm-rust/crates/config/AGENTS.md
@@ -0,0 +1,3 @@
+## Redis settings
+
+- This crate only deserializes YAML. `RedisSettings` is the raw `coordination_redis` or `cache_params` map and `CacheSettings` tells a `type: redis` cache apart from the rest. Reading those maps the way `litellm/_redis.py` does belongs to `gateway/src/redis`, and turning merged kwargs into a `Resolution` belongs to `redis-resolve`
diff --git a/litellm-rust/crates/config/Cargo.toml b/litellm-rust/crates/config/Cargo.toml
index 278aea6200e..a433aacc1c9 100644
--- a/litellm-rust/crates/config/Cargo.toml
+++ b/litellm-rust/crates/config/Cargo.toml
@@ -6,17 +6,16 @@ license.workspace = true
repository.workspace = true
[dependencies]
-litellm-secret-value.workspace = true
litellm-router-types.workspace = true
+litellm-secret-value.workspace = true
serde.workspace = true
-serde_json.workspace = true
serde_yaml_ng = "0.10.0"
strum.workspace = true
thiserror.workspace = true
-tracing.workspace = true
+veil.workspace = true
[dev-dependencies]
litellm-error.workspace = true
rstest.workspace = true
+serde_json.workspace = true
tempfile.workspace = true
-tracing-subscriber = { version = "0.3", default-features = false, features = ["registry", "std"] }
diff --git a/litellm-rust/crates/config/src/error.rs b/litellm-rust/crates/config/src/error.rs
index e5219ac992c..300f15380bd 100644
--- a/litellm-rust/crates/config/src/error.rs
+++ b/litellm-rust/crates/config/src/error.rs
@@ -11,11 +11,4 @@ pub enum Error {
path: PathBuf,
source: serde_yaml_ng::Error,
},
- #[error("Redis host does not yet support configured setting {0}")]
- RedisSetting(String),
- #[error("invalid Redis setting {setting}: {reason}")]
- RedisInvalid {
- setting: &'static str,
- reason: &'static str,
- },
}
diff --git a/litellm-rust/crates/config/src/lib.rs b/litellm-rust/crates/config/src/lib.rs
index b8af6f11d2e..46657aa91e7 100644
--- a/litellm-rust/crates/config/src/lib.rs
+++ b/litellm-rust/crates/config/src/lib.rs
@@ -14,10 +14,7 @@ pub use error::Error;
pub use litellm_router_types::{LitellmParams, Spelled};
pub use mcp::{McpAuth, McpServer, McpTransport};
pub use model::Model;
-pub use redis::{
- CacheSettings, Lookup, RedisCluster, RedisEndpoint, RedisHost, RedisNode, RedisSettings,
- ResolvedRedis, environment_has_target,
-};
+pub use redis::{CacheSettings, RedisSettings};
pub use settings::{
ClickHouseStoreSettings, GeneralSettings, LiteLlmSettings, RouterSettings, TracingSettings,
TracingStoreSettings,
diff --git a/litellm-rust/crates/config/src/redis.rs b/litellm-rust/crates/config/src/redis.rs
index 7a91691c54b..d0ba107242c 100644
--- a/litellm-rust/crates/config/src/redis.rs
+++ b/litellm-rust/crates/config/src/redis.rs
@@ -1,195 +1,21 @@
-use std::{collections::BTreeMap, fmt, time::Duration};
+use std::collections::BTreeMap;
-use litellm_router_types::Spelled;
-use litellm_secret_value::SecretValue;
use serde::{Deserialize, Deserializer, de::Error as _};
+use veil::Redact;
-use crate::{AdditionalFields, Error, Object, Value};
+use crate::{Object, Value};
-pub type Lookup<'a> = &'a dyn Fn(&str) -> Option;
-
-const SUPPORTED: &[&str] = &[
- "url",
- "host",
- "port",
- "username",
- "password",
- "ssl",
- "db",
- "socket_timeout",
- "startup_nodes",
- "namespace",
-];
-
-const IGNORED: &[&str] = &[
- "max_connections",
- "single_connection_client",
- "health_check_interval",
- "socket_keepalive",
- "socket_keepalive_options",
- "decode_responses",
- "encoding",
- "encoding_errors",
- "charset",
- "errors",
- "client_name",
- "lib_name",
- "lib_version",
- "sentinel_password",
- "gcp_ssl_ca_certs",
- "azure_client_id",
- "azure_tenant_id",
- "azure_client_secret",
- "aws_iam_user_name",
- "aws_iam_cache_name",
- "aws_iam_region",
- "aws_iam_serverless",
-];
-
-const WARN_IGNORED: &[&str] = &[
- "max_connections",
- "health_check_interval",
- "socket_keepalive",
- "socket_keepalive_options",
-];
-
-const REDIS_PY_KWARGS: &[&str] = &[
- "aws_iam_auth",
- "aws_iam_cache_name",
- "aws_iam_region",
- "aws_iam_serverless",
- "aws_iam_user_name",
- "azure_client_id",
- "azure_client_secret",
- "azure_redis_ad_token",
- "azure_tenant_id",
- "cache",
- "cache_config",
- "charset",
- "client_name",
- "db",
- "decode_responses",
- "encoding",
- "encoding_errors",
- "errors",
- "event_dispatcher",
- "gcp_service_account",
- "gcp_ssl_ca_certs",
- "health_check_interval",
- "host",
- "lib_name",
- "lib_version",
- "max_connections",
- "password",
- "port",
- "protocol",
- "redis_connect_func",
- "retry_on_error",
- "retry_on_timeout",
- "single_connection_client",
- "socket_connect_timeout",
- "socket_keepalive",
- "socket_keepalive_options",
- "socket_timeout",
- "ssl",
- "ssl_ca_certs",
- "ssl_ca_data",
- "ssl_ca_path",
- "ssl_cert_reqs",
- "ssl_certfile",
- "ssl_check_hostname",
- "ssl_ciphers",
- "ssl_keyfile",
- "ssl_min_version",
- "ssl_ocsp_context",
- "ssl_ocsp_expected_cert",
- "ssl_password",
- "ssl_validate_ocsp",
- "ssl_validate_ocsp_stapled",
- "unix_socket_path",
- "url",
- "username",
-];
-
-const UNSUPPORTED_EXTRA: &[&str] = &[
- "credential_provider",
- "sentinel_nodes",
- "retry",
- "connection_pool",
-];
-
-const CACHE_ONLY: &[&str] = &[
- "type",
- "mode",
- "ttl",
- "default_in_memory_ttl",
- "default_in_redis_ttl",
- "supported_call_types",
- "similarity_threshold",
- "max_messages",
- "_backend",
-];
-
-#[derive(Clone, Default)]
+/// A Redis section as `config.yaml` spells it, before any surface reads it.
+#[derive(Clone, Default, Deserialize, Redact)]
+#[serde(transparent)]
pub struct RedisSettings {
- pub url: Option,
- pub host: Option>,
- pub port: Option>,
- pub username: Option,
- pub password: Option,
- pub ssl: Option>,
- pub db: Option>,
- pub socket_timeout: Option>,
- pub startup_nodes: Option,
- pub namespace: Option>,
- pub additional_fields: AdditionalFields,
- unparsed: DeferredInputs,
+ #[redact(with = "[REDACTED]")]
+ fields: BTreeMap,
}
-impl<'de> Deserialize<'de> for RedisSettings {
- fn deserialize>(deserializer: D) -> Result {
- let fields = BTreeMap::::deserialize(deserializer)?;
- let scalar = |key: &str| fields.get(key).and_then(as_text);
- Ok(Self {
- url: scalar("url").map(SecretValue::new),
- host: scalar("host").map(Spelled::Value),
- port: spelled(scalar("port")),
- username: scalar("username").map(SecretValue::new),
- password: scalar("password").map(SecretValue::new),
- ssl: spelled(scalar("ssl")),
- db: spelled(scalar("db")),
- socket_timeout: spelled(scalar("socket_timeout")),
- startup_nodes: fields
- .get("startup_nodes")
- .filter(|value| !value.is_null())
- .cloned(),
- namespace: scalar("namespace").map(Spelled::Value),
- additional_fields: cloned(&fields, |key, _| !SUPPORTED.contains(&key)),
- unparsed: deferred_inputs(&fields),
- })
- }
-}
-
-impl fmt::Debug for RedisSettings {
- fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
- formatter
- .debug_struct("RedisSettings")
- .field("url", &self.url)
- .field("host", &self.host)
- .field("port", &self.port)
- .field("username", &self.username)
- .field("password", &self.password)
- .field("ssl", &self.ssl)
- .field("db", &self.db)
- .field("socket_timeout", &self.socket_timeout)
- .field(
- "startup_nodes",
- &self.startup_nodes.as_ref().map(|_| ""),
- )
- .field("namespace", &self.namespace)
- .field("additional_fields", &self.additional_fields.keys())
- .field("unparsed", &self.unparsed.keys())
- .finish()
+impl RedisSettings {
+ pub fn fields(&self) -> &BTreeMap {
+ &self.fields
}
}
@@ -216,452 +42,22 @@ impl<'de> Deserialize<'de> for CacheSettings {
}
}
-#[derive(Clone, PartialEq)]
-pub struct ResolvedRedis {
- pub endpoint: RedisEndpoint,
- pub namespace: Option,
- pub timeout: Option,
-}
+#[cfg(test)]
+mod tests {
+ use super::RedisSettings;
+ use rstest::rstest;
-impl fmt::Debug for ResolvedRedis {
- fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
- formatter
- .debug_struct("ResolvedRedis")
- .field("endpoint", &self.endpoint)
- .field("namespace", &self.namespace)
- .field("timeout", &self.timeout)
- .finish()
+ #[rstest]
+ #[case::scalar(serde_json::json!("credential-value"))]
+ #[case::object(serde_json::json!({"secret": "credential-value"}))]
+ #[case::array(serde_json::json!(["credential-value"]))]
+ fn credentials_are_redacted(#[case] password: serde_json::Value) {
+ let settings: RedisSettings = serde_json::from_value(serde_json::json!({
+ "host": "redis", "password": password,
+ }))
+ .unwrap();
+ let debug = format!("{settings:?}");
+ assert!(!debug.contains("credential-value"));
+ assert!(debug.contains("[REDACTED]"));
}
}
-
-#[derive(Clone, PartialEq)]
-pub enum RedisEndpoint {
- Url(SecretValue),
- Host(RedisHost),
- Cluster(RedisCluster),
-}
-
-impl fmt::Debug for RedisEndpoint {
- fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
- match self {
- Self::Url(_) => formatter.debug_tuple("Url").field(&"").finish(),
- Self::Host(host) => host.fmt(formatter),
- Self::Cluster(cluster) => cluster.fmt(formatter),
- }
- }
-}
-
-#[derive(Clone, Debug, PartialEq)]
-pub struct RedisHost {
- pub host: String,
- pub port: u16,
- pub username: Option,
- pub password: Option,
- pub tls: bool,
- pub db: i64,
-}
-
-#[derive(Clone, Debug, PartialEq)]
-pub struct RedisCluster {
- pub nodes: Vec,
- pub username: Option,
- pub password: Option,
- pub tls: bool,
-}
-
-#[derive(Clone, Debug, PartialEq)]
-pub struct RedisNode {
- pub host: String,
- pub port: u16,
-}
-
-#[derive(Clone)]
-struct Entry {
- value: Value,
- source: String,
-}
-
-type Kwargs = BTreeMap;
-
-impl RedisSettings {
- pub fn resolve(&self, lookup: Lookup<'_>) -> Result {
- self.resolve_with(lookup, false)
- }
-
- pub fn resolve_cache(&self, lookup: Lookup<'_>) -> Result {
- self.resolve_with(lookup, true)
- }
-
- fn resolve_with(&self, lookup: Lookup<'_>, cache: bool) -> Result {
- let block = self.kwargs(cache);
- let block: Kwargs = block
- .into_iter()
- .filter_map(|(key, entry)| {
- let value = environment_reference(&entry.value, lookup)?;
- Some((key, Entry { value, ..entry }))
- })
- .collect();
- let environment = environment_kwargs(lookup);
- let names_a_target = ["host", "startup_nodes", "sentinel_nodes"]
- .iter()
- .any(|key| block.contains_key(*key));
- let drop_environment_url = names_a_target && !block.contains_key("url");
- let merged: Kwargs = environment
- .into_iter()
- .filter(|(key, _)| !(key == "url" && drop_environment_url))
- .chain(block)
- .collect();
- reject_unsupported(&merged, lookup)?;
- warn_ignored(&merged);
-
- let startup_nodes = match merged.get("startup_nodes") {
- Some(entry) => Some(cluster_nodes(&entry.value)?),
- None => lookup("REDIS_CLUSTER_NODES")
- .map(|nodes| cluster_nodes(&Value::String(nodes)))
- .transpose()?,
- };
- let username = secret_value(&merged, "username");
- let password = secret_value(&merged, "password");
- let tls = bool_value(&merged, "ssl");
- let endpoint = match (startup_nodes, text_value(&merged, "url")) {
- (Some(nodes), _) if !nodes.is_empty() => RedisEndpoint::Cluster(RedisCluster {
- nodes,
- username,
- password,
- tls,
- }),
- (_, Some(url)) => RedisEndpoint::Url(SecretValue::new(url)),
- _ => RedisEndpoint::Host(RedisHost {
- host: text_value(&merged, "host").ok_or(Error::RedisInvalid {
- setting: "host",
- reason: "either host or url must be specified for redis",
- })?,
- port: integer(&merged, "port")?.unwrap_or(6379),
- username,
- password,
- tls,
- db: integer(&merged, "db")?.unwrap_or(0),
- }),
- };
- let timeout = merged
- .get("socket_timeout")
- .and_then(|entry| as_text(&entry.value))
- .map(|seconds| {
- seconds
- .parse::()
- .ok()
- .and_then(|seconds| Duration::try_from_secs_f64(seconds).ok())
- .ok_or(Error::RedisInvalid {
- setting: "socket_timeout",
- reason: "must be a non-negative number of seconds",
- })
- })
- .transpose()?;
- Ok(ResolvedRedis {
- endpoint,
- namespace: text_value(&merged, "namespace"),
- timeout,
- })
- }
-
- fn kwargs(&self, cache: bool) -> Kwargs {
- let known = [
- (
- "url",
- self.url.as_ref().map(|value| value.expose().to_owned()),
- ),
- ("host", spelled_text(self.host.as_ref())),
- ("port", spelled_text(self.port.as_ref())),
- (
- "username",
- self.username
- .as_ref()
- .map(|value| value.expose().to_owned()),
- ),
- (
- "password",
- self.password
- .as_ref()
- .map(|value| value.expose().to_owned()),
- ),
- ("ssl", spelled_text(self.ssl.as_ref())),
- ("db", spelled_text(self.db.as_ref())),
- ("socket_timeout", spelled_text(self.socket_timeout.as_ref())),
- ("namespace", spelled_text(self.namespace.as_ref())),
- ];
- let known = known.into_iter().filter_map(|(key, value)| {
- value.map(|value| {
- (
- key.to_owned(),
- Entry {
- value: Value::String(value),
- source: key.to_owned(),
- },
- )
- })
- });
- let startup_nodes = self.startup_nodes.clone().map(|value| {
- (
- "startup_nodes".to_owned(),
- Entry {
- value,
- source: "startup_nodes".to_owned(),
- },
- )
- });
- let extras = self.additional_fields.iter().filter_map(|(key, value)| {
- if cache && CACHE_ONLY.contains(&key.as_str()) {
- return None;
- }
- let key = match (cache, key.as_str()) {
- (true, "redis_startup_nodes") => "startup_nodes".to_owned(),
- _ => key.clone(),
- };
- Some((
- key.clone(),
- Entry {
- value: value.clone(),
- source: key,
- },
- ))
- });
- let unparsed = self.unparsed.iter().map(|(key, value)| {
- (
- key.clone(),
- Entry {
- value: value.0.clone(),
- source: key.clone(),
- },
- )
- });
- extras
- .chain(unparsed)
- .chain(known)
- .chain(startup_nodes)
- .collect()
- }
-}
-
-pub fn environment_has_target(lookup: Lookup<'_>) -> bool {
- [
- "REDIS_HOST",
- "REDIS_URL",
- "REDIS_CLUSTER_NODES",
- "REDIS_SENTINEL_NODES",
- ]
- .iter()
- .any(|name| lookup(name).is_some())
-}
-
-fn reject_unsupported(merged: &Kwargs, lookup: Lookup<'_>) -> Result<(), Error> {
- let active_auth = [
- merged
- .get("gcp_service_account")
- .map(|entry| entry.source.as_str())
- .or_else(|| lookup("REDIS_GCP_SERVICE_ACCOUNT").map(|_| "REDIS_GCP_SERVICE_ACCOUNT")),
- text_value(merged, "azure_redis_ad_token")
- .or_else(|| lookup("REDIS_AZURE_AD_TOKEN"))
- .filter(|value| value.eq_ignore_ascii_case("true"))
- .map(|_| {
- merged
- .get("azure_redis_ad_token")
- .map_or("REDIS_AZURE_AD_TOKEN", |entry| entry.source.as_str())
- }),
- bool_value(merged, "aws_iam_auth").then(|| merged["aws_iam_auth"].source.as_str()),
- merged
- .get("sentinel_nodes")
- .map(|entry| entry.source.as_str())
- .or_else(|| lookup("REDIS_SENTINEL_NODES").map(|_| "REDIS_SENTINEL_NODES")),
- ];
- if let Some(setting) = active_auth.into_iter().flatten().next() {
- return Err(Error::RedisSetting(setting.to_owned()));
- }
- match merged
- .iter()
- .find(|(key, _)| is_unsupported(key))
- .map(|(_, entry)| entry.source.clone())
- {
- Some(setting) => Err(Error::RedisSetting(setting)),
- None => Ok(()),
- }
-}
-
-fn warn_ignored(merged: &Kwargs) {
- for setting in WARN_IGNORED {
- if merged.contains_key(*setting) {
- tracing::warn!(setting, "Redis setting is accepted but ignored");
- }
- }
-}
-
-fn is_unsupported(key: &str) -> bool {
- let known = REDIS_PY_KWARGS.contains(&key) || UNSUPPORTED_EXTRA.contains(&key);
- let handled = SUPPORTED.contains(&key)
- || IGNORED.contains(&key)
- || matches!(
- key,
- "gcp_service_account" | "azure_redis_ad_token" | "aws_iam_auth"
- );
- known && !handled
-}
-
-fn environment_kwargs(lookup: Lookup<'_>) -> Kwargs {
- REDIS_PY_KWARGS
- .iter()
- .filter_map(|key| {
- let source = format!("REDIS_{}", key.to_ascii_uppercase());
- let value = lookup(&source)?;
- Some((
- (*key).to_owned(),
- Entry {
- value: Value::String(value),
- source,
- },
- ))
- })
- .collect()
-}
-
-fn environment_reference(value: &Value, lookup: Lookup<'_>) -> Option {
- match value {
- Value::String(text) => match text.strip_prefix("os.environ/") {
- Some(name) => lookup(name).map(Value::String),
- None => Some(value.clone()),
- },
- Value::Null => None,
- _ => Some(value.clone()),
- }
-}
-
-fn as_text(value: &Value) -> Option {
- match value {
- Value::String(text) => Some(text.clone()),
- Value::Number(number) => Some(number.to_string()),
- Value::Bool(flag) => Some(flag.to_string()),
- _ => None,
- }
-}
-
-fn text_value(kwargs: &Kwargs, key: &str) -> Option {
- kwargs.get(key).and_then(|entry| as_text(&entry.value))
-}
-
-fn secret_value(kwargs: &Kwargs, key: &str) -> Option {
- text_value(kwargs, key).map(SecretValue::new)
-}
-
-fn bool_value(kwargs: &Kwargs, key: &str) -> bool {
- match kwargs.get(key).map(|entry| &entry.value) {
- Some(Value::Bool(flag)) => *flag,
- Some(value) => as_text(value)
- .is_some_and(|text| matches!(text.to_ascii_lowercase().as_str(), "true" | "1" | "yes")),
- None => false,
- }
-}
-
-fn integer(kwargs: &Kwargs, key: &'static str) -> Result, Error> {
- kwargs
- .get(key)
- .and_then(|entry| as_text(&entry.value))
- .map(|text| {
- text.parse().map_err(|_| Error::RedisInvalid {
- setting: key,
- reason: "must be an integer",
- })
- })
- .transpose()
-}
-
-const INVALID_NODES: Error = Error::RedisInvalid {
- setting: "startup_nodes",
- reason: "must be a list of {host, port} objects",
-};
-
-fn cluster_nodes(value: &Value) -> Result, Error> {
- match value {
- Value::String(json) => {
- let nodes: Vec> =
- serde_json::from_str(json).map_err(|_| INVALID_NODES)?;
- nodes
- .iter()
- .map(|node| {
- cluster_node(
- node.get("host").and_then(json_text),
- node.get("port").and_then(json_text),
- )
- })
- .collect()
- }
- Value::Sequence(nodes) => nodes
- .iter()
- .map(|node| {
- cluster_node(
- node.get("host").and_then(as_text),
- node.get("port").and_then(as_text),
- )
- })
- .collect(),
- _ => Err(INVALID_NODES),
- }
-}
-
-fn cluster_node(host: Option, port: Option) -> Result {
- match (host, port.and_then(|port| port.parse().ok())) {
- (Some(host), Some(port)) => Ok(RedisNode { host, port }),
- _ => Err(INVALID_NODES),
- }
-}
-
-fn json_text(value: &serde_json::Value) -> Option {
- match value {
- serde_json::Value::String(text) => Some(text.clone()),
- serde_json::Value::Number(number) => Some(number.to_string()),
- _ => None,
- }
-}
-
-fn spelled_text(value: Option<&Spelled>) -> Option {
- value.map(|value| match value {
- Spelled::Value(value) => value.to_string(),
- Spelled::Text(value) => value.clone(),
- })
-}
-
-fn spelled(text: Option) -> Option> {
- text.map(|text| match text.parse() {
- Ok(value) => Spelled::Value(value),
- Err(_) => Spelled::Text(text),
- })
-}
-
-fn cloned(
- fields: &BTreeMap,
- keep: impl Fn(&str, &Value) -> bool,
-) -> AdditionalFields {
- fields
- .iter()
- .filter(|(key, value)| keep(key, value))
- .map(|(key, value)| (key.clone(), value.clone()))
- .collect()
-}
-
-#[derive(Clone)]
-struct DeferredInput(Value);
-
-type DeferredInputs = BTreeMap;
-
-fn deferred_inputs(fields: &BTreeMap) -> DeferredInputs {
- fields
- .iter()
- .filter(|(key, value)| is_unparsed(key, value))
- .map(|(key, value)| (key.clone(), DeferredInput(value.clone())))
- .collect()
-}
-
-fn is_unparsed(key: &str, value: &Value) -> bool {
- SUPPORTED.contains(&key)
- && match key {
- "startup_nodes" => value.is_null(),
- _ => as_text(value).is_none(),
- }
-}
diff --git a/litellm-rust/crates/config/tests/config.rs b/litellm-rust/crates/config/tests/config.rs
index 3c7d02203e5..fec042e41be 100644
--- a/litellm-rust/crates/config/tests/config.rs
+++ b/litellm-rust/crates/config/tests/config.rs
@@ -440,3 +440,29 @@ fn mcp_config_redacts_nested_credentials_and_preserves_policy_for_validation() {
fn rejects_invalid_typed_mcp_settings(#[case] setting: &str) {
assert!(Config::from_yaml(&format!("mcp_servers:\n docs:\n {setting}\n")).is_err());
}
+
+#[rstest]
+fn inactive_redis_cache_keeps_malformed_known_inputs_deferred() {
+ let config = Config::from_yaml(
+ "litellm_settings:\n cache: false\n cache_params:\n type: redis\n host: []\n port: invalid\n socket_timeout: invalid",
+ )
+ .unwrap();
+
+ assert!(matches!(
+ config.litellm_settings.cache_params,
+ Some(litellm_config::CacheSettings::Redis(_))
+ ));
+}
+
+#[rstest]
+fn non_redis_cache_settings_remain_opaque() {
+ let config = Config::from_yaml(
+ "litellm_settings:\n cache_params:\n type: local\n custom_option:\n nested: true",
+ )
+ .unwrap();
+
+ assert!(matches!(
+ config.litellm_settings.cache_params,
+ Some(litellm_config::CacheSettings::Other(_))
+ ));
+}
diff --git a/litellm-rust/crates/config/tests/redis.rs b/litellm-rust/crates/config/tests/redis.rs
deleted file mode 100644
index ab409541c8e..00000000000
--- a/litellm-rust/crates/config/tests/redis.rs
+++ /dev/null
@@ -1,405 +0,0 @@
-use std::{
- collections::HashMap,
- fmt::Write,
- sync::{Arc, Mutex},
-};
-
-use litellm_config::{CacheSettings, Config, Error, RedisEndpoint, RedisSettings, ResolvedRedis};
-use rstest::rstest;
-use tracing::{Event, Subscriber, field::Visit};
-use tracing_subscriber::{Layer, layer::Context, prelude::*};
-
-fn resolve(
- yaml: &str,
- environment: &[(&str, &str)],
-) -> Result {
- let environment: HashMap = environment
- .iter()
- .map(|(key, value)| ((*key).to_owned(), (*value).to_owned()))
- .collect();
- let config = Config::from_yaml(yaml).unwrap();
- config
- .general_settings
- .coordination_redis
- .as_ref()
- .unwrap_or(&RedisSettings::default())
- .resolve(&|name| environment.get(name).cloned())
-}
-
-fn resolve_cache(yaml: &str) -> Result {
- let config = Config::from_yaml(yaml).unwrap();
- match config.litellm_settings.cache_params.unwrap() {
- CacheSettings::Redis(settings) => settings.resolve_cache(&|_| None),
- CacheSettings::Other(_) => panic!("expected a Redis cache"),
- }
-}
-
-fn cluster_ports(resolved: &ResolvedRedis) -> Vec {
- match &resolved.endpoint {
- RedisEndpoint::Cluster(cluster) => cluster.nodes.iter().map(|node| node.port).collect(),
- endpoint => panic!("expected a cluster, got {endpoint:?}"),
- }
-}
-
-#[rstest]
-#[case::host(
- "general_settings:\n coordination_redis:\n host: redis\n port: 6380\n db: 2\n ssl: true\n namespace: app\n socket_timeout: 1.5",
- &[],
- "host:redis:6380:2:true:app:1500"
-)]
-#[case::environment_url(
- "model_list: []",
- &[("REDIS_URL", "redis://environment:6379")],
- "url:redis://environment:6379"
-)]
-#[case::explicit_host_drops_environment_url(
- "general_settings:\n coordination_redis:\n host: configured",
- &[("REDIS_URL", "redis://environment:6379")],
- "host:configured:6379:0:false::none"
-)]
-#[case::yaml_cluster(
- "general_settings:\n coordination_redis:\n startup_nodes:\n - host: node-a\n port: 7000\n - host: node-b\n port: \"7001\"",
- &[],
- "cluster:node-a:7000,node-b:7001"
-)]
-#[case::json_environment_cluster(
- "model_list: []",
- &[("REDIS_CLUSTER_NODES", r#"[{"host":"node-a","port":"7000"},{"host":"node-b","port":7001}]"#)],
- "cluster:node-a:7000,node-b:7001"
-)]
-#[case::missing_reference_keeps_environment_fallback(
- "general_settings:\n coordination_redis:\n host: os.environ/MISSING_HOST",
- &[("REDIS_HOST", "environment")],
- "host:environment:6379:0:false::none"
-)]
-#[case::configured_values_beat_the_same_environment_values(
- "general_settings:\n coordination_redis:\n host: configured\n port: 6380\n db: 3",
- &[("REDIS_HOST", "environment"), ("REDIS_PORT", "6390"), ("REDIS_DB", "9")],
- "host:configured:6380:3:false::none"
-)]
-#[case::environment_url_beats_the_environment_host(
- "model_list: []",
- &[("REDIS_URL", "redis://environment:6379"), ("REDIS_HOST", "ignored")],
- "url:redis://environment:6379"
-)]
-#[case::startup_nodes_beat_a_url(
- "general_settings:\n coordination_redis:\n url: redis://single:6379\n startup_nodes:\n - host: node-a\n port: 7000",
- &[],
- "cluster:node-a:7000"
-)]
-#[case::environment_references_resolve_the_host_and_port(
- "general_settings:\n coordination_redis:\n host: os.environ/COORD_HOST\n port: os.environ/COORD_PORT",
- &[("COORD_HOST", "referenced"), ("COORD_PORT", "6400")],
- "host:referenced:6400:0:false::none"
-)]
-#[case::a_null_host_falls_back_to_the_environment(
- "general_settings:\n coordination_redis:\n host: null",
- &[("REDIS_HOST", "environment")],
- "host:environment:6379:0:false::none"
-)]
-fn resolves_redis_through_the_parsed_config(
- #[case] yaml: &str,
- #[case] environment: &[(&str, &str)],
- #[case] expected: &str,
-) {
- let resolved = resolve(yaml, environment).unwrap();
- let description = match resolved.endpoint {
- RedisEndpoint::Url(url) => format!("url:{}", url.expose()),
- RedisEndpoint::Host(host) => format!(
- "host:{}:{}:{}:{}:{}:{}",
- host.host,
- host.port,
- host.db,
- host.tls,
- resolved.namespace.unwrap_or_default(),
- resolved
- .timeout
- .map(|timeout| timeout.as_millis().to_string())
- .unwrap_or_else(|| "none".to_owned())
- ),
- RedisEndpoint::Cluster(cluster) => format!(
- "cluster:{}",
- cluster
- .nodes
- .iter()
- .map(|node| format!("{}:{}", node.host, node.port))
- .collect::>()
- .join(",")
- ),
- };
-
- assert_eq!(description, expected);
-}
-
-#[rstest]
-#[case::custom_tls_file(
- "general_settings:\n coordination_redis:\n host: redis\n ssl_ca_certs: /ca.pem",
- &[],
- "ssl_ca_certs"
-)]
-#[case::gcp_auth_from_config(
- "general_settings:\n coordination_redis:\n host: redis\n gcp_service_account: account",
- &[],
- "gcp_service_account"
-)]
-#[case::gcp_auth_from_environment(
- "model_list: []",
- &[("REDIS_HOST", "redis"), ("REDIS_GCP_SERVICE_ACCOUNT", "account")],
- "REDIS_GCP_SERVICE_ACCOUNT"
-)]
-#[case::azure_auth_from_config(
- "general_settings:\n coordination_redis:\n host: redis\n azure_redis_ad_token: true",
- &[],
- "azure_redis_ad_token"
-)]
-#[case::azure_auth_from_environment(
- "model_list: []",
- &[("REDIS_HOST", "redis"), ("REDIS_AZURE_AD_TOKEN", "true")],
- "REDIS_AZURE_AD_TOKEN"
-)]
-#[case::aws_auth_from_config(
- "general_settings:\n coordination_redis:\n host: redis\n aws_iam_auth: true",
- &[],
- "aws_iam_auth"
-)]
-#[case::aws_auth_from_environment(
- "model_list: []",
- &[("REDIS_HOST", "redis"), ("REDIS_AWS_IAM_AUTH", "true")],
- "REDIS_AWS_IAM_AUTH"
-)]
-#[case::sentinel_from_config(
- "general_settings:\n coordination_redis:\n sentinel_nodes: [[redis, 26379]]",
- &[],
- "sentinel_nodes"
-)]
-#[case::sentinel_from_environment(
- "model_list: []",
- &[("REDIS_SENTINEL_NODES", "[[\"redis\",26379]]")],
- "REDIS_SENTINEL_NODES"
-)]
-fn rejected_settings_name_their_source(
- #[case] yaml: &str,
- #[case] environment: &[(&str, &str)],
- #[case] expected: &str,
-) {
- let error = resolve(yaml, environment).unwrap_err();
-
- assert!(matches!(error, Error::RedisSetting(setting) if setting == expected));
-}
-
-#[rstest]
-fn an_explicit_host_keeps_environment_credentials() {
- let resolved = resolve(
- "general_settings:\n coordination_redis:\n host: configured",
- &[
- ("REDIS_PASSWORD", "environment-secret"),
- ("REDIS_SSL", "True"),
- ],
- )
- .unwrap();
-
- let RedisEndpoint::Host(host) = resolved.endpoint else {
- panic!("expected a host endpoint");
- };
- assert_eq!(
- host.password.as_ref().map(|password| password.expose()),
- Some("environment-secret")
- );
- assert!(host.tls);
-}
-
-#[rstest]
-#[case::only_a_port("general_settings:\n coordination_redis:\n port: 6379", &[])]
-#[case::a_structured_host_with_an_environment_target(
- "general_settings:\n coordination_redis:\n host: []",
- &[("REDIS_HOST", "environment"), ("REDIS_URL", "redis://environment:6379")]
-)]
-fn a_block_without_a_usable_target_fails(#[case] yaml: &str, #[case] environment: &[(&str, &str)]) {
- let error = resolve(yaml, environment).unwrap_err();
-
- assert!(matches!(
- error,
- Error::RedisInvalid {
- setting: "host",
- ..
- }
- ));
-}
-
-#[rstest]
-fn the_cache_alias_names_cluster_nodes() {
- let resolved = resolve_cache(
- "litellm_settings:\n cache_params:\n type: redis\n redis_startup_nodes:\n - host: node-a\n port: 7000",
- )
- .unwrap();
-
- assert_eq!(cluster_ports(&resolved), vec![7000]);
-}
-
-#[rstest]
-fn canonical_startup_nodes_beat_the_cache_alias() {
- let resolved = resolve_cache(
- "litellm_settings:\n cache_params:\n type: redis\n startup_nodes:\n - host: canonical\n port: 7000\n redis_startup_nodes:\n - host: alias\n port: 7100\n - host: alias\n port: 7101",
- )
- .unwrap();
-
- assert_eq!(cluster_ports(&resolved), vec![7000]);
-}
-
-#[rstest]
-fn a_null_canonical_startup_nodes_setting_shadows_the_cache_alias() {
- let resolved = resolve_cache(
- "litellm_settings:\n cache_params:\n type: redis\n host: cache\n startup_nodes: null\n redis_startup_nodes:\n - host: alias\n port: 7100",
- )
- .unwrap();
-
- assert!(matches!(resolved.endpoint, RedisEndpoint::Host(_)));
-}
-
-#[rstest]
-fn a_missing_canonical_startup_nodes_reference_shadows_the_cache_alias() {
- let resolved = resolve_cache(
- "litellm_settings:\n cache_params:\n type: redis\n host: cache\n startup_nodes: os.environ/MISSING_NODES\n redis_startup_nodes:\n - host: alias\n port: 7100",
- )
- .unwrap();
-
- assert!(matches!(resolved.endpoint, RedisEndpoint::Host(_)));
-}
-
-#[rstest]
-fn rejected_environment_settings_never_print_the_value() {
- let error = resolve(
- "model_list: []",
- &[
- ("REDIS_HOST", "redis"),
- ("REDIS_SSL_CERT_REQS", "private-option-value"),
- ],
- )
- .unwrap_err();
- let message = error.to_string();
-
- assert!(message.contains("REDIS_SSL_CERT_REQS"));
- assert!(!message.contains("private-option-value"));
-}
-
-#[rstest]
-#[case::negative("-1")]
-#[case::not_a_number("slow")]
-fn rejects_bad_socket_timeout(#[case] timeout: &str) {
- let yaml = format!(
- "general_settings:\n coordination_redis:\n host: redis\n socket_timeout: {timeout}"
- );
- let error = resolve(&yaml, &[]).unwrap_err();
-
- assert!(matches!(
- error,
- Error::RedisInvalid {
- setting: "socket_timeout",
- ..
- }
- ));
-}
-
-#[rstest]
-fn defaults_socket_timeout_for_the_gateway() {
- let resolved = resolve(
- "general_settings:\n coordination_redis:\n host: redis",
- &[],
- )
- .unwrap();
-
- assert_eq!(resolved.timeout, None);
-}
-
-#[rstest]
-fn debug_redacts_redis_secrets_and_url() {
- let config = Config::from_yaml(
- "general_settings:\n coordination_redis:\n url: redis://user:password@redis:6379\n username: credential-user\n password: credential-password",
- )
- .unwrap();
- let settings = config.general_settings.coordination_redis.unwrap();
- let resolved = settings.resolve(&|_| None).unwrap();
- let debug = format!("{settings:?} {resolved:?}");
-
- assert!(!debug.contains("redis://user:password"));
- assert!(!debug.contains("credential-user"));
- assert!(!debug.contains("credential-password"));
- assert!(debug.contains("redacted") || debug.contains("REDACTED"));
-}
-
-#[rstest]
-fn inactive_redis_cache_keeps_malformed_known_inputs_deferred() {
- let config = Config::from_yaml(
- "litellm_settings:\n cache: false\n cache_params:\n type: redis\n host: []\n port: invalid\n socket_timeout: invalid",
- )
- .unwrap();
-
- assert!(matches!(
- config.litellm_settings.cache_params,
- Some(litellm_config::CacheSettings::Redis(_))
- ));
-}
-
-#[rstest]
-fn non_redis_cache_settings_remain_opaque() {
- let config = Config::from_yaml(
- "litellm_settings:\n cache_params:\n type: local\n custom_option:\n nested: true",
- )
- .unwrap();
-
- assert!(matches!(
- config.litellm_settings.cache_params,
- Some(litellm_config::CacheSettings::Other(_))
- ));
-}
-
-#[rstest]
-fn legacy_unknown_settings_do_not_change_resolution() {
- let resolved = resolve(
- "general_settings:\n coordination_redis:\n host: redis\n redis_flush_size: 42\n service_name: primary",
- &[],
- )
- .unwrap();
-
- assert!(matches!(resolved.endpoint, RedisEndpoint::Host(_)));
-}
-
-struct WarningCapture(Arc>>);
-
-impl Layer for WarningCapture {
- fn on_event(&self, event: &Event<'_>, _context: Context<'_, S>) {
- let mut fields = Fields::default();
- event.record(&mut fields);
- self.0.lock().unwrap().push(fields.0);
- }
-}
-
-#[derive(Default)]
-struct Fields(String);
-
-impl Visit for Fields {
- fn record_debug(&mut self, field: &tracing::field::Field, value: &dyn std::fmt::Debug) {
- write!(self.0, "{}={value:?} ", field.name()).unwrap();
- }
-
- fn record_str(&mut self, field: &tracing::field::Field, value: &str) {
- write!(self.0, "{}={value} ", field.name()).unwrap();
- }
-}
-
-#[rstest]
-fn ignored_setting_warning_names_the_setting_without_its_value() {
- let events = Arc::new(Mutex::new(Vec::new()));
- let subscriber = tracing_subscriber::registry().with(WarningCapture(events.clone()));
- tracing::subscriber::with_default(subscriber, || {
- resolve(
- "general_settings:\n coordination_redis:\n host: redis\n max_connections: 47",
- &[],
- )
- .unwrap();
- });
- let warnings = events.lock().unwrap().join(" ");
-
- assert!(warnings.contains("max_connections"));
- assert!(!warnings.contains("47"));
-}
diff --git a/litellm-rust/crates/gateway-forward/AGENTS.md b/litellm-rust/crates/gateway-forward/AGENTS.md
new file mode 100644
index 00000000000..2135980266f
--- /dev/null
+++ b/litellm-rust/crates/gateway-forward/AGENTS.md
@@ -0,0 +1,37 @@
+# Forwarding gateway
+
+`Forward` is a reverse proxy in front of the Python proxy on loopback. Python must see each request as if the client had connected to it directly, because every Python route (inference, MCP, pass-through, UI, WebSocket) is served behind it without Rust knowing which one it is
+
+- Serve HTTP/1.1 only, as uvicorn does. The supervisor never puts Rust in front of a Python server that speaks HTTP/2
+- Never branch on a route or path. A behavior one endpoint needs is a protocol behavior every endpoint gets
+- Forward the method, request target, headers, and body byte for byte within the [URL contract](FORWARDING.md#url-fragments-and-literal-hashes). An unescaped `#` starts a fragment and is omitted with everything after it, while `%23` is preserved. Only drop hop-by-hop headers (RFC 9110 section 7.6.1) and set `x-litellm-rust-peer`, which replaces any client copy
+- Stream both bodies. Never buffer, cap, or time out a body, since uploads and provider streams have no upper bound
+- A client disconnect closes the upstream connection at every stage, because that is how Python learns to stop a provider stream
+- An upstream that dies mid-response fails the client body. Never end it cleanly, or SSE clients treat a truncated stream as complete
+- Never invent a Host. Python builds absolute URLs from it, so a missing one must never become the private upstream address
+- A request Python could not have received unchanged (HTTP/1.1 without Host, a transfer coding other than chunked, a target without a path) gets the status uvicorn would have given, from Rust
+- Pooled upstream connections are retired well before Python's keep-alive timeout, so a request never lands on a connection Python is closing
+- Idle client connections close after Python's keep-alive timeout, as uvicorn closed them, so a client that vanishes cannot hold a socket forever. Only the wait for the next request is timed, never an exchange in progress
+- WebSocket is the one protocol switch Python serves. Upgrades are relayed as raw bytes after the 101 and drained on shutdown
+
+## Layout
+
+Every forwarding decision has one home. `rewrite.rs` owns what Python and the client receive, as pure functions over requests and responses. `Rejection` in `error.rs` is the only place that picks a status for a request Rust answers itself. `settings.rs` parses the environment into typed values, so an `Upstream` is a loopback origin by construction. `server.rs` accepts client connections and owns their timeouts, since `axum::serve` exposes none, and `connections.rs` tracks every connection and relay for shutdown. `forward.rs` and `websocket.rs` only move requests, bodies and upgraded connections and never touch a header themselves. `connector.rs` keeps hyper reading Python's answer after Python stops reading the request
+
+## Environment
+
+`Settings` reads only these variables, and only when `LITELLM_FORWARD_UPSTREAM` is set. The Python supervisor in `litellm/proxy/rust_forward/` sets every one of them from the values Python actually runs with, so all of them are required and a missing or invalid one fails startup naming it. The crate never reads Python's own settings (`HOST`, `PORT`, `KEEPALIVE_TIMEOUT`, `GRACEFUL_SHUTDOWN_TIMEOUT`) and keeps no copy of Python's defaults, since Python can get those values from CLI flags the environment never shows
+
+| Variable | Meaning |
+|---|---|
+| `LITELLM_FORWARD_UPSTREAM` | Python's loopback origin, e.g. `http://127.0.0.1:53211` |
+| `LITELLM_FORWARD_HOST` | public bind host |
+| `LITELLM_FORWARD_PORT` | public bind port |
+| `LITELLM_FORWARD_UPSTREAM_KEEPALIVE` | seconds uvicorn keeps an idle connection open. Pooled connections are retired at half of it, and idle client connections are closed after it |
+| `LITELLM_FORWARD_SHUTDOWN_DRAIN` | seconds to drain on shutdown, already including Python's graceful timeout and teardown headroom |
+
+## Tests
+
+Test through real sockets with `tests/support`, never `oneshot`. `Gateway` runs the forwarder on a listener like production, `recording` is a well-behaved upstream that captures what Python would receive, and `raw_upstream` and `Raw` drive bytes by hand for framing, timing and disconnects a real server would not produce. Waits measured in minutes use paused tokio time instead of real sleeps
+
+The forwarder sees HTTP/1.1 messages, not routes, so the tests are split into one file per traffic shape uvicorn can emit: `exchange` (plain request and response), `framing` (requests whose framing a parser could read two ways), `slow_head` (minutes before response headers), `stream` (incremental response bodies), `idle_stream` (long-lived streams that go quiet), `upload`, `download`, `websocket`, `client_identity`, `lifecycle` (disconnects, pooling, idle timeouts, shutdown) and `roundtrip` (the forwarding invariant for generated messages). A new route never gets its own test. Add its behavior to the shape it already has, and when it needs a wire behavior none of the files covers, add a new shape file for it
diff --git a/litellm-rust/crates/gateway-forward/Cargo.toml b/litellm-rust/crates/gateway-forward/Cargo.toml
new file mode 100644
index 00000000000..2fef19dca34
--- /dev/null
+++ b/litellm-rust/crates/gateway-forward/Cargo.toml
@@ -0,0 +1,30 @@
+[package]
+name = "litellm-gateway-forward"
+version = "0.1.0"
+edition.workspace = true
+rust-version.workspace = true
+license.workspace = true
+repository.workspace = true
+
+[dependencies]
+axum.workspace = true
+envy = "0.4.2"
+hyper = { version = "1.12.0", features = ["http1", "server"] }
+hyper-util = { workspace = true, features = ["client-legacy", "http1", "tokio"] }
+serde.workspace = true
+thiserror.workspace = true
+tokio = { workspace = true, features = ["io-util"] }
+tokio-util = { version = "0.7", features = ["rt"] }
+tower-service = "0.3.3"
+tracing.workspace = true
+
+[dev-dependencies]
+bytes.workspace = true
+futures-util.workspace = true
+http-body-util = "0.1"
+httparse = "1"
+hyper = { version = "1.12.0", features = ["client"] }
+proptest.workspace = true
+rstest.workspace = true
+tokio = { workspace = true, features = ["io-util", "sync", "test-util"] }
+tokio-tungstenite.workspace = true
diff --git a/litellm-rust/crates/gateway-forward/FORWARDING.md b/litellm-rust/crates/gateway-forward/FORWARDING.md
new file mode 100644
index 00000000000..532f341aba5
--- /dev/null
+++ b/litellm-rust/crates/gateway-forward/FORWARDING.md
@@ -0,0 +1,31 @@
+# Python forwarding compatibility
+
+The opt-in forwarding gateway serves HTTP/1.1 and WebSockets in front of Python. Python still owns every route and policy. Its protocol contract and environment settings are in [AGENTS.md](AGENTS.md)
+
+## URL fragments and literal hashes
+
+The gateway follows Hyper's `http::Uri` parsing. If a raw client sends an unescaped `#` in the request target, only the path and query before the first `#` reach Python. For example, `/v1/files?purpose=batch#retry&limit=1` becomes `/v1/files?purpose=batch`, and `/gemini/v1beta/files/a#b?key=k` becomes `/gemini/v1beta/files/a`
+
+This is accepted behavior because `#` starts a URL fragment, which is separated before dereferencing ([RFC 3986 section 3.5](https://www.rfc-editor.org/rfc/rfc3986.html#section-3.5)). HTTP origin-form contains only the path and optional query, not a fragment ([RFC 9112 section 3.2.1](https://www.rfc-editor.org/rfc/rfc9112.html#section-3.2.1)). Browsers and URL-based clients omit the fragment before sending the request, so UI hash routing is unaffected
+
+A literal hash in path or query data must be encoded as `%23`, which the gateway forwards unchanged. Direct Python with h11 also accepts unescaped hashes as data, but that permissive raw-target behavior is outside this URL contract. Active socket tests cover both [fragment omission](tests/exchange.rs) and [encoded-hash preservation](tests/exchange.rs)
+
+## Known differences from uvicorn with h11
+
+These cases remain executable as ignored socket tests. They are not passing parity checks
+
+| Input | Direct Python | Through Rust | Regression test |
+|---|---|---|---|
+| Header after a removed hop-by-hop field | Keeps the remaining header order | `HeaderMap::remove` can move a later header | [dropping_a_hop_by_hop_header_keeps_the_others_in_order](tests/exchange.rs) |
+| Repeated header names interleaved with other names | Keeps the interleaving | Groups values by name, keeping their order within that name | [interleaved_repeated_headers_reach_python_interleaved](tests/exchange.rs) |
+| Obsolete folded header, such as `x-a: b\r\n c` | Unfolds to `b c` | Rejects with 400 | [a_folded_header_reaches_python_unfolded](tests/exchange.rs) |
+| Identical Content-Length list, such as `7, 7` | Accepts length 7 | Rejects with 400 | [a_content_length_list_of_one_value_reaches_python](tests/framing.rs) |
+| HTTP/1.0 request with chunked transfer coding | Decodes the body | Rejects with 400 | [an_http10_chunked_upload_reaches_python](tests/framing.rs) |
+
+Clients can avoid the parser rejections by sending unfolded headers, one Content-Length value, and HTTP/1.1 for chunked uploads. Integrations must not depend on header ordering across different field names
+
+Run `cargo test -p litellm-gateway-forward --test exchange --test framing -- --ignored` from this directory to reproduce the differences
+
+## Connection header handling
+
+The gateway strips standard hop-by-hop fields and fields named by Connection, except routing, representation, credential, forwarding, and WebSocket handshake headers. A client cannot nominate those end-to-end fields for removal. The private `x-litellm-rust-peer` always replaces any client-supplied copy
diff --git a/litellm-rust/crates/gateway-forward/src/connections.rs b/litellm-rust/crates/gateway-forward/src/connections.rs
new file mode 100644
index 00000000000..10a4867fa51
--- /dev/null
+++ b/litellm-rust/crates/gateway-forward/src/connections.rs
@@ -0,0 +1,58 @@
+use tokio_util::{
+ sync::{CancellationToken, DropGuard},
+ task::TaskTracker,
+};
+
+/// Every client connection and WebSocket relay the gateway runs, so shutdown can wait for them
+/// and then close whatever outlives the drain.
+#[derive(Clone, Default)]
+pub(crate) struct Connections {
+ tasks: TaskTracker,
+ draining: CancellationToken,
+ closed: CancellationToken,
+}
+
+impl Connections {
+ /// Runs `work` until it finishes or the gateway closes every connection.
+ pub fn spawn(&self, work: impl Future