Merge remote-tracking branch 'upstream/litellm_internal_staging' into runware-provider

2026-10-13 03:59:12 +00:00 · 2026-06-25 22:28:02 +04:00 · 2026-06-25 22:28:02 +04:00 · 95dd73a5c8
commit 95dd73a5c8
parent ecbafd5221 17bfd415ae
569 changed files with 33511 additions and 6174 deletions
--- a/.github/deploy-on-aws.png
+++ b/.github/deploy-on-aws.png
--- a/.github/deploy-on-gcp.png
+++ b/.github/deploy-on-gcp.png
--- a/.github/workflows/guard-fork-dependencies.yml
+++ b/.github/workflows/guard-fork-dependencies.yml
@ -5,7 +5,7 @@ on:
    branches:
      - main
      - litellm_internal_staging
-      - litellm_oss_branch
+      - litellm_oss_staging
      - "litellm_**"
    paths:
      - "uv.lock"
--- a/.github/workflows/guard-main-branch.yml
+++ b/.github/workflows/guard-main-branch.yml
@ -31,12 +31,12 @@ jobs:
          echo "PR head repo: $HEAD_REPO"
          echo "PR head branch: $HEAD_REF"
          if [ "$HEAD_REPO" != "$BASE_REPO" ]; then
-            echo "::error::PRs to main must originate from the canonical repository ($BASE_REPO), not a fork ($HEAD_REPO). External contributors should open PRs against the 'litellm_oss_branch' branch instead."
+            echo "::error::PRs to main must originate from the canonical repository ($BASE_REPO), not a fork ($HEAD_REPO). External contributors should open PRs against the 'litellm_oss_staging' branch instead."
            exit 1
          fi
          if [ "$HEAD_REF" = "litellm_internal_staging" ] || [[ "$HEAD_REF" == litellm_hotfix_?* ]]; then
            echo "Allowed source branch."
            exit 0
          fi
-          echo "::error::PRs to main must originate from 'litellm_internal_staging' or a 'litellm_hotfix_*' branch. Got: '$HEAD_REF'. If this is a contribution, retarget the PR against 'litellm_oss_branch' instead."
+          echo "::error::PRs to main must originate from 'litellm_internal_staging' or a 'litellm_hotfix_*' branch. Got: '$HEAD_REF'. If this is a contribution, retarget the PR against 'litellm_oss_staging' instead."
          exit 1
--- a/.github/workflows/osv-scan.yml
+++ b/.github/workflows/osv-scan.yml
@ -5,13 +5,8 @@ on:
    branches:
      - main
      - litellm_internal_staging
-      - litellm_oss_branch
+      - litellm_oss_staging
      - "litellm_**"
-    paths:
-      - uv.lock
-      - ui/litellm-dashboard/package-lock.json
-      - osv-scanner.toml
-      - .github/workflows/osv-scan.yml
  schedule:
    - cron: "23 6 * * *"
  workflow_dispatch:
--- a/.github/workflows/test-code-quality.yml
+++ b/.github/workflows/test-code-quality.yml
@ -5,7 +5,7 @@ on:
    branches:
      - main
      - litellm_internal_staging
-      - litellm_oss_branch
+      - litellm_oss_staging
      - "litellm_**"

 permissions:
--- a/.github/workflows/test-linting.yml
+++ b/.github/workflows/test-linting.yml
@ -5,7 +5,7 @@ on:
    branches:
      - main
      - litellm_internal_staging
-      - litellm_oss_branch
+      - litellm_oss_staging
      - "litellm_**"

 permissions:
@ -14,7 +14,7 @@ permissions:
 jobs:
  lint:
    runs-on: ubuntu-latest
-    timeout-minutes: 10
+    timeout-minutes: 15

    steps:
      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
@ -50,10 +50,10 @@ jobs:
        run: |
          uv sync --frozen

-      - name: Check Black formatting
+      - name: Check ruff format
        run: |
          cd litellm
-          uv run --no-sync black --check --exclude '/enterprise/' .
+          uv run --no-sync ruff format --check --line-length 88 --exclude '/enterprise/' .
          cd ..

      - name: Debug - Check file state
@ -87,9 +87,11 @@ jobs:
        run: |
          uv run --no-sync python -c "import openai; print(f'OpenAI version: {openai.__version__}')"

-      - name: Run basedpyright type checking
+      - name: Check basedpyright budget (delta vs base)
+        env:
+          BASE_SHA: ${{ github.event.pull_request.base.sha }}
        run: |
-          (uv run --no-sync basedpyright --outputjson || true) | uv run --no-sync python scripts/type_check_gate.py
+          (uv run --no-sync basedpyright --outputjson || true) | uv run --no-sync python scripts/type_check_gate.py --base "$BASE_SHA"

      - name: Check for circular imports
        run: |
--- a/.github/workflows/test-litellm-ui-build.yml
+++ b/.github/workflows/test-litellm-ui-build.yml
@ -7,7 +7,7 @@ on:
    branches:
      - main
      - litellm_internal_staging
-      - litellm_oss_branch
+      - litellm_oss_staging
      - "litellm_**"

 jobs:
@ -111,4 +111,4 @@ jobs:
        if: ${{ !cancelled() && steps.changed.outputs.has_files == 'true' }}
        run: |
          npx eslint . -f json -o "$RUNNER_TEMP/lint-report.json" || true
-          node scripts/check-lint-budgets.mjs "$RUNNER_TEMP/lint-report.json" eslint-budgets.json
+          node scripts/check-lint-budgets.mjs "$RUNNER_TEMP/lint-report.json" eslint-budgets.json --check eslint-metrics.json
--- a/.github/workflows/test-mcp.yml
+++ b/.github/workflows/test-mcp.yml
@ -5,7 +5,7 @@ on:
    branches:
      - main
      - litellm_internal_staging
-      - litellm_oss_branch
+      - litellm_oss_staging
      - "litellm_**"

 permissions:
--- a/.github/workflows/test-model-map.yaml
+++ b/.github/workflows/test-model-map.yaml
@ -5,7 +5,7 @@ on:
    branches:
      - main
      - litellm_internal_staging
-      - litellm_oss_branch
+      - litellm_oss_staging
      - "litellm_**"

 permissions:
--- a/.github/workflows/test-rust.yml
+++ b/.github/workflows/test-rust.yml
@ -0,0 +1,65 @@
+name: LiteLLM Rust
+
+on:
+  push:
+    paths:
+      - "litellm-rust/**"
+      - ".github/workflows/test-rust.yml"
+  pull_request:
+    branches:
+      - main
+      - litellm_internal_staging
+      - litellm_oss_staging
+      - "litellm_**"
+    paths:
+      - "litellm-rust/**"
+      - ".github/workflows/test-rust.yml"
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  rust-checks:
+    name: rustfmt, clippy, test
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    defaults:
+      run:
+        working-directory: litellm-rust
+    env:
+      CARGO_TERM_COLOR: always
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          persist-credentials: false
+
+      - name: Set up Rust
+        run: |
+          rustup toolchain install stable --profile minimal --component clippy,rustfmt
+          rustup default stable
+
+      - name: Cache Cargo registry and target
+        uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
+        with:
+          path: |
+            ~/.cargo/registry
+            ~/.cargo/git
+            litellm-rust/target
+          key: ${{ runner.os }}-cargo-${{ hashFiles('litellm-rust/Cargo.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-cargo-
+
+      - name: Check Rust formatting
+        run: cargo fmt --check
+
+      - name: Run Clippy
+        run: cargo clippy --workspace --all-targets --locked -- -D warnings
+
+      - name: Run Rust tests
+        run: cargo test --workspace --locked
--- a/.github/workflows/test-semgrep.yml
+++ b/.github/workflows/test-semgrep.yml
@ -5,7 +5,7 @@ on:
    branches:
      - main
      - litellm_internal_staging
-      - litellm_oss_branch
+      - litellm_oss_staging
      - "litellm_**"

 permissions:
--- a/.github/workflows/test-unit-core-utils.yml
+++ b/.github/workflows/test-unit-core-utils.yml
@ -5,7 +5,7 @@ on:
    branches:
      - main
      - litellm_internal_staging
-      - litellm_oss_branch
+      - litellm_oss_staging
      - "litellm_**"

 permissions:
--- a/.github/workflows/test-unit-documentation.yml
+++ b/.github/workflows/test-unit-documentation.yml
@ -5,7 +5,7 @@ on:
    branches:
      - main
      - litellm_internal_staging
-      - litellm_oss_branch
+      - litellm_oss_staging
      - "litellm_**"

 permissions:
--- a/.github/workflows/test-unit-enterprise-routing.yml
+++ b/.github/workflows/test-unit-enterprise-routing.yml
@ -5,7 +5,7 @@ on:
    branches:
      - main
      - litellm_internal_staging
-      - litellm_oss_branch
+      - litellm_oss_staging
      - "litellm_**"

 permissions:
--- a/.github/workflows/test-unit-integrations.yml
+++ b/.github/workflows/test-unit-integrations.yml
@ -5,7 +5,7 @@ on:
    branches:
      - main
      - litellm_internal_staging
-      - litellm_oss_branch
+      - litellm_oss_staging
      - "litellm_**"

 permissions:
--- a/.github/workflows/test-unit-llm-providers.yml
+++ b/.github/workflows/test-unit-llm-providers.yml
@ -5,7 +5,7 @@ on:
    branches:
      - main
      - litellm_internal_staging
-      - litellm_oss_branch
+      - litellm_oss_staging
      - "litellm_**"

 permissions:
--- a/.github/workflows/test-unit-misc.yml
+++ b/.github/workflows/test-unit-misc.yml
@ -5,7 +5,7 @@ on:
    branches:
      - main
      - litellm_internal_staging
-      - litellm_oss_branch
+      - litellm_oss_staging
      - "litellm_**"

 permissions:
@ -32,7 +32,9 @@ jobs:
        tests/test_litellm/repositories
        tests/test_litellm/images
        tests/test_litellm/interactions
+        tests/test_litellm/ocr
        tests/test_litellm/passthrough
+        tests/test_litellm/sandbox
        tests/test_litellm/vector_stores
        tests/test_litellm/test_*.py
      workers: 2
--- a/.github/workflows/test-unit-proxy-auth.yml
+++ b/.github/workflows/test-unit-proxy-auth.yml
@ -5,7 +5,7 @@ on:
    branches:
      - main
      - litellm_internal_staging
-      - litellm_oss_branch
+      - litellm_oss_staging
      - "litellm_**"

 permissions:
--- a/.github/workflows/test-unit-proxy-endpoints.yml
+++ b/.github/workflows/test-unit-proxy-endpoints.yml
@ -5,7 +5,7 @@ on:
    branches:
      - main
      - litellm_internal_staging
-      - litellm_oss_branch
+      - litellm_oss_staging
      - "litellm_**"
  workflow_dispatch:

--- a/.github/workflows/test-unit-proxy-infra.yml
+++ b/.github/workflows/test-unit-proxy-infra.yml
@ -5,7 +5,7 @@ on:
    branches:
      - main
      - litellm_internal_staging
-      - litellm_oss_branch
+      - litellm_oss_staging
      - "litellm_**"

 permissions:
@ -29,6 +29,7 @@ jobs:
        tests/test_litellm/proxy/_experimental
        tests/test_litellm/proxy/experimental
        tests/test_litellm/proxy/common_utils
+        tests/test_litellm/proxy/logging_endpoints
        tests/test_litellm/proxy/test_*.py
      workers: 2
      reruns: 2
--- a/.github/workflows/test-unit-proxy-legacy.yml
+++ b/.github/workflows/test-unit-proxy-legacy.yml
@ -5,7 +5,7 @@ on:
    branches:
      - main
      - litellm_internal_staging
-      - litellm_oss_branch
+      - litellm_oss_staging
      - "litellm_**"

 permissions:
--- a/.github/workflows/test-unit-responses-caching-types.yml
+++ b/.github/workflows/test-unit-responses-caching-types.yml
@ -5,7 +5,7 @@ on:
    branches:
      - main
      - litellm_internal_staging
-      - litellm_oss_branch
+      - litellm_oss_staging
      - "litellm_**"

 permissions:
--- a/.github/workflows/test_server_root_path.yml
+++ b/.github/workflows/test_server_root_path.yml
@ -7,7 +7,7 @@ on:
    branches:
      - main
      - litellm_internal_staging
-      - litellm_oss_branch
+      - litellm_oss_staging
      - "litellm_**"

 jobs:
--- a/.gitignore
+++ b/.gitignore
@ -123,3 +123,6 @@ crash.*.log
 # and should be committed.
 .vscode
 .pin_list.txt
+
+# pytest coverage data
+.coverage
--- a/CLAUDE.md
+++ b/CLAUDE.md
@ -29,7 +29,7 @@ If you ever make public-facing PR descriptions, comments, issues, commit message
 - don't use "—". Instead, reach for ";", ".", etc.
 - don't use the pattern "It's not X, it's Y", "You're not X, you're Y", etc.
 - don't use bulleted or numbered lists unless it would be nonsensical not to. Instead, prefer prose
- don't add a trailing "." at the end of paragraphs (just like this file)
+- don't add a trailing "." at the end of paragraphs (just like this file). That means every paragraph, not just the last one (of the markdown file, PR description, GitHub comment, etc.). Rule of thumb: unless there's a sentence immediately after, don't add a "."
 - don't use →. Instead, prefer not to use arrows, and if need be, use -> instead

 Don't hesitate to use values in .env to get needed API keys and other secrets, as long as you never add them to conversation history, commit them, or include them in GitHub issues / PRs
--- a/4
+++ b/4
@ -1,8 +1,8 @@
 # Base image for building
-ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:31da6565f35af6401031c1d7aa91dc84ac76c5c48edd17fb90f0ed9e3173c7a9
+ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:c61ac6919b811ea53c4782d69f1fe05218ba3c25d53f01b6ab7892e621bd4370

 # Runtime image
-ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:31da6565f35af6401031c1d7aa91dc84ac76c5c48edd17fb90f0ed9e3173c7a9
+ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:c61ac6919b811ea53c4782d69f1fe05218ba3c25d53f01b6ab7892e621bd4370
 ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a

 FROM $UV_IMAGE AS uvbin
--- a/19
+++ b/19
@ -20,13 +20,13 @@ help:
 	@echo "  make install-test-deps  - Install the full local test environment"
 	@echo "  make install-helm-unittest - Install helm unittest plugin"
 	@echo "  make install-hooks      - Install git hooks (Conventional Commits + Branches)"
-	@echo "  make format             - Apply Black code formatting"
-	@echo "  make format-check       - Check Black code formatting (matches CI)"
-	@echo "  make lint               - Run all linting (Ruff, basedpyright, Black check, circular imports, import safety)"
+	@echo "  make format             - Apply ruff format code formatting"
+	@echo "  make format-check       - Check ruff format code formatting (matches CI)"
+	@echo "  make lint               - Run all linting (Ruff, basedpyright, format check, circular imports, import safety)"
 	@echo "  make lint-ruff          - Run Ruff linting only"
 	@echo "  make lint-basedpyright  - Run basedpyright strict, gated by per-rule error counts"
 	@echo "  make lint-basedpyright-budget-update - Re-capture the basedpyright per-rule budget (ratchet)"
-	@echo "  make lint-black         - Check Black formatting (matches CI)"
+	@echo "  make lint-format        - Check ruff format formatting (matches CI)"
 	@echo "  make lint-ruff-budget - Gate the codebase total of each strict ruff rule against its ceiling"
 	@echo "  make lint-gate        - Strict ruff gate in CI-parity mode (fetches staging, simulates the merge)"
 	@echo "  make lint-ruff-budget-update - Re-capture per-rule baselines in ruff-strict-budget.json (ratchet)"
@ -82,11 +82,13 @@ install-hooks:
 	./scripts/install_git_hooks.sh

 # Formatting
+# 88-column wrap matches the Black width the whole repo is formatted to; ruff.toml's
+# global line-length is 120 (for E501/isort), so 88 is forced here.
 format: install-dev
-	cd litellm && $(UV_RUN) black . && cd ..
+	cd litellm && $(UV_RUN) ruff format --line-length 88 --exclude '/enterprise/' . && cd ..

 format-check: install-dev
-	cd litellm && $(UV_RUN) black --check . && cd ..
+	cd litellm && $(UV_RUN) ruff format --check --line-length 88 --exclude '/enterprise/' . && cd ..

 # Linting targets
 lint-ruff: install-dev
@ -125,12 +127,13 @@ lint-ruff-FULL-dev: install-dev
 	else echo "No changed .py files to check."; fi

 lint-basedpyright: install-dev
-	($(UV_RUN) basedpyright --outputjson || true) | $(UV_RUN) python scripts/type_check_gate.py
+	git fetch origin litellm_internal_staging
+	($(UV_RUN) basedpyright --outputjson || true) | $(UV_RUN) python scripts/type_check_gate.py --base origin/litellm_internal_staging

 lint-basedpyright-budget-update: install-dev
 	($(UV_RUN) basedpyright --outputjson || true) | $(UV_RUN) python scripts/type_check_gate.py --update

-lint-black: format-check
+lint-format: format-check

 lint-ruff-budget: install-dev
 	$(UV_RUN) python scripts/ruff_strict_gate.py
--- a/README.md
+++ b/README.md
@ -6,10 +6,10 @@
        </p>
        <p align="center">Open Source AI Gateway for 100+ LLMs. Self-hosted. Enterprise-ready. Call any LLM in OpenAI format.</p>
        <p align="center">
-        <a href="https://render.com/deploy?repo=https://github.com/BerriAI/litellm" target="_blank" rel="nofollow"><img src="https://render.com/images/deploy-to-render-button.svg" alt="Deploy to Render"></a>
-        <a href="https://railway.com/deploy/RhvhdC?referralCode=7mRv9K&utm_medium=integration&utm_source=template&utm_campaign=generic">
-          <img src="https://railway.com/button.svg" alt="Deploy on Railway">
-        </a>
+        <a href="https://render.com/deploy?repo=https://github.com/BerriAI/litellm" target="_blank" rel="nofollow"><img src="https://render.com/images/deploy-to-render-button.svg" alt="Deploy to Render" height="40"></a>
+        <a href="https://railway.com/deploy/RhvhdC?referralCode=7mRv9K&utm_medium=integration&utm_source=template&utm_campaign=generic"><img src="https://railway.com/button.svg" alt="Deploy on Railway" height="40"></a>
+        <a href="https://console.aws.amazon.com/cloudshell/home" target="_blank" rel="nofollow"><img src="./.github/deploy-on-aws.png" alt="Deploy on AWS" height="40"></a>
+        <a href="https://ssh.cloud.google.com/cloudshell/editor?cloudshell_git_repo=https%3A%2F%2Fgithub.com%2FBerriAI%2Flitellm&cloudshell_workspace=terraform%2Flitellm%2Fgcp%2Fexamples%2Fdefault&cloudshell_tutorial=TUTORIAL.md&cloudshell_image=gcr.io/ds-artifacts-cloudshell/deploystack_custom_image&shellonly=true" target="_blank" rel="nofollow"><img src="./.github/deploy-on-gcp.png" alt="Deploy on GCP" height="40"></a>
        </p>
    </p>
 <h4 align="center"><a href="https://docs.litellm.ai/docs/simple_proxy" target="_blank">LiteLLM Proxy Server (AI Gateway)</a> | <a href="https://docs.litellm.ai/docs/enterprise#hosted-litellm-proxy" target="_blank"> Hosted Proxy</a> | <a href="https://litellm.ai/enterprise"target="_blank">Enterprise Tier</a> | <a href="https://www.litellm.ai/ai-gateway" target="_blank">Website</a></h4>
@ -406,6 +406,140 @@ You can use LiteLLM through either the Proxy Server or Python SDK. Both give you

 Support for more providers. Missing a provider or LLM Platform, raise a [feature request](https://github.com/BerriAI/litellm/issues/new?assignees=&labels=enhancement&projects=&template=feature_request.yml&title=%5BFeature%5D%3A+).

+### Deploy on AWS or GCP with Terraform
+
+Run the LiteLLM proxy as a production-ready componentized stack (gateway, backend, UI on separate services; managed Postgres + Redis + object store) using the published Terraform modules. Both modules are on the [public Terraform Registry](https://registry.terraform.io/namespaces/BerriAI) — no auth needed.
+
+#### AWS — ECS Fargate + Aurora + ElastiCache + ALB
+
+[![Launch in AWS CloudShell](https://img.shields.io/badge/Launch-AWS_CloudShell-FF9900?logo=amazon-aws&logoColor=white)](https://console.aws.amazon.com/cloudshell/home) — opens an in-browser shell, already authenticated to your AWS account. Once inside, run:
+
+```bash
+git clone https://github.com/BerriAI/litellm.git
+cd litellm/terraform/litellm/aws/examples/default
+cp terraform.tfvars.example terraform.tfvars   # edit region/tenant/env
+terraform init && terraform apply
+```
+
+[Module page →](https://registry.terraform.io/modules/BerriAI/litellm/aws/latest)
+
+Or call the module from your own root config:
+
+```hcl
+# main.tf
+terraform {
+  required_version = ">= 1.6.0"
+  required_providers {
+    aws = { source = "hashicorp/aws", version = "~> 5.60" }
+  }
+}
+
+provider "aws" {
+  region = "us-west-2"
+}
+
+module "litellm" {
+  source  = "BerriAI/litellm/aws"
+  version = "~> 1.89"
+
+  region = "us-west-2"
+  azs    = ["us-west-2a", "us-west-2b"]
+  tenant = "acme"
+  env    = "prod"
+
+  # Production: provide an ACM cert. Without one, set allow_plaintext_alb = true
+  # (dev/trial only).
+  # acm_certificate_arn = "arn:aws:acm:us-west-2:111122223333:certificate/..."
+  allow_plaintext_alb = true
+}
+
+output "litellm_url" {
+  value = module.litellm.alb_dns_name
+}
+```
+
+```bash
+terraform init
+terraform apply
+```
+
+Provider API keys live in AWS Secrets Manager; reference ARNs via `gateway_extra_secrets`. Full input list and architecture diagram on the [registry page](https://registry.terraform.io/modules/BerriAI/litellm/aws/latest?tab=inputs).
+
+#### GCP — Cloud Run + Cloud SQL + Memorystore + HTTPS LB
+
+[![Open in Cloud Shell](https://gstatic.com/cloudssh/images/open-btn.png)](https://ssh.cloud.google.com/cloudshell/editor?cloudshell_git_repo=https%3A%2F%2Fgithub.com%2FBerriAI%2Flitellm&cloudshell_workspace=terraform%2Flitellm%2Fgcp%2Fexamples%2Fdefault&cloudshell_tutorial=TUTORIAL.md&cloudshell_image=gcr.io/ds-artifacts-cloudshell/deploystack_custom_image&shellonly=true)
+
+Real 1-click. Opens Cloud Shell, clones this repo, and walks you through `terraform apply` via a built-in [DeployStack tutorial](./terraform/litellm/gcp/examples/default/TUTORIAL.md) — pick the project, the tutorial sets up the Artifact Registry remote repo, writes `terraform.tfvars` from your answers, and runs apply.
+
+[Module page →](https://registry.terraform.io/modules/BerriAI/litellm/google/latest)
+
+To call the module from your own config instead, Cloud Run can't pull from `ghcr.io` directly, so first set up a one-time Artifact Registry remote repo backed by GHCR:
+
+```bash
+gcloud artifacts repositories create litellm \
+  --location=us-central1 \
+  --repository-format=docker \
+  --mode=remote-repository \
+  --remote-docker-repo=https://ghcr.io \
+  --project=my-gcp-project
+```
+
+Then:
+
+```hcl
+# main.tf
+terraform {
+  required_version = ">= 1.6.0"
+  required_providers {
+    google      = { source = "hashicorp/google",      version = "~> 6.10" }
+    google-beta = { source = "hashicorp/google-beta", version = "~> 6.10" }
+  }
+}
+
+provider "google"      { project = "my-gcp-project"; region = "us-central1" }
+provider "google-beta" { project = "my-gcp-project"; region = "us-central1" }
+
+module "litellm" {
+  source  = "BerriAI/litellm/google"
+  version = "~> 1.89"
+
+  project_id = "my-gcp-project"
+  region     = "us-central1"
+  tenant     = "acme"
+  env        = "prod"
+
+  # Replace my-gcp-project with your GCP project ID (same value as project_id above).
+  image_registry = "us-central1-docker.pkg.dev/my-gcp-project/litellm/berriai"
+
+  # Production: provide DNS already pointing at the LB IP for Google-managed certs.
+  # Without one, set allow_plaintext_lb = true (dev/trial only).
+  # lb_domains         = ["proxy.example.com"]
+  allow_plaintext_lb = true
+}
+
+output "litellm_url" {
+  value = module.litellm.load_balancer_url
+}
+```
+
+```bash
+terraform init
+terraform apply
+```
+
+Provider API keys live in Secret Manager; reference resource IDs (e.g. `projects/my-gcp-project/secrets/openai-api-key`) via `gateway_extra_secrets`. Full input list and architecture diagram on the [registry page](https://registry.terraform.io/modules/BerriAI/litellm/google/latest?tab=inputs).
+
+#### Both stacks include
+
+- The full componentized split (gateway / backend / UI as independent services)
+- Managed Postgres (writer + reader) and Redis
+- Versioned object store for proxy state + file uploads
+- An auto-generated `LITELLM_MASTER_KEY` in your cloud's secret manager
+- A one-off migration job that runs `prisma migrate deploy` before the proxy starts
+- The same `proxy_config` surface as the [Helm chart](./helm/litellm/) — pass YAML as a typed map
+
+The Terraform modules live at [`terraform/litellm/aws/`](./terraform/litellm/aws/) and [`terraform/litellm/gcp/`](./terraform/litellm/gcp/) in this repo; the registry entries are read-only mirrors updated on each release.
+
 ### Run in Developer Mode
 #### Services
 1. Setup .env file in root
--- a/backend/Dockerfile
+++ b/backend/Dockerfile
@ -1,5 +1,5 @@
-ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:31da6565f35af6401031c1d7aa91dc84ac76c5c48edd17fb90f0ed9e3173c7a9
-ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:31da6565f35af6401031c1d7aa91dc84ac76c5c48edd17fb90f0ed9e3173c7a9
+ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:c61ac6919b811ea53c4782d69f1fe05218ba3c25d53f01b6ab7892e621bd4370
+ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:c61ac6919b811ea53c4782d69f1fe05218ba3c25d53f01b6ab7892e621bd4370
 ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a

 FROM $UV_IMAGE AS uvbin
--- a/backend/routes/allowlist.py
+++ b/backend/routes/allowlist.py
@ -84,6 +84,8 @@ BACKEND_PATH_PREFIXES: tuple[str, ...] = (
    "/active/callbacks",
    "/callbacks",
    "/team_callback",
+    # Rust data-plane gateway → proxy control-plane API (logging today, auth later)
+    "/v1/rust_control_plane/",
    # Alerting / email / IP allowlist
    "/alerting/",
    "/email/",
--- a/basedpyright-code-budget.json
+++ b/basedpyright-code-budget.json
@ -69,7 +69,7 @@
  },
  "reportMatchNotExhaustive": {
    "baseline": 1,
-    "slack": 3
+    "slack": 0
  },
  "reportMissingParameterType": {
    "baseline": 3933,
@ -121,7 +121,7 @@
  },
  "reportReturnType": {
    "baseline": 126,
-    "slack": 13
+    "slack": 100
  },
  "reportTypedDictNotRequiredAccess": {
    "baseline": 20,
@ -157,7 +157,7 @@
  },
  "reportUnnecessaryComparison": {
    "baseline": 683,
-    "slack": 10
+    "slack": 100
  },
  "reportUnnecessaryContains": {
    "baseline": 4,
--- a/docker/Dockerfile.database
+++ b/docker/Dockerfile.database
@ -1,8 +1,8 @@
 # Base image for building
-ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:3258be472764337fd13095bcbb3182da170243b5819fd67ad4c0754590588b31
+ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:c61ac6919b811ea53c4782d69f1fe05218ba3c25d53f01b6ab7892e621bd4370

 # Runtime image
-ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:3258be472764337fd13095bcbb3182da170243b5819fd67ad4c0754590588b31
+ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:c61ac6919b811ea53c4782d69f1fe05218ba3c25d53f01b6ab7892e621bd4370
 ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a

 FROM $UV_IMAGE AS uvbin
--- a/docker/Dockerfile.non_root
+++ b/docker/Dockerfile.non_root
@ -1,6 +1,6 @@
 # Base images
-ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:3258be472764337fd13095bcbb3182da170243b5819fd67ad4c0754590588b31
-ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:3258be472764337fd13095bcbb3182da170243b5819fd67ad4c0754590588b31
+ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:c61ac6919b811ea53c4782d69f1fe05218ba3c25d53f01b6ab7892e621bd4370
+ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:c61ac6919b811ea53c4782d69f1fe05218ba3c25d53f01b6ab7892e621bd4370
 ARG PROXY_EXTRAS_SOURCE=published
 ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a

--- a/enterprise/litellm_enterprise/proxy/hooks/managed_files.py
+++ b/enterprise/litellm_enterprise/proxy/hooks/managed_files.py
@ -13,7 +13,9 @@ from litellm import Router, verbose_logger
 from litellm._uuid import uuid
 from litellm.caching.caching import DualCache
 from litellm.integrations.custom_logger import CustomLogger
-from litellm.litellm_core_utils.prompt_templates.common_utils import extract_file_data
+from litellm.litellm_core_utils.prompt_templates.common_utils import (
+    extract_file_metadata,
+)
 from litellm.llms.base_llm.files.transformation import BaseFileEndpoints
 from litellm.llms.base_llm.managed_resources.isolation import (
    build_list_page,
@ -981,9 +983,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
        target_model_names_list: List[str],
    ) -> OpenAIFileObject:
        ## GET THE FILE TYPE FROM THE CREATE FILE REQUEST
-        file_data = extract_file_data(create_file_request["file"])
-
-        file_type = file_data["content_type"]
+        _, file_type = extract_file_metadata(create_file_request["file"])

        output_file_id = file_objects[0].id
        model_id = file_objects[0]._hidden_params.get("model_id")
--- a/gateway/Dockerfile
+++ b/gateway/Dockerfile
@ -1,5 +1,5 @@
-ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:31da6565f35af6401031c1d7aa91dc84ac76c5c48edd17fb90f0ed9e3173c7a9
-ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:31da6565f35af6401031c1d7aa91dc84ac76c5c48edd17fb90f0ed9e3173c7a9
+ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:c61ac6919b811ea53c4782d69f1fe05218ba3c25d53f01b6ab7892e621bd4370
+ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:c61ac6919b811ea53c4782d69f1fe05218ba3c25d53f01b6ab7892e621bd4370
 ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a

 FROM $UV_IMAGE AS uvbin
--- a/litellm-rust/.cargo/config.toml
+++ b/litellm-rust/.cargo/config.toml
@ -0,0 +1,10 @@
+# PyO3 cdylib (`litellm-python-bridge`) links against the host interpreter's
+# symbols, which are not present at link time when building an extension module.
+# On macOS, tell the linker to resolve undefined `_Py*` symbols dynamically at
+# load time (the standard pyo3 extension-module flag) so the cdylib links without
+# a libpython on the link line.
+[target.x86_64-apple-darwin]
+rustflags = ["-C", "link-arg=-undefined", "-C", "link-arg=dynamic_lookup"]
+
+[target.aarch64-apple-darwin]
+rustflags = ["-C", "link-arg=-undefined", "-C", "link-arg=dynamic_lookup"]
--- a/litellm-rust/.gitignore
+++ b/litellm-rust/.gitignore
@ -0,0 +1 @@
+/target/
--- a/litellm-rust/ADDING_A_PROVIDER.md
+++ b/litellm-rust/ADDING_A_PROVIDER.md
@ -0,0 +1,9 @@
+# Adding a provider / route to litellm-rust
+
+Three layers, same for every route (see `ocr` and `realtime` as references):
+
+1. **Transform contract (pure)** — `crates/core/src/<route>/transformation.rs`: a `…ProviderConfig` trait (URL build + request/response transforms) + types in `types.rs`. No network, env, or auth.
+2. **Provider config (pure)** — `crates/providers/src/<provider>/<route>/transformation.rs`: implement that trait as a `const <PROVIDER>_<ROUTE>_CONFIG`, mirroring the Python provider tree. Add parity unit tests.
+3. **HTTP / transport (the host)** — `crates/providers/src/<route>.rs` (e.g. `ocr.rs`, `realtime.rs`): the callable fn (`run_ocr`, `realtime`). It resolves the key, builds the auth header, builds URL + transforms via the config, then does the network call. This is the only layer allowed to do I/O.
+
+**Calling:** the host invokes the route fn — the Python bridge calls `run_ocr`; the `ai-gateway` server calls `realtime`. Register new modules in `lib.rs` / `mod.rs`, then run `cargo fmt && cargo clippy --workspace -- -D warnings && cargo test --workspace`.
--- a/litellm-rust/AGENTS.md
+++ b/litellm-rust/AGENTS.md
@ -0,0 +1,17 @@
+# AGENTS.md
+
+litellm-rust has exactly THREE crates. A crate is a LAYER, not a route. Routes (ocr, realtime, chat) and providers (mistral, openai) are MODULES inside the layers.
+
+## Crates
+
+| Crate | Role | Pure / I/O |
+|-------|------|------------|
+| litellm-core | Translation layer — types, route contracts (traits), provider transforms (modules under providers/), and the router. Builds requests/responses; no network. | Pure |
+| litellm-ai-gateway | Routes + host — the only crate that touches the network. HTTP/WebSocket I/O (modules under io/) plus the axum server binary (behind the `server` feature). | I/O |
+| litellm-python-bridge | PyO3 cdylib exposing Rust to the litellm Python SDK — a thin adapter over litellm-ai-gateway's I/O. | Binding |
+
+Dependency direction (acyclic): litellm-core ← litellm-ai-gateway ← litellm-python-bridge.
+
+Adding a crate: default to a MODULE. New crate ONLY on a real trigger — separate artifact (binary/cdylib), proc-macro, shared foundation, or publishable standalone. A new provider or route is none of these.
+
+Adding a crate fails crates/core/tests/workspace_crate_allowlist.rs until you update its allowlist and this file — intentional.
--- a/litellm-rust/CLAUDE.md
+++ b/litellm-rust/CLAUDE.md
@ -0,0 +1,109 @@
+# CLAUDE.md
+
+This file defines the rules for Rust work in LiteLLM.
+
+## Crates (exactly three — see AGENTS.md)
+
+`litellm-core` describes work; `litellm-ai-gateway` executes it; `litellm-python-bridge`
+exposes it to the Python SDK. A crate is a **layer**, not a route — add modules, not crates.
+
+## Core Boundary
+
+`litellm-core` is the pure translation layer; the `litellm-ai-gateway` host executes work.
+
+Route-level Rust structure mirrors LiteLLM's Python responsibilities:
+- `core/src/<route>/` owns the route contract, shared types, and provider
+  template traits. For OCR, this means `core/src/ocr`.
+- `core/src/providers/<provider>/<route>/transformation.rs` owns the
+  provider-specific transform. For Mistral OCR, this means
+  `core/src/providers/mistral/ocr/transformation.rs`.
+- Network execution lives in the host crate `ai-gateway` (`ai-gateway/src/io/`),
+  never inside `core`.
+
+Allowed in `core`:
+- Pure request transforms
+- Pure response transforms
+- Pure stream chunk normalization
+- Shared data types and validation errors
+- Deterministic token/cost helper logic
+
+Not allowed in `core`:
+- Network calls
+- Environment variable or secret reads
+- Filesystem access
+- Database or cache access
+- Provider SDK signing or auth flows
+- Logging callbacks, spend writes, or custom callbacks
+- Global mutable runtime state
+
+Python owns rollout state and fallback while Rust is being introduced. Rust
+paths must be off by default until parity tests prove equivalence with Python.
+
+## Production Bar
+
+Rust code in this workspace is held to a strict parity and robustness bar from
+the first PR:
+
+- Correctness parity is proven with tests. Do not rely on README claims or
+  manual inspection for a port that mirrors Python behavior.
+- Every provider transform must have unit tests for supported-parameter
+  filtering, request body shape, response normalization, missing/null fields,
+  and bad-input errors.
+- When Rust is exposed through Python, add Python tests that prove disabled,
+  enabled, and unavailable-bridge fallback behavior.
+- Avoid panics on user/provider input. Return typed errors and let the host map
+  them to Python exceptions or HTTP responses.
+- OCR handles documents that often contain personal data. Do not log document
+  contents, base64 payloads, provider response bodies, or secrets.
+- Error messages must be useful but data-minimized. Truncate or sanitize any
+  upstream body before it crosses a host boundary.
+- Treat empty or whitespace-only credentials, URLs, and config values as absent
+  at the host/config resolution layer.
+- Preserve Python output shape intentionally. If a field is always serialized as
+  `null` for Python parity, leave a short comment explaining that parity choice.
+
+## Host I/O Rules
+
+These rules apply when adding future crates or modules that execute network I/O,
+such as `ai-gateway`, router hosts, or standalone servers:
+
+- Set connect and full-request timeouts. No unbounded waits.
+- Reuse HTTP clients; do not construct clients per request.
+- Prefer rustls TLS for portable Python wheels and Linux images unless there is
+  a documented reason not to.
+- Add request IDs and structured tracing at the host layer, without logging OCR
+  document contents or secrets.
+- Do not echo raw upstream response bodies to callers. Sanitize and bound them.
+- Avoid `expect`/`unwrap` in server startup and request paths unless the panic is
+  impossible by construction and documented.
+
+## Constants
+
+Magic numbers and fixed strings go in a crate-level `constants.rs`, never
+hardcoded inline — the Rust mirror of Python's `litellm/constants.py`.
+
+- Each crate that needs them has `src/constants.rs` (declared `mod constants;`);
+  import from it (`use crate::constants::...`). Don't scatter `const` values at
+  the top of feature modules.
+- An env-overridable tunable still lives in `constants.rs` as its `DEFAULT_*`
+  value; the env read (with fallback to that default) happens at the host/config
+  resolution layer, not in `core`/`providers`.
+- Exception: a value that is purely local to one function and has no meaning
+  elsewhere may stay inline, but prefer `constants.rs` when in doubt.
+
+## Checks
+
+Run these before pushing Rust changes. The same checks run in GitHub Actions
+for changes under `litellm-rust/`.
+
+```bash
+cd litellm-rust
+cargo fmt --check
+# the ai-gateway binary + server code is behind the `server` feature
+cargo clippy -p litellm-ai-gateway --all-targets --features server -- -D warnings
+cargo clippy -p litellm-core -p litellm-python-bridge --all-targets -- -D warnings
+cargo test --workspace
+```
+
+When a Rust path is exposed through Python, add Python parity tests that compare
+the existing Python output with the Rust-backed output.
--- a/litellm-rust/Cargo.lock
+++ b/litellm-rust/Cargo.lock
--- a/litellm-rust/Cargo.toml
+++ b/litellm-rust/Cargo.toml
@ -0,0 +1,28 @@
+[workspace]
+members = [
+    "crates/core",
+    "crates/ai-gateway",
+    "crates/python-bridge",
+]
+resolver = "2"
+
+[workspace.package]
+edition = "2021"
+license = "MIT"
+repository = "https://github.com/BerriAI/litellm"
+
+[workspace.dependencies]
+litellm-core = { path = "crates/core" }
+litellm-ai-gateway = { path = "crates/ai-gateway", default-features = false }
+axum = "0.7"
+pyo3 = "0.23.5"
+rand = "0.8"
+reqwest = { version = "0.12", default-features = false, features = ["blocking", "json", "rustls-tls"] }
+serde = { version = "1.0", features = ["derive"] }
+serde_json = "1.0"
+sha2 = "0.10"
+subtle = "2"
+thiserror = "2.0"
+tokio = { version = "1", features = ["rt-multi-thread", "macros", "time"] }
+tokio-tungstenite = { version = "0.24", default-features = false, features = ["connect", "rustls-tls-native-roots"] }
+futures-util = { version = "0.3", default-features = false, features = ["sink", "std"] }
--- a/litellm-rust/README.md
+++ b/litellm-rust/README.md
@ -0,0 +1,44 @@
+# LiteLLM Rust
+
+This workspace contains the staged Rust implementation for LiteLLM.
+
+Rust starts as a pure transform core used by the existing Python host. Python
+continues to own auth, configuration, network I/O, retries, routing, logging,
+callbacks, spend tracking, and customer plugins until each Rust path has parity
+coverage and production evidence.
+
+## Crates
+
+| Crate | Role | Pure / I/O |
+|-------|------|------------|
+| litellm-core | Translation layer — types, route contracts (traits), provider transforms (modules under providers/), and the router. Builds requests/responses; no network. | Pure |
+| litellm-ai-gateway | Routes + host — the only crate that touches the network. HTTP/WebSocket I/O (modules under io/) plus the axum server binary (behind the `server` feature). | I/O |
+| litellm-python-bridge | PyO3 cdylib exposing Rust to the litellm Python SDK — a thin adapter over litellm-ai-gateway's I/O. | Binding |
+
+Dependency direction (acyclic): litellm-core ← litellm-ai-gateway ← litellm-python-bridge.
+
+## Layout
+
+```text
+crates/
+  core/           Route contracts, shared pure types, errors, and templates.
+    src/ocr/
+  providers/      Provider-specific pure transforms.
+    src/mistral/ocr/transformation.rs
+  python-bridge/  PyO3 bridge for Python LiteLLM.
+```
+
+The folder shape should follow the Python provider tree:
+`providers/src/<provider>/<route>/transformation.rs`. The bridge should expose
+one function per top-level route, starting with `ocr(payload)`.
+
+## Checks
+
+Run these before pushing Rust changes. GitHub Actions runs the same checks for
+changes under `litellm-rust/`.
+
+```bash
+cargo fmt --check
+cargo clippy --workspace --all-targets -- -D warnings
+cargo test --workspace
+```
--- a/litellm-rust/crates/ai-gateway/AGENTS.md
+++ b/litellm-rust/crates/ai-gateway/AGENTS.md
@ -0,0 +1,50 @@
+# ai-gateway — folder architecture
+
+The Axum server that fronts the Rust gateway. It owns transport + config + auth
+only; deployment selection lives in `core::router`, transforms in `core`/`providers`.
+
+```
+src/
+  main.rs            # entrypoint: build AppState (router + master key), bind, serve
+  state.rs           # AppState — shared Arc<Router> + master_key
+  gil.rs             # GIL-activity tracker (records Python acquisitions)
+  auth/              # authentication as an axum extractor — added to handler args
+    mod.rs           #   RequireMasterKey: FromRequestParts, single master key (LITELLM_MASTER_KEY)
+  routes/            # one module per route, all matching the same template
+    AGENTS.md        #   ← the route template (read this before adding a route)
+    mod.rs           #   app(): merges every module's router()
+    health.rs        #   simple route (one file): router() + liveness/readiness
+    gil.rs           #   simple route (one file): router() + GET /health/gil
+    realtime/        #   route with logic → axum surface + a no-axum service:
+      mod.rs         #     router() + handler + WS<->events adapter (the axum surface)
+      service.rs     #     business logic (select deployment, call provider) — no axum, testable
+  python/            # Python interop (feature: python-config) — load-time only
+    mod.rs, config.rs, AGENTS.md
+```
+
+## Rules
+
+- **Routes follow one template.** Each route module exposes
+  `pub fn router() -> Router<AppState>`; `routes/mod.rs` only merges them. Simple
+  routes are one file; non-trivial routes are a folder (`handler`/`service`/
+  `transport`). See `routes/AGENTS.md`.
+- **Auth is an extractor.** Add `crate::auth::RequireMasterKey` to a handler's
+  args; it runs during extraction. Never re-implement the check per route.
+- **Handlers are thin.** A handler validates and delegates to its `service`. No
+  business logic, no provider calls, no transforms in handlers.
+- **State is shared and cheap to clone.** Long-lived handles live behind `Arc` in
+  `state.rs`; read env/config only in `main.rs` when building state.
+
+## Auth (interim)
+
+A single **master key** (`LITELLM_MASTER_KEY`), enforced by the
+`auth::RequireMasterKey` extractor: any caller presenting it as
+`Authorization: Bearer <key>` may invoke the gateway. Fails closed (500) when
+unset; constant-time compare. The server binds `127.0.0.1` by default (`HOST` to
+override). Full per-key auth + budgets/rate-limits are delegated to the Python
+proxy in a later phase. Health routes don't add the extractor (unauthenticated).
+
+## Python interop
+
+Anything that calls into Python lives in `python/` and is **load-time only** — see
+`python/AGENTS.md`. The realtime data path never takes the GIL.
--- a/litellm-rust/crates/ai-gateway/ARCHITECTURE.md
+++ b/litellm-rust/crates/ai-gateway/ARCHITECTURE.md
@ -0,0 +1,12 @@
+# ai-gateway architecture
+
+The Rust ai-gateway does LLM inference (realtime WebSocket). Spend tracking is an
+API callback: it POSTs each finished session to the LiteLLM proxy, which records
+spend and runs the usual callbacks.
+
+```mermaid
+flowchart LR
+  C[client] <--> G[Rust ai-gateway<br/>LLM inference]
+  G <--> O[OpenAI realtime]
+  G -. spend tracking callback .-> P[litellm proxy]
+```
--- a/litellm-rust/crates/ai-gateway/Cargo.toml
+++ b/litellm-rust/crates/ai-gateway/Cargo.toml
@ -0,0 +1,42 @@
+[package]
+name = "litellm-ai-gateway"
+version = "0.1.0"
+edition.workspace = true
+license.workspace = true
+repository.workspace = true
+
+[lib]
+name = "litellm_ai_gateway"
+
+[[bin]]
+name = "litellm-ai-gateway"
+path = "src/main.rs"
+required-features = ["server"]
+
+[dependencies]
+litellm-core.workspace = true
+# reqwest (rustls + json) is used by io/ocr and ships realtime logs to the
+# Python proxy callbacks API.
+reqwest.workspace = true
+# `sync` powers the bounded mpsc channel the realtime logger drains.
+tokio = { workspace = true, features = ["rt-multi-thread", "macros", "net", "time", "sync"] }
+tokio-tungstenite.workspace = true
+futures-util.workspace = true
+serde_json.workspace = true
+axum = { workspace = true, features = ["ws"], optional = true }
+serde = { workspace = true, optional = true }
+subtle = { workspace = true, optional = true }
+# sha2 hashes the master key into user_api_key_hash (matches the proxy's
+# SHA-256 hash_token) so the plaintext credential never enters a log payload.
+sha2 = { workspace = true, optional = true }
+pyo3 = { workspace = true, features = ["auto-initialize"], optional = true }
+
+[features]
+default = []
+server = ["dep:axum", "dep:subtle", "dep:serde", "dep:sha2"]
+# Build the gateway's config from the proxy YAML via an embedded Python
+# interpreter (links libpython; requires `litellm` importable at runtime).
+python-config = ["dep:pyo3"]
+
+[dev-dependencies]
+futures-channel = "0.3"
--- a/litellm-rust/crates/ai-gateway/Dockerfile
+++ b/litellm-rust/crates/ai-gateway/Dockerfile
@ -0,0 +1,86 @@
+# Multi-stage build for the LiteLLM Rust AI Gateway (realtime WebSocket proxy).
+#
+# Build context is the **repo root** so we can install `litellm` from this repo's
+# source (the gateway loads its model_list via litellm.proxy.read_model_list,
+# which is not in any PyPI release yet) AND build the rust workspace under
+# litellm-rust/.
+#
+#   docker build -f litellm-rust/crates/ai-gateway/Dockerfile -t litellm-ai-gateway .
+#
+# No secrets live in this file. Runtime config (LITELLM_MASTER_KEY,
+# OPENAI_API_KEY referenced by config.yaml, etc.) is injected as environment
+# variables at deploy time.
+
+# ---- Chef -------------------------------------------------------------------
+# cargo-chef caches the dependency build so only the gateway crate recompiles on
+# a source-only change. python3-dev is present in every rust stage because the
+# `python-config` feature links libpython via pyo3 (even in the cook step).
+FROM rust:1.90-slim-bookworm AS chef
+ENV PYO3_PYTHON=python3.11
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends \
+        python3 python3-dev pkg-config libssl-dev clang \
+    && rm -rf /var/lib/apt/lists/* \
+    && cargo install cargo-chef --locked --version 0.1.77
+WORKDIR /build/litellm-rust
+
+# ---- Planner ----------------------------------------------------------------
+# Produce the dependency recipe from the rust workspace manifests + Cargo.lock.
+FROM chef AS planner
+COPY litellm-rust/ .
+RUN cargo chef prepare --recipe-path recipe.json
+
+# ---- Builder ----------------------------------------------------------------
+FROM chef AS builder
+# Cook (compile) just the dependencies first — this layer is cached and reused
+# whenever only gateway source changes.
+COPY --from=planner /build/litellm-rust/recipe.json recipe.json
+RUN cargo chef cook --locked --release \
+        -p litellm-ai-gateway --features python-config \
+        --recipe-path recipe.json
+# Now copy the real sources and build the gateway binary. Deps are already cooked
+# above, so this step only recompiles the gateway crate.
+COPY litellm-rust/ .
+RUN cargo build --locked --release -p litellm-ai-gateway --features python-config
+
+# ---- Runtime ----------------------------------------------------------------
+# python:3.11-slim-bookworm ships libpython3.11, matching the builder's PyO3
+# 3.11 ABI so the embedded interpreter links and imports cleanly.
+FROM python:3.11-slim-bookworm AS runtime
+
+# CA certificates for outbound TLS to the OpenAI realtime endpoint.
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends ca-certificates \
+    && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /app
+
+# Install litellm (with proxy extras) FROM THIS REPO'S SOURCE so
+# `import litellm.proxy.read_model_list` works — it is not on PyPI yet. Copy the
+# package + packaging metadata, then pip install the proxy extra.
+COPY pyproject.toml README.md LICENSE ./
+COPY litellm/ ./litellm/
+RUN pip install --no-cache-dir ".[proxy]"
+
+# The compiled gateway binary (pure-Rust realtime hot path; Python is load-time
+# only).
+COPY --from=builder /build/litellm-rust/target/release/litellm-ai-gateway /usr/local/bin/litellm-ai-gateway
+
+# Default config.yaml. A real deploy can override this (e.g. mount a Render
+# secret file at the same path) — never bake secrets into the image.
+COPY litellm-rust/crates/ai-gateway/config.yaml /app/config.yaml
+
+# Bind to all interfaces (Render routes to 0.0.0.0:$PORT) and load the model_list
+# from config.yaml via the embedded python config reader.
+ENV HOST=0.0.0.0 \
+    LITELLM_CONFIG_PATH=/app/config.yaml
+
+# Drop to a non-root user. The realtime hot path needs no root privileges, so
+# running unprivileged limits blast radius if the process is ever compromised.
+# The binary in /usr/local/bin is world-executable (COPY default mode 755); we
+# only need /app (and the config.yaml it reads) owned by the unprivileged user.
+RUN useradd --system --no-create-home --uid 10001 appuser \
+    && chown -R appuser:appuser /app
+USER appuser
+
+ENTRYPOINT ["/usr/local/bin/litellm-ai-gateway"]
--- a/litellm-rust/crates/ai-gateway/Dockerfile.dockerignore
+++ b/litellm-rust/crates/ai-gateway/Dockerfile.dockerignore
@ -0,0 +1,45 @@
+# Dockerfile-specific ignore-file for the Rust AI Gateway build.
+#
+# The build context is the repo root (so the image can pip install litellm from
+# source AND build the rust workspace). BuildKit honors `<Dockerfile>.dockerignore`
+# next to the Dockerfile and it takes precedence over the repo-root `.dockerignore`,
+# so this file shrinks the (large) repo-root context for THIS build only without
+# touching the root `.dockerignore` used by the main litellm images.
+#
+# Strategy: ignore everything, then re-include only what the build needs:
+#   - litellm/            (pip install . needs the full package + proxy reader)
+#   - litellm-rust/       (the rust workspace; Cargo.lock + crate sources)
+#   - pyproject.toml / README.md / LICENSE  (packaging metadata for pip install)
+*
+
+# --- re-include the build inputs ---
+!litellm/
+!litellm-rust/
+!pyproject.toml
+!README.md
+!LICENSE
+
+# --- prune heavy / irrelevant subpaths back out of the re-included trees ---
+# Rust build artifacts (huge; regenerated in the builder).
+**/target/
+# Python caches and compiled bytecode.
+**/__pycache__/
+**/*.pyc
+**/*.pyo
+**/.pytest_cache/
+**/.ruff_cache/
+**/.mypy_cache/
+# Node / UI build output bundled under the python package (not needed to import
+# litellm.proxy.read_model_list).
+**/node_modules/
+litellm/proxy/_experimental/out/
+# Tests, logs, and local scratch.
+**/tests/
+**/test/
+*.log
+log.txt
+*.tgz
+# VCS / editor / CI metadata that may live under re-included trees.
+**/.git/
+.git/
+**/.DS_Store
--- a/litellm-rust/crates/ai-gateway/README.md
+++ b/litellm-rust/crates/ai-gateway/README.md
@ -0,0 +1,198 @@
+# LiteLLM Rust AI Gateway
+
+A minimal Axum service that fronts OpenAI's realtime API. Clients open a
+WebSocket to `GET /v1/realtime`; the gateway authenticates, selects a deployment,
+dials OpenAI upstream, and splices the two sockets frame-by-frame.
+
+## Crates
+
+`litellm-rust` is exactly three crates (a crate is a **layer**, not a route):
+
+| Crate | Role | Pure / I/O |
+|-------|------|------------|
+| litellm-core | Translation layer — types, route contracts (traits), provider transforms (modules under `providers/`), and the router. Builds requests/responses; no network. | Pure |
+| litellm-ai-gateway | Routes + host — the only crate that touches the network. HTTP/WebSocket I/O (modules under `io/`) plus the Axum server binary (behind the `server` feature). | I/O |
+| litellm-python-bridge | PyO3 cdylib exposing Rust to the litellm Python SDK — a thin adapter over litellm-ai-gateway's I/O. | Binding |
+
+Dependency direction (acyclic): litellm-core ← litellm-ai-gateway ← litellm-python-bridge.
+
+- **Client endpoint:** `wss://<host>/v1/realtime?model=<model>` (WebSocket)
+- **Auth:** `Authorization: Bearer $LITELLM_MASTER_KEY` (fails closed if unset)
+- **Health:** `GET /health/readiness`, `GET /health/liveness`, `GET /health/gil`
+- **Request logs:** POSTed to a LiteLLM proxy at `/v1/rust_control_plane/logs` (see [Request logging](#request-logging))
+
+> **Realtime serving is pure Rust.** Python is used at **load time only** — to
+> read the config once at boot. The realtime hot path never touches Python.
+
+## Configuration (config.yaml)
+
+The gateway loads its `model_list` from a **config.yaml**, the same as the
+LiteLLM proxy. Point `LITELLM_CONFIG_PATH` at the file:
+
+```yaml
+# config.yaml
+model_list:
+  - model_name: gpt-realtime
+    litellm_params:
+      model: openai/gpt-realtime
+      api_key: os.environ/OPENAI_API_KEY
+```
+
+```bash
+LITELLM_CONFIG_PATH=./config.yaml ./litellm-ai-gateway
+```
+
+At boot the gateway calls into `litellm.proxy.read_model_list`, which reuses the
+**real proxy config reader** (`ProxyConfig.get_config`). That means everything
+the proxy supports in config.yaml works here too:
+
+- `include:` to merge in other config files,
+- `os.environ/VAR` secret references (resolved via the secret manager, never
+  inlined),
+- DB-stored models (when a database is configured).
+
+Secrets stay out of the config — reference them with `os.environ/...` and set
+the env var at deploy time. The shipped Docker image is built with the
+`python-config` feature and **bundles litellm**, so config loading works out of
+the box; the default baked config lives at `/app/config.yaml` and can be
+overridden at deploy time (e.g. a Render secret file mounted at the same path).
+
+### Environment variables
+
+| Var | Required | Default | Purpose |
+|---|---|---|---|
+| `LITELLM_CONFIG_PATH` | yes (config mode) | — | Path to the config.yaml the gateway loads its `model_list` from. The Docker image defaults this to `/app/config.yaml`. |
+| `LITELLM_MASTER_KEY` | yes | — | Bearer token clients must send. Unset ⇒ all `/v1/realtime` requests are rejected (fail closed). |
+| `OPENAI_API_KEY` | yes | — | Upstream OpenAI key. Referenced by config.yaml as `os.environ/OPENAI_API_KEY` for the gateway→OpenAI dial. |
+| `HOST` | no | `127.0.0.1` | **Set to `0.0.0.0` in any container/deploy** or external traffic is refused. |
+| `PORT` | no | `4001` | Listen port. Render and most PaaS inject this automatically. |
+| `LITELLM_PROXY_BASE_URL` | no | `http://localhost:4000` | LiteLLM proxy that request logs are POSTed to. See [Request logging](#request-logging). |
+
+> Secrets (`LITELLM_MASTER_KEY`, `OPENAI_API_KEY`) are never baked into the image
+> or `render.yaml` — inject them at deploy time only.
+
+### Lean env stand-in (fallback)
+
+If the binary is built **without** `python-config` (default features), or
+`LITELLM_CONFIG_PATH` is unset, the gateway falls back to a single-deployment
+stand-in built from the environment:
+
+| Var | Default | Purpose |
+|---|---|---|
+| `OPENAI_REALTIME_MODEL` | `gpt-realtime` | The single deployment's model name (also the `?model=` clients pass). |
+
+This mode links no libpython and needs no config file, but it only supports one
+hard-coded OpenAI deployment. **config.yaml is the recommended path** — use the
+stand-in only for the leanest possible build.
+
+## Request logging
+
+The gateway runs no spend logic. When a session ends it builds one
+`StandardLoggingPayload` and POSTs it to `{LITELLM_PROXY_BASE_URL}/v1/rust_control_plane/logs`
+(admin-only, bearer = `LITELLM_MASTER_KEY`), and the proxy replays it through its
+normal callbacks (spend logs, Langfuse, etc.). The POST is non-blocking: a bounded
+channel drained by a background worker, dropping with a counter if the proxy is
+down. It sends one payload per session. Both env vars are in the table above.
+
+Worker tuning, rarely needed: `LITELLM_LOG_CHANNEL_CAPACITY` (4096),
+`LITELLM_LOG_BATCH_SIZE` (256), `LITELLM_LOG_FLUSH_INTERVAL_MS` (500).
+
+## Build & run with Docker
+
+The image is built `--features python-config` and installs litellm **from this
+repo's source** (the config reader is newer than any PyPI release), so the build
+**context is the repo root**:
+
+```bash
+# from the repo root
+docker build -f litellm-rust/crates/ai-gateway/Dockerfile -t litellm-ai-gateway .
+
+docker run --rm -p 4001:4001 \
+  -e HOST=0.0.0.0 -e PORT=4001 \
+  -e LITELLM_MASTER_KEY=sk-local \
+  -e OPENAI_API_KEY=$OPENAI_API_KEY \
+  litellm-ai-gateway          # LITELLM_CONFIG_PATH defaults to /app/config.yaml
+
+# smoke test
+curl -s -o /dev/null -w '%{http_code}\n' localhost:4001/health/readiness   # -> 200
+curl -s -o /dev/null -w '%{http_code}\n' localhost:4001/v1/realtime         # -> 401 (auth fails closed)
+```
+
+On boot you should see `loaded model_list from /app/config.yaml via python
+config reader` — that confirms the config path (not the env stand-in fallback).
+To use your own config, mount it over the default:
+
+```bash
+docker run --rm -p 4001:4001 \
+  -e HOST=0.0.0.0 -e LITELLM_MASTER_KEY=sk-local -e OPENAI_API_KEY=$OPENAI_API_KEY \
+  -v $(pwd)/my-config.yaml:/app/config.yaml:ro \
+  litellm-ai-gateway
+```
+
+### Cargo-only (no Docker)
+
+```bash
+# config.yaml mode — needs litellm importable in the active python env
+LITELLM_CONFIG_PATH=./crates/ai-gateway/config.yaml \
+  cargo run --release -p litellm-ai-gateway --features python-config
+
+# env stand-in mode — no python, no config
+cargo run --release -p litellm-ai-gateway
+```
+
+## Deploy on Render
+
+The service is a Docker **web service**; Render terminates TLS and supports
+WebSockets, so the public endpoint is `wss://<service>.onrender.com/v1/realtime`.
+
+### Option A — Blueprint (`render.yaml`)
+
+`crates/ai-gateway/render.yaml` describes the service (Docker runtime,
+`healthCheckPath: /health/readiness`, repo-root `dockerContext: .`,
+`dockerfilePath: ./litellm-rust/crates/ai-gateway/Dockerfile`,
+`LITELLM_CONFIG_PATH: /app/config.yaml`). `LITELLM_MASTER_KEY` and
+`OPENAI_API_KEY` are `sync: false` — set them in the dashboard after the first
+deploy. To use a non-default model_list, mount a **Render Secret File** at
+`/app/config.yaml`. Point a Render Blueprint at this repo/branch and apply.
+
+### Option B — Render API
+
+```bash
+# create a Docker web service from this repo+branch, then set env vars:
+curl -X POST https://api.render.com/v1/services \
+  -H "Authorization: Bearer $RENDER_API_KEY" -H "Content-Type: application/json" \
+  -d '{
+    "type": "web_service", "name": "litellm-rust-ai-gateway",
+    "ownerId": "<owner-id>", "repo": "https://github.com/BerriAI/litellm",
+    "branch": "<branch-with-this-dockerfile>",
+    "serviceDetails": {
+      "env": "docker",
+      "envSpecificDetails": {
+        "dockerfilePath": "./litellm-rust/crates/ai-gateway/Dockerfile",
+        "dockerContext": "."
+      },
+      "healthCheckPath": "/health/readiness"
+    }
+  }'
+# then set env vars LITELLM_MASTER_KEY, OPENAI_API_KEY, HOST=0.0.0.0,
+# LITELLM_CONFIG_PATH=/app/config.yaml
+```
+
+Health check path **must** be `/health/readiness`. `autoDeploy` is off by default
+in the blueprint — trigger deploys manually (or flip it on) to pick up new commits.
+
+## Scaling
+
+Concurrency is what matters, not total connections: each in-flight session holds
+one client socket + one upstream socket. To scale, raise the instance count /
+enable autoscaling on the Render service (e.g. baseline 10, max 100). Each
+instance needs file descriptors for `2 × peak_concurrent_sessions` — raise
+`ulimit -n` if you push very high concurrency.
+
+## Latency note
+
+The gateway adds the cost of one extra hop: client→gateway, then a fresh
+gateway→OpenAI realtime handshake (TLS + WS upgrade + `session.created`). In
+benchmarks this is ~100–150 ms of added session-establishment time; first-audio
+and steady-state streaming add no measurable overhead. To minimize it, deploy the
+gateway in the Render region with the lowest RTT to OpenAI's realtime endpoint.
--- a/litellm-rust/crates/ai-gateway/benchmarks/realtime/README.md
+++ b/litellm-rust/crates/ai-gateway/benchmarks/realtime/README.md
@ -0,0 +1,55 @@
+# Realtime gateway benchmark — pool on/off
+
+Measures what the gateway adds over talking to OpenAI's realtime WebSocket
+directly, and what the pre-warmed connection pool removes. See
+`../../src/routes/realtime/README.md` for how the pool works.
+
+## Results
+
+5000 calls / 500 concurrency, gateway at 10 instances, pool ON
+(`REALTIME_POOL_SIZE=64`), upstream OpenAI `gpt-realtime`. Each leg run twice.
+Times in **ms**. Phases per connection: **dial** = TCP+TLS+WS upgrade,
+**session** = upgrade → `session.created` (the phase the pool removes),
+**1st-audio** = `response.create` → first audio delta (OpenAI inference),
+**total** = full wall-clock.
+
+| metric             | Direct OpenAI | Gateway (pool ON) | Overhead (ms) | vs OpenAI  |
+| ------------------ | ------------- | ----------------- | ------------- | ---------- |
+| success rate (%)   | 99.8          | 99.8              | —             | —          |
+| dial p50 (ms)      | 276           | 158               | −118          | **faster** |
+| session p50 (ms)   | 7             | 0                 | −7            | **faster** |
+| 1st-audio p50 (ms) | 440           | 664               | +224          | slower¹    |
+| total p50 (ms)     | 816           | 1010              | +194          | slower¹    |
+| total p95 (ms)     | 2152          | 1970              | −182          | **faster** |
+| total p99 (ms)     | 2692          | 2610              | −82           | **faster** |
+
+The gateway is **faster than direct on 4 of 6 metrics**. The warm pool makes the
+**session phase sub-millisecond** at the median — ~76% of connects hit the pool,
+~70% had session < 1 ms. ¹ The two "slower" rows are not gateway overhead:
+`1st-audio` is OpenAI's own inference time (the gateway only relays it), which ran
+slower during the gateway legs and drags `total p50` with it.
+
+**Pool OFF** (control, `REALTIME_POOL_SIZE=0`): session p50 was **367 ms** — the
+fresh-dial overhead the pool removes.
+
+## Reproduce
+
+The load generator lives in a separate repo:
+**https://github.com/ishaan-berri/litellm-realtime-bench**
+
+```bash
+git clone https://github.com/ishaan-berri/litellm-realtime-bench
+cd litellm-realtime-bench && go build -o wsbench .
+
+# Direct to OpenAI (baseline)
+./wsbench -host api.openai.com -key "$OPENAI_API_KEY" -m gpt-realtime -n 5000 -c 500 -t 60
+
+# Through the gateway — run once with pool ON, once with REALTIME_POOL_SIZE=0
+./wsbench -host <gateway-host> -key "$LITELLM_MASTER_KEY" -m gpt-realtime -n 5000 -c 500 -t 60
+```
+
+Run the gateway with the env stand-in (`OPENAI_REALTIME_MODEL=gpt-realtime`,
+`OPENAI_API_KEY`, `LITELLM_MASTER_KEY`, `REALTIME_POOL_SIZE`, `HOST=0.0.0.0`). At
+500 concurrency over N instances, size the pool to `≈ 500 / N` per instance (64 was
+used here for 10 instances). The bench repo's README covers running 500-concurrency
+legs from a hosted multi-vCPU runner. **Never commit keys — pass them via `-key`.**
--- a/litellm-rust/crates/ai-gateway/config.yaml
+++ b/litellm-rust/crates/ai-gateway/config.yaml
@ -0,0 +1,13 @@
+# Sample realtime config for the LiteLLM Rust AI Gateway.
+#
+# The gateway loads this model_list at boot via the embedded python config
+# reader (litellm.proxy.read_model_list), which reuses the proxy's own reader —
+# so include:, os.environ/ secrets, and DB-stored models all work here too.
+#
+# Secrets are referenced (never inlined) via os.environ/. A real deploy can
+# override this file (e.g. mount a Render secret file at LITELLM_CONFIG_PATH).
+model_list:
+  - model_name: gpt-realtime
+    litellm_params:
+      model: openai/gpt-realtime
+      api_key: os.environ/OPENAI_API_KEY
--- a/litellm-rust/crates/ai-gateway/render.yaml
+++ b/litellm-rust/crates/ai-gateway/render.yaml
@ -0,0 +1,35 @@
+# Render blueprint for the LiteLLM Rust AI Gateway (realtime WebSocket proxy).
+#
+# Single instance for now (no autoscaling). The public endpoint is a
+# WebSocket served over TLS: wss://<service>.onrender.com/v1/realtime
+#
+# Paths are relative to the **repo root** (Render's convention). The build
+# context is the repo root so the image can install litellm from source — the
+# gateway loads its model_list via litellm.proxy.read_model_list at boot.
+#
+# Secrets (LITELLM_MASTER_KEY, OPENAI_API_KEY) are marked sync: false — set
+# them in the Render dashboard or via the API, never inline here.
+services:
+  - type: web
+    name: litellm-rust-ai-gateway
+    runtime: docker
+    plan: standard
+    dockerfilePath: ./litellm-rust/crates/ai-gateway/Dockerfile
+    dockerContext: .
+    healthCheckPath: /health/readiness
+    numInstances: 1
+    envVars:
+      # The gateway loads its model_list from this config.yaml via the embedded
+      # python config reader. The image bakes a default config at /app/config.yaml;
+      # a real deploy can override it by mounting a Render secret file at this
+      # same path (Dashboard → Environment → Secret Files) — never inline secrets.
+      - key: LITELLM_CONFIG_PATH
+        value: /app/config.yaml
+      - key: HOST
+        value: 0.0.0.0
+      # Bearer token clients must send on /v1/realtime (fail closed if unset).
+      - key: LITELLM_MASTER_KEY
+        sync: false
+      # Referenced by config.yaml as os.environ/OPENAI_API_KEY for the upstream dial.
+      - key: OPENAI_API_KEY
+        sync: false
--- a/litellm-rust/crates/ai-gateway/src/auth/mod.rs
+++ b/litellm-rust/crates/ai-gateway/src/auth/mod.rs
@ -0,0 +1,93 @@
+//! Gateway authentication, as an axum **extractor** (the idiomatic pattern —
+//! keeps handlers clean and auth testable).
+//!
+//! For now this is a single **master key**: any caller presenting it as
+//! `Authorization: Bearer <key>` may invoke the gateway. Per-key auth, budgets,
+//! and rate limits are delegated to the Python proxy in a later phase.
+//!
+//! A handler opts in by adding [`RequireMasterKey`] to its arguments; auth then
+//! runs during extraction, before the handler body. Routes never re-implement it.
+
+use axum::extract::FromRequestParts;
+use axum::http::header::AUTHORIZATION;
+use axum::http::request::Parts;
+use axum::http::StatusCode;
+use sha2::{Digest, Sha256};
+use subtle::ConstantTimeEq;
+
+use crate::state::AppState;
+
+/// SHA-256 hex digest of a token — the exact transform the Python proxy applies
+/// (`litellm.proxy.utils.hash_token`).
+///
+/// STRICT REQUIREMENT: a raw key (`LITELLM_MASTER_KEY`, a virtual key, …) must
+/// **never** leave this gateway in a log payload. Spend logs and every callback
+/// integration receive `user_api_key_hash`, so that field must be this hash, not
+/// the credential. Hashing here also means the value matches the key's hash in
+/// `LiteLLM_SpendLogs.api_key`, so realtime spend joins with the rest of LiteLLM.
+pub fn hash_token(token: &str) -> String {
+    let digest = Sha256::digest(token.as_bytes());
+    let mut hex = String::with_capacity(digest.len() * 2);
+    for byte in digest {
+        use std::fmt::Write;
+        let _ = write!(hex, "{byte:02x}");
+    }
+    hex
+}
+
+/// Extractor that requires the configured master key as a bearer token.
+///
+/// Rejections: `500` when no master key is configured (permanent
+/// misconfiguration, not a transient outage); `401` on a missing/incorrect
+/// token. The comparison is constant-time.
+pub struct RequireMasterKey;
+
+#[axum::async_trait]
+impl FromRequestParts<AppState> for RequireMasterKey {
+    type Rejection = (StatusCode, String);
+
+    async fn from_request_parts(
+        parts: &mut Parts,
+        state: &AppState,
+    ) -> Result<Self, Self::Rejection> {
+        let Some(expected) = state.master_key.as_deref() else {
+            return Err((
+                StatusCode::INTERNAL_SERVER_ERROR,
+                "gateway auth not configured (set LITELLM_MASTER_KEY)".to_string(),
+            ));
+        };
+        let provided = parts
+            .headers
+            .get(AUTHORIZATION)
+            .and_then(|value| value.to_str().ok())
+            .and_then(|value| value.strip_prefix("Bearer "))
+            .map(str::trim);
+        match provided {
+            Some(token) if bool::from(token.as_bytes().ct_eq(expected.as_bytes())) => Ok(Self),
+            _ => Err((
+                StatusCode::UNAUTHORIZED,
+                "missing or invalid bearer token".to_string(),
+            )),
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::hash_token;
+
+    #[test]
+    fn hash_token_matches_python_sha256_hexdigest() {
+        // Must equal hashlib.sha256("sk-1234".encode()).hexdigest() — the value
+        // the proxy stores in LiteLLM_SpendLogs.api_key.
+        assert_eq!(
+            hash_token("sk-1234"),
+            "88dc28d0f030c55ed4ab77ed8faf098196cb1c05df778539800c9f1243fe6b4b"
+        );
+        // 64 lowercase hex chars, and never the raw input.
+        let h = hash_token("sk-secret");
+        assert_eq!(h.len(), 64);
+        assert!(h.chars().all(|c| c.is_ascii_hexdigit()));
+        assert_ne!(h, "sk-secret");
+    }
+}
--- a/litellm-rust/crates/ai-gateway/src/constants.rs
+++ b/litellm-rust/crates/ai-gateway/src/constants.rs
@ -0,0 +1,29 @@
+//! Crate-level constants for the ai-gateway.
+//!
+//! Per `litellm-rust/CLAUDE.md`, magic numbers and fixed strings live here
+//! (the Rust mirror of Python's `litellm/constants.py`), not inline in feature
+//! modules. Env-overridable tunables keep their `DEFAULT_*` value here; the env
+//! read + fallback happens at the host/config layer.
+
+/// Default LiteLLM control-plane base URL for request-log egress when
+/// `LITELLM_PROXY_BASE_URL` is unset.
+pub(crate) const DEFAULT_PROXY_BASE_URL: &str = "http://localhost:4000";
+
+/// The logs ingest path appended to the proxy base. Not a tunable; it is the
+/// proxy's API contract (the rust-control-plane router on the Python proxy).
+pub(crate) const RUST_CONTROL_PLANE_LOGS_PATH: &str = "/v1/rust_control_plane/logs";
+
+/// Default bounded channel depth for the log-egress worker.
+/// Override: `LITELLM_LOG_CHANNEL_CAPACITY`.
+pub(crate) const DEFAULT_CHANNEL_CAPACITY: usize = 4096;
+
+/// Default max records POSTed per request to the control plane.
+/// Override: `LITELLM_LOG_BATCH_SIZE`.
+pub(crate) const DEFAULT_MAX_BATCH_SIZE: usize = 256;
+
+/// Default partial-batch flush cadence, in ms.
+/// Override: `LITELLM_LOG_FLUSH_INTERVAL_MS`.
+pub(crate) const DEFAULT_FLUSH_INTERVAL_MS: u64 = 500;
+
+/// Provider attributed to realtime sessions in the logging payload.
+pub(crate) const DEFAULT_PROVIDER: &str = "openai";
--- a/litellm-rust/crates/ai-gateway/src/gil.rs
+++ b/litellm-rust/crates/ai-gateway/src/gil.rs
@ -0,0 +1,58 @@
+//! GIL-activity tracking.
+//!
+//! Every acquisition of the Python GIL is recorded here so the `/health/gil`
+//! endpoint can report whether Python was touched recently. The design goal is
+//! that the GIL is acquired **only at load time** (config read) and never on the
+//! realtime hot path — polling this endpoint during traffic should show the
+//! count holding steady and `acquired_last_30s` falling to `false`.
+
+use std::sync::atomic::{AtomicU64, Ordering};
+use std::time::{SystemTime, UNIX_EPOCH};
+
+/// Window (seconds) for the "recently acquired" signal.
+pub const RECENT_WINDOW_SECS: u64 = 30;
+
+static GIL_ACQUISITIONS: AtomicU64 = AtomicU64::new(0);
+/// Unix seconds of the last acquisition; `0` means "never".
+static LAST_GIL_UNIX_SECS: AtomicU64 = AtomicU64::new(0);
+
+fn now_unix_secs() -> u64 {
+    SystemTime::now()
+        .duration_since(UNIX_EPOCH)
+        .map(|d| d.as_secs())
+        .unwrap_or(0)
+}
+
+/// Record that the GIL was just acquired. Call immediately before taking the GIL.
+///
+/// Only invoked under the `python-config` feature; without it the gateway never
+/// touches Python, so the recorder is unused (and the endpoint reports zero).
+#[cfg_attr(not(feature = "python-config"), allow(dead_code))]
+pub fn record_acquisition() {
+    GIL_ACQUISITIONS.fetch_add(1, Ordering::Relaxed);
+    LAST_GIL_UNIX_SECS.store(now_unix_secs(), Ordering::Relaxed);
+}
+
+/// Point-in-time view of GIL activity.
+pub struct GilSnapshot {
+    pub total_acquisitions: u64,
+    pub seconds_since_last: Option<u64>,
+    pub acquired_last_30s: bool,
+}
+
+/// Read the current GIL-activity snapshot.
+pub fn snapshot() -> GilSnapshot {
+    let total = GIL_ACQUISITIONS.load(Ordering::Relaxed);
+    let last = LAST_GIL_UNIX_SECS.load(Ordering::Relaxed);
+    let seconds_since_last = if last == 0 {
+        None
+    } else {
+        Some(now_unix_secs().saturating_sub(last))
+    };
+    let acquired_last_30s = seconds_since_last.is_some_and(|secs| secs <= RECENT_WINDOW_SECS);
+    GilSnapshot {
+        total_acquisitions: total,
+        seconds_since_last,
+        acquired_last_30s,
+    }
+}
--- a/litellm-rust/crates/ai-gateway/src/integrations/custom_logger.rs
+++ b/litellm-rust/crates/ai-gateway/src/integrations/custom_logger.rs
@ -0,0 +1,24 @@
+//! The `CustomLogger` trait — the Rust mirror of Python
+//! `litellm/integrations/custom_logger.py::CustomLogger`.
+//!
+//! Synchronous (no `async_trait`): callbacks are O(1) enqueue-and-return so the
+//! realtime splice never blocks on a logger. Default bodies are no-ops so a
+//! logger can implement only the events it cares about.
+
+use crate::integrations::types::{LogError, LoggingError, StandardLoggingPayload};
+
+pub trait CustomLogger: Send + Sync {
+    /// Record a successful call. Default: no-op.
+    fn log_success_event(&self, _payload: &StandardLoggingPayload) -> Result<(), LogError> {
+        Ok(())
+    }
+
+    /// Record a failed call. Default: no-op.
+    fn log_failure_event(
+        &self,
+        _payload: &StandardLoggingPayload,
+        _error: &LoggingError,
+    ) -> Result<(), LogError> {
+        Ok(())
+    }
+}
--- a/litellm-rust/crates/ai-gateway/src/integrations/litellm_python_proxy_api.rs
+++ b/litellm-rust/crates/ai-gateway/src/integrations/litellm_python_proxy_api.rs
@ -0,0 +1,209 @@
+//! A `CustomLogger` that ships finished events to the LiteLLM Python proxy's
+//! `/v1/rust_control_plane/logs` endpoint.
+//!
+//! The callback path is non-blocking: `log_success_event` / `log_failure_event`
+//! build a `LogRecord` and `try_send` it onto a bounded channel, returning a
+//! `LogError` (never panicking, never awaiting) if the channel is full or the
+//! worker has gone away. A spawned background worker drains the channel, batches
+//! records into `{"records":[...]}`, and POSTs them to the proxy with a pooled
+//! `reqwest::Client`.
+
+use std::sync::Arc;
+use std::time::Duration;
+
+use reqwest::Client;
+use tokio::sync::mpsc::{self, Receiver, Sender};
+use tokio::time::interval;
+
+use crate::constants::{
+    DEFAULT_CHANNEL_CAPACITY, DEFAULT_FLUSH_INTERVAL_MS, DEFAULT_MAX_BATCH_SIZE,
+    DEFAULT_PROXY_BASE_URL, RUST_CONTROL_PLANE_LOGS_PATH,
+};
+use crate::integrations::custom_logger::CustomLogger;
+use crate::integrations::types::{
+    CallbackLogsRequest, LogError, LogRecord, LoggingError, StandardLoggingPayload,
+};
+
+/// Egress worker tunables. Each field defaults to the matching `DEFAULT_*` const
+/// in `crate::constants` and is overridable via an env var (read once at logger
+/// construction).
+struct EgressTunables {
+    channel_capacity: usize,
+    max_batch_size: usize,
+    flush_interval: Duration,
+}
+
+impl EgressTunables {
+    fn from_env() -> Self {
+        Self {
+            channel_capacity: env_positive(
+                "LITELLM_LOG_CHANNEL_CAPACITY",
+                DEFAULT_CHANNEL_CAPACITY,
+            ),
+            max_batch_size: env_positive("LITELLM_LOG_BATCH_SIZE", DEFAULT_MAX_BATCH_SIZE),
+            flush_interval: Duration::from_millis(env_positive(
+                "LITELLM_LOG_FLUSH_INTERVAL_MS",
+                DEFAULT_FLUSH_INTERVAL_MS,
+            )),
+        }
+    }
+}
+
+/// Parse a positive integer env var, falling back to `default` on missing,
+/// unparseable, or non-positive values. Generic over the integer type so one
+/// helper serves both the `usize` capacities and the `u64` interval.
+fn env_positive<T>(name: &str, default: T) -> T
+where
+    T: std::str::FromStr + PartialOrd + From<u8>,
+{
+    let zero = T::from(0u8);
+    std::env::var(name)
+        .ok()
+        .and_then(|value| value.trim().parse::<T>().ok())
+        .filter(|n| *n > zero)
+        .unwrap_or(default)
+}
+
+/// Ships realtime logging events to the LiteLLM Python proxy.
+pub struct LiteLLMPythonProxyAPILogger {
+    sink: Sender<LogRecord>,
+}
+
+impl LiteLLMPythonProxyAPILogger {
+    /// Spawn the background worker and return a logger handle. `base` is the
+    /// proxy base URL (no trailing path); `master_key` is sent as a bearer token.
+    pub fn start(base: String, master_key: String) -> Arc<Self> {
+        let tunables = EgressTunables::from_env();
+        let (sink, receiver) = mpsc::channel::<LogRecord>(tunables.channel_capacity);
+        let url = format!(
+            "{}{}",
+            base.trim_end_matches('/'),
+            RUST_CONTROL_PLANE_LOGS_PATH
+        );
+        let client = Client::new();
+        tokio::spawn(worker_loop(
+            receiver,
+            client,
+            url,
+            master_key,
+            tunables.max_batch_size,
+            tunables.flush_interval,
+        ));
+        Arc::new(Self { sink })
+    }
+
+    /// Build a logger from the environment: `LITELLM_PROXY_BASE_URL` (default
+    /// `http://localhost:4000`) and `LITELLM_MASTER_KEY`.
+    ///
+    /// `LITELLM_PROXY_BASE_URL` is treated as the full base and the route is
+    /// appended verbatim, so if the proxy runs under a `SERVER_ROOT_PATH`
+    /// (e.g. served at `https://host/litellm`), include it in the base
+    /// (`LITELLM_PROXY_BASE_URL=https://host/litellm`) and the POST lands at
+    /// `https://host/litellm/v1/rust_control_plane/logs`.
+    pub fn from_env() -> Arc<Self> {
+        let base = std::env::var("LITELLM_PROXY_BASE_URL")
+            .ok()
+            .filter(|value| !value.trim().is_empty())
+            .unwrap_or_else(|| DEFAULT_PROXY_BASE_URL.to_string());
+        let key = std::env::var("LITELLM_MASTER_KEY").unwrap_or_default();
+        Self::start(base, key)
+    }
+
+    fn enqueue(&self, record: LogRecord) -> Result<(), LogError> {
+        self.sink.try_send(record).map_err(|err| match err {
+            mpsc::error::TrySendError::Full(_) => LogError::channel_full(),
+            mpsc::error::TrySendError::Closed(_) => LogError::channel_closed(),
+        })
+    }
+}
+
+impl CustomLogger for LiteLLMPythonProxyAPILogger {
+    fn log_success_event(&self, payload: &StandardLoggingPayload) -> Result<(), LogError> {
+        self.enqueue(LogRecord {
+            status: "success".to_string(),
+            payload: payload.clone(),
+            error: None,
+        })
+    }
+
+    fn log_failure_event(
+        &self,
+        payload: &StandardLoggingPayload,
+        error: &LoggingError,
+    ) -> Result<(), LogError> {
+        self.enqueue(LogRecord {
+            status: "failure".to_string(),
+            payload: payload.clone(),
+            error: Some(format!("{}: {}", error.kind, error.message)),
+        })
+    }
+}
+
+/// Drain the channel, batching records and POSTing them to the proxy. Exits when
+/// the channel is closed (all senders dropped) and drained.
+async fn worker_loop(
+    mut receiver: Receiver<LogRecord>,
+    client: Client,
+    url: String,
+    master_key: String,
+    max_batch_size: usize,
+    flush_interval: Duration,
+) {
+    let mut ticker = interval(flush_interval);
+    let mut batch: Vec<LogRecord> = Vec::with_capacity(max_batch_size);
+
+    loop {
+        tokio::select! {
+            maybe_record = receiver.recv() => {
+                match maybe_record {
+                    Some(record) => {
+                        batch.push(record);
+                        if batch.len() >= max_batch_size {
+                            flush(&client, &url, &master_key, &mut batch).await;
+                        }
+                    }
+                    None => {
+                        // Channel closed: flush remaining and exit.
+                        flush(&client, &url, &master_key, &mut batch).await;
+                        break;
+                    }
+                }
+            }
+            _ = ticker.tick() => {
+                flush(&client, &url, &master_key, &mut batch).await;
+            }
+        }
+    }
+}
+
+/// POST the current batch (if any), clearing it. Errors are logged, not fatal.
+async fn flush(client: &Client, url: &str, master_key: &str, batch: &mut Vec<LogRecord>) {
+    if batch.is_empty() {
+        return;
+    }
+    let records = std::mem::take(batch)
+        .into_iter()
+        .map(LogRecord::into_callback_record)
+        .collect();
+    let body = CallbackLogsRequest { records };
+
+    let response = client
+        .post(url)
+        .bearer_auth(master_key)
+        .json(&body)
+        .send()
+        .await;
+
+    match response {
+        Ok(resp) if resp.status().is_success() => {}
+        Ok(resp) => {
+            eprintln!(
+                "litellm-ai-gateway: callback logs POST returned {} to {url}",
+                resp.status()
+            );
+        }
+        Err(err) => {
+            eprintln!("litellm-ai-gateway: callback logs POST failed to {url}: {err}");
+        }
+    }
+}
--- a/litellm-rust/crates/ai-gateway/src/integrations/mod.rs
+++ b/litellm-rust/crates/ai-gateway/src/integrations/mod.rs
@ -0,0 +1,10 @@
+//! Pure-Rust logging integrations. Names map 1:1 to Python
+//! `litellm/integrations/`:
+//!   - [`custom_logger::CustomLogger`]  — the callback trait
+//!   - [`litellm_python_proxy_api::LiteLLMPythonProxyAPILogger`] — ships events
+//!     to the Python proxy's `/v1/callbacks/logs` endpoint
+//!   - [`types`] — the typed `StandardLoggingPayload` wire contract
+
+pub mod custom_logger;
+pub mod litellm_python_proxy_api;
+pub mod types;
--- a/litellm-rust/crates/ai-gateway/src/integrations/types.rs
+++ b/litellm-rust/crates/ai-gateway/src/integrations/types.rs
@ -0,0 +1,164 @@
+//! Typed payloads for the LiteLLM `/v1/callbacks/logs` realtime-logging contract.
+//!
+//! Field names below are the EXACT JSON keys the Python replay path + spend-logs
+//! builder read. Note the deliberate mix:
+//!   - `startTime` / `endTime` are camelCase (epoch f64 seconds)
+//!   - `response_cost` / `prompt_tokens` / etc. are snake_case
+//!
+//! Mirrors Python `litellm/integrations/` + the proxy `CallbackLogsRequest`
+//! contract 1:1.
+
+use serde::Serialize;
+use serde_json::Value;
+use std::collections::HashMap;
+
+/// Cumulative token usage for a realtime session.
+#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
+pub struct Usage {
+    pub prompt_tokens: u64,
+    pub completion_tokens: u64,
+    pub total_tokens: u64,
+}
+
+/// Cost-attribution metadata threaded from the authenticated request.
+#[derive(Clone, Debug, Default)]
+pub struct RequestMetadata {
+    pub user_api_key_hash: Option<String>,
+    pub user_api_key_user_id: Option<String>,
+    pub user_api_key_team_id: Option<String>,
+}
+
+/// A logging-callback failure (e.g. a custom logger raised). Mirrors the Python
+/// failure-event shape: a message plus an exception kind/class name.
+#[derive(Clone, Debug)]
+pub struct LoggingError {
+    pub message: String,
+    pub kind: String,
+}
+
+/// A non-fatal error returned by a `CustomLogger` when it cannot enqueue an
+/// event (channel full or the background worker has shut down).
+#[derive(Clone, Debug)]
+pub struct LogError {
+    pub message: String,
+    pub kind: String,
+}
+
+impl LogError {
+    pub fn channel_full() -> Self {
+        Self {
+            message: "logging channel is full; dropping record".to_string(),
+            kind: "ChannelFull".to_string(),
+        }
+    }
+
+    pub fn channel_closed() -> Self {
+        Self {
+            message: "logging channel is closed; worker has shut down".to_string(),
+            kind: "ChannelClosed".to_string(),
+        }
+    }
+}
+
+impl std::fmt::Display for LogError {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "{}: {}", self.kind, self.message)
+    }
+}
+
+impl std::error::Error for LogError {}
+
+/// Batch wrapper — the top-level request body.
+/// Matches Python `CallbackLogsRequest { records: list[CallbackLogRecord] }`.
+#[derive(Serialize)]
+pub struct CallbackLogsRequest {
+    pub records: Vec<CallbackLogRecord>,
+}
+
+/// One finished logging event.
+/// Matches `CallbackLogRecord { status, standard_logging_payload, error? }`.
+#[derive(Serialize)]
+pub struct CallbackLogRecord {
+    /// "success" | "failure". On "failure", `error` (or payload.error_str)
+    /// becomes the replayed exception string.
+    pub status: String,
+
+    pub standard_logging_payload: StandardLoggingPayload,
+
+    /// Only meaningful when status == "failure". Omitted on success.
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub error: Option<String>,
+}
+
+/// The self-describing payload. Field names are the EXACT JSON keys the Python
+/// replay path + spend-logs builder read.
+#[derive(Clone, Debug, Serialize)]
+pub struct StandardLoggingPayload {
+    pub id: String,
+    pub litellm_call_id: String,
+
+    /// e.g. "realtime", "acompletion". Falls back to "acompletion" if absent.
+    pub call_type: String,
+
+    pub model: String,
+    pub custom_llm_provider: String,
+
+    /// Spend ($) written to LiteLLM_SpendLogs.spend.
+    pub response_cost: f64,
+
+    pub prompt_tokens: u64,
+    pub completion_tokens: u64,
+    pub total_tokens: u64,
+
+    /// EPOCH SECONDS as float — camelCase keys, NOT snake_case.
+    #[serde(rename = "startTime")]
+    pub start_time: f64,
+    #[serde(rename = "endTime")]
+    pub end_time: f64,
+
+    pub stream: bool,
+
+    pub metadata: StandardLoggingMetadata,
+
+    /// Optional; stored as request input on the spend log row.
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub messages: Option<Value>,
+}
+
+/// Cost-attribution keys. The replayer maps these into litellm_params.metadata,
+/// which the spend-logs builder reads to set user / team_id / organization_id.
+#[derive(Clone, Debug, Serialize, Default)]
+pub struct StandardLoggingMetadata {
+    pub user_api_key_hash: Option<String>, // -> SpendLogs.api_key
+    pub user_api_key_user_id: Option<String>, // -> SpendLogs.user
+    pub user_api_key_team_id: Option<String>, // -> SpendLogs.team_id
+
+    // Optional but read by the builder; include when known:
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub user_api_key_alias: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub user_api_key_org_id: Option<String>, // -> SpendLogs.organization_id
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub user_api_key_end_user_id: Option<String>, // -> SpendLogs.end_user
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub spend_logs_metadata: Option<HashMap<String, Value>>,
+}
+
+/// The unit handed to a `CustomLogger` sink: a finished payload plus its status
+/// and (on failure) the replayed error string.
+#[derive(Clone, Debug)]
+pub struct LogRecord {
+    pub status: String,
+    pub payload: StandardLoggingPayload,
+    pub error: Option<String>,
+}
+
+impl LogRecord {
+    pub fn into_callback_record(self) -> CallbackLogRecord {
+        CallbackLogRecord {
+            status: self.status,
+            standard_logging_payload: self.payload,
+            error: self.error,
+        }
+    }
+}
--- a/litellm-rust/crates/ai-gateway/src/io/mod.rs
+++ b/litellm-rust/crates/ai-gateway/src/io/mod.rs
@ -0,0 +1,3 @@
+pub mod ocr;
+pub mod realtime;
+pub mod realtime_pool;
--- a/litellm-rust/crates/ai-gateway/src/io/ocr.rs
+++ b/litellm-rust/crates/ai-gateway/src/io/ocr.rs
@ -0,0 +1,127 @@
+//! End-to-end OCR orchestration.
+//!
+//! Owns the whole Mistral OCR call so the Python side stays a thin bridge:
+//! resolve the API key, build the URL + body via the pure transforms, POST it,
+//! and normalize the response. The HTTP client is built once and reused.
+
+use std::sync::OnceLock;
+use std::time::Duration;
+
+use litellm_core::error::CoreError;
+use litellm_core::ocr::transformation::OcrProviderConfig;
+use litellm_core::CoreResult;
+use serde_json::{Map, Value};
+
+use litellm_core::providers::mistral::ocr::transformation as mistral;
+use litellm_core::providers::mistral::ocr::transformation::MISTRAL_OCR_CONFIG;
+
+/// OCR over large documents can take a while; bound it generously rather than
+/// hanging forever on an unresponsive upstream. The client-level limit is the
+/// outer ceiling; callers can tighten it per request via ``run_ocr``'s ``timeout``.
+const OCR_TIMEOUT_SECS: u64 = 600;
+
+/// Maximum upstream body characters retained in error messages. OCR responses
+/// can echo document contents and prompts; keep enough for debugging without
+/// forwarding sensitive payloads across the host boundary.
+const ERROR_BODY_MAX_CHARS: usize = 256;
+
+/// Process-wide blocking HTTP client (connection pool + TLS reused across calls).
+fn http_client() -> &'static reqwest::blocking::Client {
+    static CLIENT: OnceLock<reqwest::blocking::Client> = OnceLock::new();
+    CLIENT.get_or_init(|| {
+        reqwest::blocking::Client::builder()
+            .timeout(Duration::from_secs(OCR_TIMEOUT_SECS))
+            .build()
+            .expect("failed to build reqwest client")
+    })
+}
+
+fn truncate_error_body(body: &str) -> String {
+    if body.chars().count() <= ERROR_BODY_MAX_CHARS {
+        return body.to_string();
+    }
+    let truncated: String = body.chars().take(ERROR_BODY_MAX_CHARS).collect();
+    format!("{truncated}... (truncated)")
+}
+
+/// Perform a Mistral OCR call end to end and return the normalized response as
+/// JSON (the shape the Python `OCRResponse` model expects).
+///
+/// Blocking: intended to be called with the GIL released from the Python bridge.
+pub fn run_ocr(
+    model: &str,
+    document: Value,
+    api_key: Option<&str>,
+    api_base: Option<&str>,
+    optional_params: Map<String, Value>,
+    timeout: Option<Duration>,
+) -> CoreResult<Value> {
+    let config = &MISTRAL_OCR_CONFIG;
+
+    let api_key = mistral::resolve_api_key(api_key, &|key| std::env::var(key).ok())?;
+    let url = mistral::complete_url(api_base);
+    let filtered_params = config.map_ocr_params(&optional_params);
+    let body = config
+        .transform_ocr_request(model, document, filtered_params)?
+        .data;
+
+    let mut request = http_client().post(&url).bearer_auth(&api_key).json(&body);
+    if let Some(duration) = timeout {
+        request = request.timeout(duration);
+    }
+
+    let response = request
+        .send()
+        .map_err(|err| CoreError::Network(err.to_string()))?;
+
+    let status = response.status();
+    let text = response
+        .text()
+        .map_err(|err| CoreError::Network(err.to_string()))?;
+
+    if !status.is_success() {
+        return Err(CoreError::Http {
+            status: status.as_u16(),
+            body: truncate_error_body(&text),
+        });
+    }
+
+    let response_json: Value = serde_json::from_str(&text)
+        .map_err(|err| CoreError::InvalidResponse(format!("invalid OCR response JSON: {err}")))?;
+
+    Ok(config
+        .transform_ocr_response(model, response_json)?
+        .into_json())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn truncate_error_body_passes_short_strings_through() {
+        let body = "Unauthorized";
+        assert_eq!(truncate_error_body(body), "Unauthorized");
+    }
+
+    #[test]
+    fn truncate_error_body_caps_long_payloads() {
+        let body = "x".repeat(ERROR_BODY_MAX_CHARS + 50);
+        let truncated = truncate_error_body(&body);
+
+        assert!(truncated.ends_with("... (truncated)"));
+        let prefix_chars = truncated
+            .strip_suffix("... (truncated)")
+            .expect("truncated marker present")
+            .chars()
+            .count();
+        assert_eq!(prefix_chars, ERROR_BODY_MAX_CHARS);
+    }
+
+    #[test]
+    fn truncate_error_body_does_not_split_multibyte_chars() {
+        let body = "é".repeat(ERROR_BODY_MAX_CHARS + 10);
+        let truncated = truncate_error_body(&body);
+        assert!(truncated.is_char_boundary(truncated.len()));
+    }
+}
--- a/litellm-rust/crates/ai-gateway/src/io/realtime.rs
+++ b/litellm-rust/crates/ai-gateway/src/io/realtime.rs
@ -0,0 +1,390 @@
+//! End-to-end OpenAI realtime invocation.
+//!
+//! The host-facing entry point, mirroring `crate::io::ocr::run_ocr`: open the
+//! WebSocket to OpenAI, then splice a client realtime stream to the upstream,
+//! driving typed events through the pure `OPENAI_REALTIME_CONFIG` transforms.
+//! Network, auth header, key resolution, and wire (de)serialization live here so
+//! the `transformation` module stays pure and typed.
+//!
+//! The dial and splice steps are factored out ([`dial_upstream`], [`splice`]) so
+//! the connection pool ([`crate::io::realtime_pool`]) can pre-establish an upstream,
+//! buffer its `session.created`, and later hand the live socket to the same
+//! splice loop a fresh dial uses.
+
+use std::time::Duration;
+
+use futures_util::stream::{SplitSink, SplitStream};
+use futures_util::{Sink, SinkExt, Stream, StreamExt};
+use litellm_core::error::CoreError;
+use litellm_core::realtime::transformation::RealtimeProviderConfig;
+use litellm_core::realtime::types::RealtimeEvent;
+use litellm_core::CoreResult;
+use tokio::net::TcpStream;
+use tokio_tungstenite::tungstenite::client::IntoClientRequest;
+use tokio_tungstenite::tungstenite::http::header::AUTHORIZATION;
+use tokio_tungstenite::tungstenite::http::HeaderValue;
+use tokio_tungstenite::tungstenite::Message;
+use tokio_tungstenite::{connect_async, MaybeTlsStream, WebSocketStream};
+
+use litellm_core::providers::openai::realtime::transformation::OPENAI_REALTIME_CONFIG;
+
+/// Environment variable holding the OpenAI API key (last-resort fallback).
+const OPENAI_API_KEY_ENV: &str = "OPENAI_API_KEY";
+
+const MISSING_KEY_MESSAGE: &str = "Missing OpenAI API Key - a realtime call is being made but no key was passed via params or the OPENAI_API_KEY environment variable";
+
+/// Default **idle** timeout: if neither side sends a frame for this long, the
+/// session is reaped. It resets on any activity, so it does not cap a healthy
+/// (continuously streaming) session — it only frees a stalled one (e.g. a
+/// half-open upstream that keeps the socket open but stops sending).
+const DEFAULT_IDLE_TIMEOUT_SECS: u64 = 300;
+
+/// The concrete upstream WebSocket type (TLS or plain). Shared by the dial path
+/// and the pool so warm sockets and fresh sockets are the exact same type.
+pub type UpstreamWs = WebSocketStream<MaybeTlsStream<TcpStream>>;
+pub(crate) type UpstreamTx = SplitSink<UpstreamWs, Message>;
+pub(crate) type UpstreamRx = SplitStream<UpstreamWs>;
+
+/// Resolve the OpenAI API key from the explicit param or the environment.
+///
+/// Blank/whitespace values are treated as absent (guard at resolution time).
+pub(crate) fn resolve_api_key(api_key: Option<&str>) -> CoreResult<String> {
+    api_key
+        .map(str::trim)
+        .filter(|key| !key.is_empty())
+        .map(str::to_string)
+        .or_else(|| {
+            std::env::var(OPENAI_API_KEY_ENV)
+                .ok()
+                .filter(|key| !key.trim().is_empty())
+        })
+        .ok_or_else(|| CoreError::Auth(MISSING_KEY_MESSAGE.to_string()))
+}
+
+/// Open the upstream WebSocket to OpenAI for `(model, api_key, api_base)`.
+///
+/// This is the dial half of [`realtime`], factored out so the pool can
+/// pre-establish sockets ahead of any client. `api_key` here is already resolved
+/// (non-blank) — the pool resolves it once when it is created.
+pub(crate) async fn dial_upstream(
+    model: &str,
+    api_key: &str,
+    api_base: Option<&str>,
+) -> CoreResult<UpstreamWs> {
+    let url = OPENAI_REALTIME_CONFIG.complete_url(api_base, model);
+
+    let mut request = url
+        .as_str()
+        .into_client_request()
+        .map_err(|err| CoreError::Network(err.to_string()))?;
+    // GA realtime: only Authorization. The legacy OpenAI-Beta header triggers
+    // beta_api_shape_disabled, so we do not send it.
+    request.headers_mut().insert(
+        AUTHORIZATION,
+        HeaderValue::from_str(&format!("Bearer {api_key}"))
+            .map_err(|err| CoreError::Auth(err.to_string()))?,
+    );
+
+    let (upstream, _response) = connect_async(request)
+        .await
+        .map_err(|err| CoreError::Network(err.to_string()))?;
+    Ok(upstream)
+}
+
+/// Read the next text frame from the upstream and decode it as a typed event.
+///
+/// Used by the pool to pre-read OpenAI's unprompted `session.created`. Returns an
+/// error on a non-text frame, a closed socket, or undecodable JSON so the pool can
+/// discard a misbehaving socket rather than warm it.
+pub(crate) async fn read_event(upstream_rx: &mut UpstreamRx) -> CoreResult<RealtimeEvent> {
+    loop {
+        let message = upstream_rx
+            .next()
+            .await
+            .ok_or_else(|| CoreError::Network("upstream closed before first event".to_string()))?
+            .map_err(|err| CoreError::Network(err.to_string()))?;
+        match message {
+            Message::Text(text) => {
+                return serde_json::from_str(&text)
+                    .map_err(|err| CoreError::InvalidResponse(err.to_string()));
+            }
+            // Ignore protocol frames (ping/pong) while waiting for the first event.
+            Message::Ping(_) | Message::Pong(_) => continue,
+            Message::Close(_) => {
+                return Err(CoreError::Network(
+                    "upstream closed before first event".to_string(),
+                ))
+            }
+            _ => continue,
+        }
+    }
+}
+
+/// Splice an already-connected upstream to the client streams.
+///
+/// `prelude` is relayed to the client first (the pool passes the buffered
+/// `session.created` here; the fresh-dial path passes `None` and lets the upstream
+/// deliver it). Then a single select loop forwards both directions through the
+/// transforms until either side closes or the idle timeout fires.
+/// `observe` is invoked on **upstream→client** events only (the trusted side that
+/// carries `session.created` and `response.done` usage) — never on client events,
+/// so a client cannot fabricate usage into its own logs.
+#[allow(clippy::too_many_arguments)]
+pub(crate) async fn splice<In, Out>(
+    model: &str,
+    mut upstream_tx: UpstreamTx,
+    mut upstream_rx: UpstreamRx,
+    prelude: Option<RealtimeEvent>,
+    idle_timeout: Option<Duration>,
+    mut observe: impl FnMut(&RealtimeEvent) + Send,
+    mut client_in: In,
+    mut client_out: Out,
+) -> CoreResult<()>
+where
+    In: Stream<Item = RealtimeEvent> + Unpin + Send,
+    Out: Sink<RealtimeEvent> + Unpin + Send,
+    <Out as Sink<RealtimeEvent>>::Error: std::fmt::Display,
+{
+    let config = &OPENAI_REALTIME_CONFIG;
+
+    // Relay a buffered backend event (warm handoff's session.created) first, so a
+    // warm session looks identical to a fresh one from the client's view.
+    if let Some(event) = prelude {
+        for outbound in config.transform_realtime_response(&event, model)?.events {
+            client_out
+                .send(outbound)
+                .await
+                .map_err(|err| CoreError::Network(err.to_string()))?;
+        }
+    }
+
+    let idle = idle_timeout.unwrap_or(Duration::from_secs(DEFAULT_IDLE_TIMEOUT_SECS));
+
+    // One loop forwarding both directions. The `sleep(idle)` arm is rebuilt every
+    // iteration, so any frame (either way) resets it — it fires only when the
+    // session has been fully idle for `idle`, reaping a stalled connection
+    // (task + upstream TCP socket) instead of leaking it.
+    loop {
+        tokio::select! {
+            // client -> upstream
+            client_event = client_in.next() => {
+                let Some(event) = client_event else { break }; // client disconnected
+                // NOTE: do NOT observe client events. session.created / response.done
+                // (carrying usage) are server→client events; observing the client arm
+                // would let an authenticated client POST a fabricated response.done and
+                // inflate its own spend log. Logging observes upstream events only.
+                for outbound in config.transform_realtime_request(&event, model)?.events {
+                    let payload = serde_json::to_string(&outbound)
+                        .map_err(|err| CoreError::InvalidResponse(err.to_string()))?;
+                    upstream_tx
+                        .send(Message::Text(payload))
+                        .await
+                        .map_err(|err| CoreError::Network(err.to_string()))?;
+                }
+            }
+            // upstream -> client
+            upstream_message = upstream_rx.next() => {
+                let Some(message) = upstream_message else { break }; // upstream closed
+                match message.map_err(|err| CoreError::Network(err.to_string()))? {
+                    Message::Text(text) => {
+                        let event: RealtimeEvent = serde_json::from_str(&text)
+                            .map_err(|err| CoreError::InvalidResponse(err.to_string()))?;
+                        observe(&event);
+                        for outbound in config.transform_realtime_response(&event, model)?.events {
+                            client_out
+                                .send(outbound)
+                                .await
+                                .map_err(|err| CoreError::Network(err.to_string()))?;
+                        }
+                    }
+                    Message::Close(_) => break,
+                    _ => {}
+                }
+            }
+            // idle timeout: no activity from either side within `idle`
+            _ = tokio::time::sleep(idle) => break,
+        }
+    }
+    Ok(())
+}
+
+/// Splice a client realtime stream to OpenAI: forward client events upstream
+/// (via `transform_realtime_request`) and backend events downstream (via
+/// `transform_realtime_response`). Returns when either side closes.
+///
+/// Generic over the client transport (typed events) so this crate stays
+/// framework-agnostic; the gateway adapts its axum socket to these. This is the
+/// fresh-dial path: dial, then splice. The pool's warm-handoff path skips the dial
+/// and calls [`splice`] directly with a buffered `session.created`.
+#[allow(clippy::too_many_arguments)]
+pub async fn realtime<In, Out>(
+    model: &str,
+    api_key: Option<&str>,
+    api_base: Option<&str>,
+    idle_timeout: Option<Duration>,
+    observe: impl FnMut(&RealtimeEvent) + Send,
+    client_in: In,
+    client_out: Out,
+) -> CoreResult<()>
+where
+    In: Stream<Item = RealtimeEvent> + Unpin + Send,
+    Out: Sink<RealtimeEvent> + Unpin + Send,
+    <Out as Sink<RealtimeEvent>>::Error: std::fmt::Display,
+{
+    let api_key = resolve_api_key(api_key)?;
+    let upstream = dial_upstream(model, &api_key, api_base).await?;
+    let (upstream_tx, upstream_rx) = upstream.split();
+    splice(
+        model,
+        upstream_tx,
+        upstream_rx,
+        None,
+        idle_timeout,
+        observe,
+        client_in,
+        client_out,
+    )
+    .await
+}
+
+/// Splice a pre-warmed upstream (taken from [`crate::io::realtime_pool`]) to the
+/// client. Relays the buffered `session.created` first, then splices exactly like
+/// the fresh-dial path — so a warm session is indistinguishable from a fresh one.
+#[allow(clippy::too_many_arguments)]
+pub async fn realtime_warm<In, Out>(
+    model: &str,
+    handoff: crate::io::realtime_pool::WarmHandoff,
+    idle_timeout: Option<Duration>,
+    observe: impl FnMut(&RealtimeEvent) + Send,
+    client_in: In,
+    client_out: Out,
+) -> CoreResult<()>
+where
+    In: Stream<Item = RealtimeEvent> + Unpin + Send,
+    Out: Sink<RealtimeEvent> + Unpin + Send,
+    <Out as Sink<RealtimeEvent>>::Error: std::fmt::Display,
+{
+    splice(
+        model,
+        handoff.tx,
+        handoff.rx,
+        Some(handoff.session_created),
+        idle_timeout,
+        observe,
+        client_in,
+        client_out,
+    )
+    .await
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn event(raw: &str) -> RealtimeEvent {
+        serde_json::from_str(raw).expect("valid event json")
+    }
+
+    #[test]
+    fn resolve_api_key_prefers_param_then_blank_falls_through() {
+        assert_eq!(resolve_api_key(Some("sk-test")).unwrap(), "sk-test");
+        // A blank param with no env set should error.
+        if std::env::var(OPENAI_API_KEY_ENV).is_err() {
+            assert!(resolve_api_key(Some("   ")).is_err());
+        }
+    }
+
+    /// Live end-to-end check against OpenAI. Ignored by default (CI never runs
+    /// it); run explicitly with `OPENAI_API_KEY` set:
+    ///   `cargo test -p litellm-ai-gateway --features server realtime_invokes_openai -- --ignored --nocapture`
+    #[tokio::test]
+    #[ignore = "hits the live OpenAI realtime API; needs OPENAI_API_KEY"]
+    async fn realtime_invokes_openai_and_responds() {
+        use futures_channel::mpsc;
+
+        let key =
+            std::env::var(OPENAI_API_KEY_ENV).expect("set OPENAI_API_KEY to run this ignored test");
+
+        // client -> provider (we hold `client_tx` to push events upstream)
+        let (mut client_tx, client_in) = mpsc::unbounded::<RealtimeEvent>();
+        // provider -> client (we hold `backend_rx` to read backend events)
+        let (client_out, mut backend_rx) = mpsc::unbounded::<RealtimeEvent>();
+
+        // Clone the key so the spawned task owns its `String` (no borrow across await).
+        let key_owned = key.clone();
+        let call = tokio::spawn(async move {
+            realtime(
+                "gpt-realtime",
+                Some(&key_owned),
+                None,
+                None,
+                |_| {},
+                client_in,
+                client_out,
+            )
+            .await
+        });
+
+        // 1. First backend event should be session.created.
+        let first = tokio::time::timeout(Duration::from_secs(30), backend_rx.next())
+            .await
+            .expect("timed out waiting for session.created")
+            .expect("backend stream closed before session.created");
+        assert_eq!(
+            first.event_type, "session.created",
+            "expected session.created, got: {}",
+            first.event_type
+        );
+
+        // 2. Ask for a short audio response.
+        client_tx
+            .send(event(
+                r#"{"type":"conversation.item.create","item":{"type":"message","role":"user","content":[{"type":"input_text","text":"Say hi."}]}}"#,
+            ))
+            .await
+            .expect("send conversation.item.create");
+        client_tx
+            .send(event(r#"{"type":"response.create"}"#))
+            .await
+            .expect("send response.create");
+
+        // 3. Read backend events; require a non-empty audio delta, then response.done.
+        let mut saw_audio_delta = false;
+        let mut saw_done = false;
+        for _ in 0..500 {
+            let next = tokio::time::timeout(Duration::from_secs(30), backend_rx.next()).await;
+            let event = match next {
+                Ok(Some(event)) => event,
+                Ok(None) => break,
+                Err(_) => panic!("timed out waiting for backend events"),
+            };
+            match event.event_type.as_str() {
+                "response.output_audio.delta" => {
+                    let delta = event
+                        .data
+                        .get("delta")
+                        .and_then(|value| value.as_str())
+                        .unwrap_or("");
+                    if !delta.is_empty() {
+                        saw_audio_delta = true;
+                    }
+                }
+                "response.done" => {
+                    saw_done = true;
+                    break;
+                }
+                _ => {}
+            }
+        }
+
+        assert!(
+            saw_audio_delta,
+            "expected a response.output_audio.delta with non-empty delta"
+        );
+        assert!(saw_done, "expected a response.done event");
+
+        // Drop the client sender so the provider's to_upstream side finishes.
+        drop(client_tx);
+        let _ = call.await;
+    }
+}
--- a/litellm-rust/crates/ai-gateway/src/io/realtime_pool.rs
+++ b/litellm-rust/crates/ai-gateway/src/io/realtime_pool.rs
@ -0,0 +1,712 @@
+//! Pre-warmed upstream realtime connection pool.
+//!
+//! The gateway's realtime overhead lives entirely in session establishment: on
+//! every client connect it dials a fresh upstream WS to OpenAI and waits for
+//! `session.created` before it can serve. This pool keeps a small set of upstream
+//! sockets **already connected and already past `session.created`** so a connect
+//! can be served from a warm socket and the handshake is off the critical path.
+//!
+//! Layering: this lives in the gateway's `io` module next to the dial/splice it
+//! reuses. The gateway holds an `Arc<RealtimePool>` in its state and asks for a
+//! warm socket per connect; on a miss it fresh-dials exactly as before. The pool
+//! is a latency optimization, never a correctness dependency — see the gateway's
+//! `src/routes/realtime/README.md`.
+//!
+//! ## Caveats (enforced here)
+//! - One warm socket serves exactly one session (realtime isn't multiplexed), so
+//!   the pool is sized to the connect *rate*, not concurrent connections.
+//! - `session.created` is pre-read once and buffered; nothing else is read from a
+//!   warm socket before handoff, so a warm session starts at OpenAI defaults just
+//!   like a fresh one (`session.update` semantics unchanged).
+//! - Warm sockets are short-lived (`max_idle`) and liveness-checked at handoff to
+//!   bound idle billing / dodge OpenAI's idle timeout.
+//! - On miss or dead socket the caller fresh-dials; the pool never blocks or fails
+//!   a connect because it is empty.
+
+use std::collections::HashMap;
+use std::sync::{Arc, Mutex};
+use std::time::{Duration, Instant};
+
+use futures_util::StreamExt;
+use litellm_core::realtime::types::RealtimeEvent;
+use litellm_core::CoreResult;
+
+use crate::io::realtime::{
+    dial_upstream, read_event, resolve_api_key, UpstreamRx, UpstreamTx, UpstreamWs,
+};
+
+/// Default target warm sockets per key when pooling is enabled.
+pub const DEFAULT_POOL_SIZE: usize = 4;
+
+/// Default max time a warm socket may sit before it is closed and replaced.
+pub const DEFAULT_MAX_IDLE: Duration = Duration::from_secs(30);
+
+/// Env var: target warm sockets per key. `0` disables pooling (fresh-dial only).
+pub const POOL_SIZE_ENV: &str = "REALTIME_POOL_SIZE";
+
+/// Env var: max warm-socket idle lifetime, in seconds.
+pub const MAX_IDLE_ENV: &str = "REALTIME_POOL_MAX_IDLE_SECS";
+
+/// How often the background replenisher wakes to top up and reap stale sockets.
+const REPLENISH_TICK: Duration = Duration::from_millis(250);
+
+/// Backoff floor after a key's warm-up dials all fail. The first failed pass
+/// waits this long before retrying that key.
+const BACKOFF_BASE: Duration = Duration::from_millis(500);
+
+/// Backoff ceiling. A key that keeps failing (invalid credentials, an
+/// unreachable upstream) is retried at most once per this interval — instead of
+/// firing `needed` concurrent TLS dials every 250 ms tick, which would hammer
+/// the upstream and risk rate-limit exhaustion that degrades valid cold-path
+/// traffic. Backoff resets the moment a dial for the key succeeds.
+const BACKOFF_MAX: Duration = Duration::from_secs(30);
+
+/// Identifies an upstream connection: the tuple that fully determines the dial.
+/// `api_key` is included so a warm socket is only ever reused for the same key
+/// (no cross-tenant reuse).
+#[derive(Clone, PartialEq, Eq, Hash)]
+pub struct UpstreamKey {
+    pub model: String,
+    pub api_key: String,
+    pub api_base: Option<String>,
+}
+
+impl std::fmt::Debug for UpstreamKey {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("UpstreamKey")
+            .field("model", &self.model)
+            .field("api_key", &"[REDACTED]")
+            .field("api_base", &self.api_base)
+            .finish()
+    }
+}
+
+/// A warm upstream: split halves + the buffered `session.created` + when it was
+/// warmed (for `max_idle` expiry).
+struct WarmConnection {
+    tx: UpstreamTx,
+    rx: UpstreamRx,
+    session_created: RealtimeEvent,
+    warmed_at: Instant,
+}
+
+/// A live upstream taken from the pool, ready to splice. The caller relays
+/// `session_created` to the client first, then splices `(tx, rx)` as usual.
+pub struct WarmHandoff {
+    pub tx: UpstreamTx,
+    pub rx: UpstreamRx,
+    pub session_created: RealtimeEvent,
+}
+
+/// Pool configuration, resolved once at startup from the environment.
+#[derive(Clone, Copy, Debug)]
+pub struct PoolConfig {
+    /// Target warm sockets per key. `0` disables pooling.
+    pub target_size: usize,
+    /// Max time a warm socket may sit before it is closed and replaced.
+    pub max_idle: Duration,
+}
+
+impl Default for PoolConfig {
+    fn default() -> Self {
+        Self {
+            target_size: DEFAULT_POOL_SIZE,
+            max_idle: DEFAULT_MAX_IDLE,
+        }
+    }
+}
+
+impl PoolConfig {
+    /// Read config from the environment, falling back to defaults. An invalid
+    /// value warns and uses the default rather than failing startup.
+    pub fn from_env() -> Self {
+        let target_size = match std::env::var(POOL_SIZE_ENV) {
+            Ok(raw) => raw.trim().parse().unwrap_or_else(|_| {
+                eprintln!("warning: {POOL_SIZE_ENV}={raw:?} is not a valid size; using {DEFAULT_POOL_SIZE}");
+                DEFAULT_POOL_SIZE
+            }),
+            Err(_) => DEFAULT_POOL_SIZE,
+        };
+        let max_idle = match std::env::var(MAX_IDLE_ENV) {
+            Ok(raw) => raw
+                .trim()
+                .parse()
+                .map(Duration::from_secs)
+                .unwrap_or_else(|_| {
+                    eprintln!(
+                        "warning: {MAX_IDLE_ENV}={raw:?} is not a valid number of seconds; using {}s",
+                        DEFAULT_MAX_IDLE.as_secs()
+                    );
+                    DEFAULT_MAX_IDLE
+                }),
+            Err(_) => DEFAULT_MAX_IDLE,
+        };
+        Self {
+            target_size,
+            max_idle,
+        }
+    }
+
+    /// Whether pooling is on (`target_size > 0`).
+    pub fn enabled(&self) -> bool {
+        self.target_size > 0
+    }
+}
+
+/// Per-key warm sockets, behind a single `Mutex`. Realtime warm sockets are few
+/// (the pool is small), so a plain mutex over a `VecDeque`-ish `Vec` is simpler
+/// and faster than sharding; contention is negligible at this scale.
+type Warm = HashMap<UpstreamKey, Vec<WarmConnection>>;
+
+/// Per-key replenish backoff. Absent (or `consecutive_failures == 0`) means the
+/// key is healthy and replenished every tick. After a pass whose dials all fail,
+/// `retry_after` is pushed out with exponential backoff so a broken key (invalid
+/// credentials, unreachable upstream) is not re-dialed on every 250 ms tick.
+#[derive(Default)]
+struct Backoff {
+    /// Don't attempt warm-up dials for this key until this instant. `None` =
+    /// eligible now.
+    retry_after: Option<Instant>,
+    consecutive_failures: u32,
+}
+
+type Backoffs = HashMap<UpstreamKey, Backoff>;
+
+/// Pre-warmed upstream realtime connection pool.
+///
+/// Cheap to clone-via-`Arc`. The background replenisher is spawned by
+/// [`RealtimePool::spawn`]; a pool built with [`RealtimePool::disabled`] never
+/// warms anything and every `take` misses (callers fresh-dial).
+pub struct RealtimePool {
+    config: PoolConfig,
+    warm: Mutex<Warm>,
+    /// Per-key replenish backoff so a broken key doesn't trigger unbounded
+    /// concurrent dials every tick. Separate lock from `warm` so the request
+    /// hot path (`take`) never contends on it.
+    backoff: Mutex<Backoffs>,
+}
+
+impl RealtimePool {
+    /// A disabled pool: no background task, every `take` returns `None`.
+    pub fn disabled() -> Arc<Self> {
+        Arc::new(Self {
+            config: PoolConfig {
+                target_size: 0,
+                ..PoolConfig::default()
+            },
+            warm: Mutex::new(HashMap::new()),
+            backoff: Mutex::new(HashMap::new()),
+        })
+    }
+
+    /// Build a pool from config **without** the background replenisher. The pool
+    /// only warms when [`RealtimePool::warm_now`] is called. Used by deterministic
+    /// unit tests; production uses [`RealtimePool::spawn`].
+    #[cfg(test)]
+    fn new_unspawned(config: PoolConfig) -> Arc<Self> {
+        Arc::new(Self {
+            config,
+            warm: Mutex::new(HashMap::new()),
+            backoff: Mutex::new(HashMap::new()),
+        })
+    }
+
+    /// Build a pool from config and, if enabled, spawn the background replenisher.
+    /// Returns the shared handle the gateway stores in its state.
+    pub fn spawn(config: PoolConfig) -> Arc<Self> {
+        let pool = Arc::new(Self {
+            config,
+            warm: Mutex::new(HashMap::new()),
+            backoff: Mutex::new(HashMap::new()),
+        });
+        if config.enabled() {
+            let weak = Arc::downgrade(&pool);
+            tokio::spawn(async move {
+                let mut tick = tokio::time::interval(REPLENISH_TICK);
+                tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
+                loop {
+                    tick.tick().await;
+                    // Stop once the gateway has dropped its handle.
+                    let Some(pool) = weak.upgrade() else { break };
+                    pool.replenish_all().await;
+                }
+            });
+        }
+        pool
+    }
+
+    /// Resolved config (test/inspection).
+    pub fn config(&self) -> PoolConfig {
+        self.config
+    }
+
+    /// Register a key so the replenisher starts warming it. Idempotent. The
+    /// gateway calls this once per known deployment at startup; the pool only
+    /// warms keys it has seen, so it never dials a model nobody asked for.
+    pub fn register(&self, key: UpstreamKey) {
+        if !self.config.enabled() {
+            return;
+        }
+        self.warm.lock().unwrap().entry(key).or_default();
+    }
+
+    /// Take a warm, live socket for `key`, or `None` on miss / dead socket.
+    ///
+    /// Pops the freshest non-expired socket and liveness-checks it; a socket that
+    /// is too old or already dead is dropped (closing it) and the next candidate
+    /// tried. Never blocks: if nothing warm is live, returns `None` so the caller
+    /// fresh-dials.
+    pub fn take(&self, key: &UpstreamKey) -> Option<WarmHandoff> {
+        if !self.config.enabled() {
+            return None;
+        }
+        loop {
+            let mut candidate = {
+                let mut warm = self.warm.lock().unwrap();
+                let bucket = warm.get_mut(key)?;
+                bucket.pop()?
+            };
+            // Discard sockets past their warm lifetime (idle-billing guard).
+            if candidate.warmed_at.elapsed() > self.config.max_idle {
+                continue; // drops `candidate`, closing the socket
+            }
+            // Liveness: a non-blocking check that the socket hasn't already
+            // delivered a Close/Err. A warm socket should be silent after
+            // session.created, so anything pending means it is unhealthy.
+            if is_dead(&mut candidate.rx) {
+                continue;
+            }
+            return Some(WarmHandoff {
+                tx: candidate.tx,
+                rx: candidate.rx,
+                session_created: candidate.session_created,
+            });
+        }
+    }
+
+    /// One replenish pass over every registered key: reap stale sockets, then
+    /// dial up to `target_size`. Dials run concurrently; failures are swallowed
+    /// (a key that can't be warmed just keeps fresh-dialing on the request path)
+    /// and put the key into exponential backoff so a broken key isn't re-dialed
+    /// on every tick.
+    async fn replenish_all(&self) {
+        let keys: Vec<UpstreamKey> = { self.warm.lock().unwrap().keys().cloned().collect() };
+        for key in keys {
+            self.reap_stale(&key);
+            // Skip keys still in backoff from a prior all-failed pass — this is
+            // what bounds dials against an invalid/unreachable key to once per
+            // `BACKOFF_MAX` instead of `needed` dials every 250 ms tick.
+            if self.in_backoff(&key) {
+                continue;
+            }
+            let needed = {
+                let warm = self.warm.lock().unwrap();
+                let have = warm.get(&key).map(Vec::len).unwrap_or(0);
+                self.config.target_size.saturating_sub(have)
+            };
+            if needed == 0 {
+                continue;
+            }
+            // Dial the missing sockets CONCURRENTLY. A sequential loop here makes
+            // a full refill cost `needed × handshake` (~needed × 350 ms), which
+            // can't keep up with a high connect rate — the pool drains faster
+            // than it refills and most connects miss. Firing the dials together
+            // refills in ~one handshake window, keeping warm supply ≈ peak
+            // concurrent connects so the sub-ms warm handoff becomes the median,
+            // not the lucky-hit tail.
+            let dials = (0..needed).map(|_| warm_one(&key));
+            let results = futures_util::future::join_all(dials).await;
+            let mut any_ok = false;
+            // `.flatten()` keeps only the successful dials; a key that can't be
+            // warmed just keeps fresh-dialing on the request path.
+            for conn in results.into_iter().flatten() {
+                any_ok = true;
+                self.warm
+                    .lock()
+                    .unwrap()
+                    .entry(key.clone())
+                    .or_default()
+                    .push(conn);
+            }
+            // Reset backoff on any success; otherwise grow it. We only ever enter
+            // backoff when a pass that *attempted* dials produced none — a `needed
+            // == 0` pass is handled by the `continue` above and never touches it.
+            self.record_replenish_outcome(&key, any_ok);
+        }
+    }
+
+    /// Whether `key` is currently in a backoff window (a prior pass failed and
+    /// the retry time hasn't arrived). Eligible keys are pruned from the backoff
+    /// map so it doesn't grow unbounded for healthy keys.
+    fn in_backoff(&self, key: &UpstreamKey) -> bool {
+        let mut backoff = self.backoff.lock().unwrap();
+        match backoff.get(key).and_then(|b| b.retry_after) {
+            Some(retry_after) if Instant::now() < retry_after => true,
+            Some(_) => {
+                // Window elapsed — allow the attempt. Keep the failure count so a
+                // still-broken key backs off further, but clear the gate so this
+                // tick proceeds.
+                if let Some(b) = backoff.get_mut(key) {
+                    b.retry_after = None;
+                }
+                false
+            }
+            None => false,
+        }
+    }
+
+    /// Update a key's backoff after a replenish attempt. Success clears it;
+    /// failure grows the retry delay exponentially up to `BACKOFF_MAX`.
+    fn record_replenish_outcome(&self, key: &UpstreamKey, any_ok: bool) {
+        let mut backoff = self.backoff.lock().unwrap();
+        if any_ok {
+            backoff.remove(key);
+            return;
+        }
+        let entry = backoff.entry(key.clone()).or_default();
+        entry.consecutive_failures = entry.consecutive_failures.saturating_add(1);
+        // Exponential: BASE * 2^(failures-1), saturating at MAX. `min` of the
+        // shift exponent keeps the doubling from overflowing.
+        let shift = (entry.consecutive_failures - 1).min(16);
+        let delay = BACKOFF_BASE.saturating_mul(1u32 << shift).min(BACKOFF_MAX);
+        entry.retry_after = Some(Instant::now() + delay);
+    }
+
+    /// Drop sockets past `max_idle` or already dead for a key.
+    fn reap_stale(&self, key: &UpstreamKey) {
+        let mut warm = self.warm.lock().unwrap();
+        if let Some(bucket) = warm.get_mut(key) {
+            bucket.retain_mut(|conn| {
+                conn.warmed_at.elapsed() <= self.config.max_idle && !is_dead(&mut conn.rx)
+            });
+        }
+    }
+
+    /// Test/inspection: number of warm sockets currently held for `key`.
+    #[cfg(test)]
+    pub fn warm_len(&self, key: &UpstreamKey) -> usize {
+        self.warm
+            .lock()
+            .unwrap()
+            .get(key)
+            .map(Vec::len)
+            .unwrap_or(0)
+    }
+
+    /// Test/inspection: consecutive replenish failures recorded for `key` (0 if
+    /// the key is healthy / has no backoff entry).
+    #[cfg(test)]
+    pub fn backoff_failures(&self, key: &UpstreamKey) -> u32 {
+        self.backoff
+            .lock()
+            .unwrap()
+            .get(key)
+            .map(|b| b.consecutive_failures)
+            .unwrap_or(0)
+    }
+
+    /// Test helper: synchronously warm `target_size` sockets for `key` (no
+    /// background task). Lets tests assert handoff behavior deterministically.
+    #[cfg(test)]
+    pub async fn warm_now(&self, key: &UpstreamKey) {
+        let needed = {
+            let warm = self.warm.lock().unwrap();
+            let have = warm.get(key).map(Vec::len).unwrap_or(0);
+            self.config.target_size.saturating_sub(have)
+        };
+        for _ in 0..needed {
+            if let Ok(conn) = warm_one(key).await {
+                self.warm
+                    .lock()
+                    .unwrap()
+                    .entry(key.clone())
+                    .or_default()
+                    .push(conn);
+            }
+        }
+    }
+
+    /// Test helper: insert an already-built warm connection (used to inject a
+    /// dead socket and assert it is discarded at handoff).
+    #[cfg(test)]
+    fn insert_warm(&self, key: UpstreamKey, conn: WarmConnection) {
+        self.warm.lock().unwrap().entry(key).or_default().push(conn);
+    }
+}
+
+/// Dial one upstream and pre-read its `session.created` into a [`WarmConnection`].
+///
+/// `key.api_key` is already resolved (non-blank). The first frame OpenAI sends
+/// unprompted is `session.created`; we buffer exactly that and read nothing more.
+async fn warm_one(key: &UpstreamKey) -> CoreResult<WarmConnection> {
+    let upstream: UpstreamWs =
+        dial_upstream(&key.model, &key.api_key, key.api_base.as_deref()).await?;
+    let (tx, mut rx) = upstream.split();
+    let session_created = read_event(&mut rx).await?;
+    Ok(WarmConnection {
+        tx,
+        rx,
+        session_created,
+        warmed_at: Instant::now(),
+    })
+}
+
+/// Resolve a deployment's API key into the pool key, returning `None` when no key
+/// can be resolved (those deployments simply aren't pooled — the request path
+/// still fresh-dials and surfaces the auth error there).
+pub fn upstream_key(
+    model: &str,
+    api_key: Option<&str>,
+    api_base: Option<&str>,
+) -> Option<UpstreamKey> {
+    let api_key = resolve_api_key(api_key).ok()?;
+    Some(UpstreamKey {
+        model: model.to_string(),
+        api_key,
+        api_base: api_base.map(str::to_string),
+    })
+}
+
+/// Non-blocking liveness check: poll the upstream once. A warm socket is silent
+/// after `session.created`, so a pending `Close`/`Err`/`None` means it is dead.
+/// A pending data frame (shouldn't happen pre-handoff) is also treated as
+/// unhealthy — we'd rather discard and fresh-dial than hand over a socket in an
+/// unexpected state. `Pending` (the healthy case) returns `false`.
+fn is_dead(rx: &mut UpstreamRx) -> bool {
+    use futures_util::task::noop_waker_ref;
+    use futures_util::Stream;
+    use std::pin::Pin;
+    use std::task::{Context, Poll};
+
+    let mut cx = Context::from_waker(noop_waker_ref());
+    match Pin::new(rx).poll_next(&mut cx) {
+        Poll::Pending => false,
+        Poll::Ready(None) => true,
+        Poll::Ready(Some(Err(_))) => true,
+        // Any frame arriving before handoff is unexpected for a silent warm
+        // socket; treat it as unhealthy.
+        Poll::Ready(Some(Ok(_))) => true,
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use futures_util::SinkExt;
+    use std::net::SocketAddr;
+    use tokio::net::TcpListener;
+    use tokio_tungstenite::tungstenite::Message;
+
+    /// An in-process fake OpenAI realtime WS server. On connect it sends
+    /// `session.created`; on `response.create` it sends `response.created` +
+    /// `response.output_audio.delta` + `response.done`. Returns its `ws://` base.
+    async fn spawn_fake_openai() -> String {
+        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
+        let addr: SocketAddr = listener.local_addr().unwrap();
+        tokio::spawn(async move {
+            while let Ok((stream, _)) = listener.accept().await {
+                tokio::spawn(handle_fake_conn(stream));
+            }
+        });
+        format!("ws://{addr}")
+    }
+
+    async fn handle_fake_conn(stream: tokio::net::TcpStream) {
+        let mut ws = match tokio_tungstenite::accept_async(stream).await {
+            Ok(ws) => ws,
+            Err(_) => return,
+        };
+        // Unprompted session.created, exactly like OpenAI.
+        let _ = ws
+            .send(Message::Text(
+                r#"{"type":"session.created","session":{"id":"sess_fake"}}"#.to_string(),
+            ))
+            .await;
+        while let Some(Ok(msg)) = ws.next().await {
+            if let Message::Text(text) = msg {
+                if text.contains("response.create") {
+                    for frame in [
+                        r#"{"type":"response.created"}"#,
+                        r#"{"type":"response.output_audio.delta","delta":"AAAA"}"#,
+                        r#"{"type":"response.done"}"#,
+                    ] {
+                        let _ = ws.send(Message::Text(frame.to_string())).await;
+                    }
+                }
+            }
+        }
+    }
+
+    fn test_config() -> PoolConfig {
+        PoolConfig {
+            target_size: 2,
+            max_idle: Duration::from_secs(30),
+        }
+    }
+
+    fn key_for(base: &str) -> UpstreamKey {
+        UpstreamKey {
+            model: "gpt-realtime".to_string(),
+            api_key: "sk-test".to_string(),
+            api_base: Some(base.to_string()),
+        }
+    }
+
+    #[tokio::test]
+    async fn warm_handoff_relays_buffered_session_created() {
+        let base = spawn_fake_openai().await;
+        let pool = RealtimePool::new_unspawned(test_config());
+        let key = key_for(&base);
+        pool.register(key.clone());
+        pool.warm_now(&key).await;
+        assert_eq!(pool.warm_len(&key), 2);
+
+        let handoff = pool.take(&key).expect("a warm socket should be available");
+        assert_eq!(handoff.session_created.event_type, "session.created");
+        assert_eq!(
+            handoff
+                .session_created
+                .data
+                .get("session")
+                .and_then(|s| s.get("id"))
+                .and_then(|v| v.as_str()),
+            Some("sess_fake")
+        );
+        // Taking one leaves one.
+        assert_eq!(pool.warm_len(&key), 1);
+    }
+
+    #[tokio::test]
+    async fn pool_miss_returns_none_for_fresh_dial_fallback() {
+        let base = spawn_fake_openai().await;
+        let pool = RealtimePool::new_unspawned(test_config());
+        let key = key_for(&base);
+        // Registered but never warmed → empty bucket → miss.
+        pool.register(key.clone());
+        assert!(pool.take(&key).is_none());
+
+        // Unknown key → miss.
+        let other = key_for("ws://127.0.0.1:1");
+        assert!(pool.take(&other).is_none());
+    }
+
+    #[tokio::test]
+    async fn disabled_pool_never_hands_off() {
+        let pool = RealtimePool::disabled();
+        let key = key_for("ws://127.0.0.1:1");
+        pool.register(key.clone());
+        assert_eq!(pool.warm_len(&key), 0);
+        assert!(pool.take(&key).is_none());
+    }
+
+    #[tokio::test]
+    async fn dead_warm_socket_is_discarded() {
+        let base = spawn_fake_openai().await;
+        let pool = RealtimePool::new_unspawned(test_config());
+        let key = key_for(&base);
+        pool.register(key.clone());
+
+        // Build one real warm connection, then kill the upstream by dropping the
+        // server side: easiest is to dial, read session.created, then close our
+        // own rx's peer. Instead we forge "dead" via an already-closed socket:
+        // dial a connection and immediately send a Close from the client side so
+        // the server closes back, then warm it. Simpler: warm normally, then
+        // mark it stale by backdating warmed_at past max_idle and confirm it's
+        // dropped — that exercises the same discard path.
+        let mut conn = warm_one(&key).await.expect("warm one");
+        conn.warmed_at = Instant::now() - Duration::from_secs(3600); // past max_idle
+        pool.insert_warm(key.clone(), conn);
+        assert_eq!(pool.warm_len(&key), 1);
+
+        // take() must discard the stale socket and report a miss.
+        assert!(pool.take(&key).is_none());
+        assert_eq!(pool.warm_len(&key), 0);
+    }
+
+    #[tokio::test]
+    async fn background_replenisher_tops_up_registered_key() {
+        let base = spawn_fake_openai().await;
+        let pool = RealtimePool::spawn(test_config());
+        let key = key_for(&base);
+        pool.register(key.clone());
+
+        // Wait (bounded) for the background task to reach the target size.
+        let mut warmed = 0;
+        for _ in 0..40 {
+            tokio::time::sleep(Duration::from_millis(50)).await;
+            warmed = pool.warm_len(&key);
+            if warmed >= test_config().target_size {
+                break;
+            }
+        }
+        assert_eq!(
+            warmed,
+            test_config().target_size,
+            "background replenisher should warm up to target_size"
+        );
+        let handoff = pool.take(&key).expect("a warm socket should be available");
+        assert_eq!(handoff.session_created.event_type, "session.created");
+    }
+
+    #[tokio::test]
+    async fn closed_upstream_socket_is_detected_dead() {
+        // A genuinely dead socket: dial the fake, read session.created, then drop
+        // the server by closing from our side and waiting for the close to land.
+        let base = spawn_fake_openai().await;
+        let pool = RealtimePool::new_unspawned(test_config());
+        let key = key_for(&base);
+        pool.register(key.clone());
+
+        let mut conn = warm_one(&key).await.expect("warm one");
+        // Close the upstream from the client side; the server echoes a close.
+        let _ = conn.tx.send(Message::Close(None)).await;
+        // Give the close a moment to arrive on rx.
+        tokio::time::sleep(Duration::from_millis(50)).await;
+        pool.insert_warm(key.clone(), conn);
+
+        // Liveness check at take() should detect the close and discard it.
+        assert!(pool.take(&key).is_none());
+        assert_eq!(pool.warm_len(&key), 0);
+    }
+
+    #[tokio::test]
+    async fn broken_key_backs_off_instead_of_dialing_every_tick() {
+        // A key whose upstream is unreachable: every warm-up dial fails.
+        let pool = RealtimePool::new_unspawned(test_config());
+        let key = key_for("ws://127.0.0.1:1"); // nothing listens here
+        pool.register(key.clone());
+
+        // First pass attempts dials, they all fail → key enters backoff, no warm
+        // sockets, one recorded failure.
+        pool.replenish_all().await;
+        assert_eq!(pool.warm_len(&key), 0);
+        assert_eq!(pool.backoff_failures(&key), 1);
+        assert!(
+            pool.in_backoff(&key),
+            "a key whose dials all failed must be in backoff"
+        );
+
+        // An immediate next pass must be SKIPPED (still in the backoff window), so
+        // it does NOT fire another round of dials — the failure count is unchanged.
+        pool.replenish_all().await;
+        assert_eq!(
+            pool.backoff_failures(&key),
+            1,
+            "replenish during the backoff window must not re-dial the broken key"
+        );
+    }
+
+    #[tokio::test]
+    async fn healthy_key_never_enters_backoff_and_clears_after_recovery() {
+        let base = spawn_fake_openai().await;
+        let pool = RealtimePool::new_unspawned(test_config());
+        let key = key_for(&base);
+        pool.register(key.clone());
+
+        // A reachable upstream: the pass succeeds, so the key is never backed off.
+        pool.replenish_all().await;
+        assert_eq!(pool.warm_len(&key), test_config().target_size);
+        assert_eq!(pool.backoff_failures(&key), 0);
+        assert!(!pool.in_backoff(&key));
+    }
+}
--- a/litellm-rust/crates/ai-gateway/src/lib.rs
+++ b/litellm-rust/crates/ai-gateway/src/lib.rs
@ -0,0 +1,38 @@
+//! LiteLLM AI Gateway library.
+//!
+//! Two layers, split by feature so the Python `cdylib` can depend on the I/O
+//! without pulling in the HTTP server:
+//!
+//! - [`io`]: all network I/O (OCR HTTP call, realtime WebSocket splice, the
+//!   pre-warmed realtime pool). Always available — no feature required. The
+//!   Python bridge links this for `run_ocr`.
+//! - The server modules ([`auth`], [`routes`], [`state`]) and anything pulling
+//!   `axum` are gated behind the `server` feature, which the `litellm-ai-gateway`
+//!   binary turns on. The `python-config` feature additionally pulls in [`python`]
+//!   for the load-time config reader.
+
+pub mod io;
+
+/// GIL-activity tracking. Pure (atomics only); shared by the `server` routes and
+/// the `python-config` reader, so it is available without either feature.
+pub mod gil;
+
+#[cfg(feature = "server")]
+pub mod auth;
+#[cfg(feature = "server")]
+pub mod routes;
+#[cfg(feature = "server")]
+pub mod state;
+
+// Realtime request logging. Only the server serves realtime, so these are
+// `server`-gated; `io::realtime` exposes the generic `observe` hook while the
+// collector and callback fan-out live here.
+#[cfg(feature = "server")]
+mod constants;
+#[cfg(feature = "server")]
+pub mod integrations;
+#[cfg(feature = "server")]
+mod realtime;
+
+#[cfg(feature = "python-config")]
+pub mod python;
--- a/litellm-rust/crates/ai-gateway/src/main.rs
+++ b/litellm-rust/crates/ai-gateway/src/main.rs
@ -0,0 +1,162 @@
+//! LiteLLM AI Gateway — a minimal Axum server fronting the Rust router.
+//!
+//! Flow: client → `POST /v1/realtime` → `router.realtime()` selects a deployment
+//! (simple-shuffle) → `io::realtime::realtime()` invokes OpenAI. The
+//! server owns transport + config; routing lives in the `router` crate.
+//!
+//! The binary requires the `server` feature (declared in `Cargo.toml` via
+//! `required-features`), so cargo skips it unless that feature is on. Everything
+//! the binary needs lives in the library (`litellm_ai_gateway`); `main` just
+//! wires startup.
+
+use std::sync::Arc;
+
+use litellm_ai_gateway::io::realtime_pool::{upstream_key, PoolConfig, RealtimePool};
+use litellm_ai_gateway::routes;
+use litellm_ai_gateway::state::AppState;
+use litellm_core::router::{Deployment, LiteLLMParams, Router};
+
+use litellm_ai_gateway::integrations::custom_logger::CustomLogger;
+use litellm_ai_gateway::integrations::litellm_python_proxy_api::LiteLLMPythonProxyAPILogger;
+#[cfg(feature = "python-config")]
+use litellm_ai_gateway::python;
+
+/// Bind to localhost by default so the gateway is not a public, unauthenticated
+/// provider proxy out of the box. Override with `HOST` (e.g. `0.0.0.0`).
+const DEFAULT_HOST: &str = "127.0.0.1";
+const DEFAULT_PORT: u16 = 4001;
+
+#[tokio::main]
+async fn main() {
+    // Trim before storing so it matches the trimmed bearer token in `auth`
+    // (avoids a silent auth failure when the env var has surrounding whitespace).
+    let master_key: Option<Arc<str>> = std::env::var("LITELLM_MASTER_KEY")
+        .ok()
+        .map(|key| key.trim().to_string())
+        .filter(|key| !key.is_empty())
+        .map(Arc::from);
+    if master_key.is_none() {
+        eprintln!(
+            "warning: LITELLM_MASTER_KEY is not set; /v1/realtime will reject all requests (fail closed)"
+        );
+    }
+
+    // Spawn the realtime-logging worker (drains a channel → POSTs batches to the
+    // Python proxy's /v1/callbacks/logs). Built here so the spawn lands on the
+    // tokio runtime. `from_env` reads LITELLM_PROXY_BASE_URL + LITELLM_MASTER_KEY.
+    let proxy_logger = LiteLLMPythonProxyAPILogger::from_env();
+    let loggers: Vec<Arc<dyn CustomLogger>> = vec![proxy_logger];
+
+    let router = Arc::new(build_router());
+
+    // Build the pre-warmed realtime pool and register each deployment's upstream
+    // so the background replenisher starts warming it. `REALTIME_POOL_SIZE=0`
+    // yields a disabled pool → every connect fresh-dials (original behavior).
+    let pool_config = PoolConfig::from_env();
+    let realtime_pool = RealtimePool::spawn(pool_config);
+    if pool_config.enabled() {
+        register_deployments(&router, &realtime_pool);
+        eprintln!(
+            "realtime connection pool enabled: target {} warm sockets/key, max idle {}s",
+            pool_config.target_size,
+            pool_config.max_idle.as_secs()
+        );
+    } else {
+        eprintln!(
+            "realtime connection pool disabled (REALTIME_POOL_SIZE=0); fresh-dialing each connect"
+        );
+    }
+
+    let state = AppState {
+        router,
+        master_key,
+        loggers: Arc::new(loggers),
+        realtime_pool,
+    };
+
+    let host = std::env::var("HOST").unwrap_or_else(|_| DEFAULT_HOST.to_string());
+    let port = resolve_port();
+
+    let listener = tokio::net::TcpListener::bind((host.as_str(), port))
+        .await
+        .expect("failed to bind listener");
+    eprintln!("litellm-ai-gateway listening on {host}:{port}");
+    axum::serve(listener, routes::app(state))
+        .await
+        .expect("server error");
+}
+
+/// Register every deployment's upstream key with the pool so the replenisher
+/// pre-warms it. Mirrors `service::run`'s key derivation (strip `openai/`, resolve
+/// api_key); deployments whose key can't be resolved are skipped (they fresh-dial
+/// and surface the auth error on the request path, as before).
+fn register_deployments(router: &Router, pool: &RealtimePool) {
+    for deployment in router.deployments() {
+        let params = &deployment.litellm_params;
+        let provider_model = params
+            .model
+            .strip_prefix("openai/")
+            .unwrap_or(&params.model);
+        if let Some(key) = upstream_key(
+            provider_model,
+            params.api_key.as_deref(),
+            params.api_base.as_deref(),
+        ) {
+            pool.register(key);
+        }
+    }
+}
+
+/// Resolve `PORT`, warning (rather than silently defaulting) on an invalid value.
+fn resolve_port() -> u16 {
+    match std::env::var("PORT") {
+        Ok(raw) => raw.parse().unwrap_or_else(|_| {
+            eprintln!("warning: PORT={raw:?} is not a valid port; using {DEFAULT_PORT}");
+            DEFAULT_PORT
+        }),
+        Err(_) => DEFAULT_PORT,
+    }
+}
+
+/// Build the router. With the `python-config` feature and `LITELLM_CONFIG_PATH`
+/// set, load the resolved `model_list` from the proxy config via the embedded
+/// Python reader (load time only). Otherwise fall back to the env stand-in.
+fn build_router() -> Router {
+    #[cfg(feature = "python-config")]
+    if let Ok(config_path) = std::env::var("LITELLM_CONFIG_PATH") {
+        match python::config::load_router_from_config(&config_path) {
+            Ok(router) => {
+                eprintln!("loaded model_list from {config_path} via python config reader");
+                return router;
+            }
+            Err(err) => {
+                eprintln!("config load failed ({err}); falling back to env deployment");
+            }
+        }
+    }
+    build_router_from_env()
+}
+
+/// Build a minimal single-deployment `model_list` from the environment.
+///
+/// A real deployment loads `model_list` from config; this is the minimal stand-in
+/// so the gateway has one OpenAI deployment to route to.
+fn build_router_from_env() -> Router {
+    let model =
+        std::env::var("OPENAI_REALTIME_MODEL").unwrap_or_else(|_| "gpt-realtime".to_string());
+    let api_key = std::env::var("OPENAI_API_KEY").ok();
+    if api_key.is_none() {
+        eprintln!(
+            "warning: OPENAI_API_KEY is not set; realtime requests will fail with auth errors"
+        );
+    }
+    let deployment = Deployment {
+        model_name: model.clone(),
+        litellm_params: LiteLLMParams {
+            model,
+            api_key,
+            api_base: None,
+        },
+    };
+    Router::new(vec![deployment])
+}
--- a/litellm-rust/crates/ai-gateway/src/python/AGENTS.md
+++ b/litellm-rust/crates/ai-gateway/src/python/AGENTS.md
@ -0,0 +1,27 @@
+# ai-gateway/src/python — Python interop (load-time only)
+
+Functions here embed the Python interpreter (pyo3) and take the GIL to call into
+`litellm` (e.g. read the proxy `model_list`). Compiled only under the
+`python-config` feature.
+
+## Hard rule: non-hot-path functions only
+
+Everything in this folder MUST run **at most once per process lifetime — at
+startup / load time** (config read, warm-up). NEVER call into Python on the
+request path:
+
+- No GIL acquisition per request, per connection, or per realtime event.
+- No Python call inside a route handler, the router's hot path, or any loop that
+  scales with traffic.
+
+**Why:** the GIL serializes execution and would cap throughput; the realtime data
+path must stay pure Rust. Every acquisition is recorded by `crate::gil` — poll
+`GET /health/gil`, and `total_acquisitions` MUST stay flat under load.
+
+## How to add one
+
+Resolve whatever Python-derived data you need **once at boot** and hand the rest
+of the gateway an owned, plain-Rust value (e.g. build a `Router` from the
+resolved `model_list`). Record the acquisition via `crate::gil::record_acquisition()`
+immediately before taking the GIL. If a function would need to run per request,
+it does not belong here — move the work to Rust, or pre-resolve it at startup.
--- a/litellm-rust/crates/ai-gateway/src/python/config.rs
+++ b/litellm-rust/crates/ai-gateway/src/python/config.rs
@ -0,0 +1,39 @@
+//! Build the router by calling the Python proxy config reader (load time only).
+//!
+//! Embeds the interpreter via pyo3 and calls
+//! `litellm.proxy.read_model_list.read_model_list`, which reuses the proxy's
+//! `os.environ/` + secret-manager resolution. The GIL is taken **once at boot**
+//! (and recorded in [`crate::gil`]); the realtime hot path never touches Python.
+//!
+//! Compiled only under the `python-config` feature.
+
+use litellm_core::error::CoreError;
+use litellm_core::router::{Deployment, Router};
+use litellm_core::CoreResult;
+use pyo3::prelude::*;
+
+use crate::gil;
+
+/// Load the router's `model_list` from `config_path` via the Python reader.
+pub fn load_router_from_config(config_path: &str) -> CoreResult<Router> {
+    gil::record_acquisition();
+    Python::with_gil(|py| {
+        let model_list = py
+            .import("litellm.proxy.read_model_list")
+            .and_then(|module| module.getattr("read_model_list"))
+            .and_then(|reader| reader.call1((config_path,)))
+            .map_err(|err| CoreError::Routing(format!("read_model_list failed: {err}")))?;
+
+        let model_list_json: String = py
+            .import("json")
+            .and_then(|json| json.getattr("dumps"))
+            .and_then(|dumps| dumps.call1((model_list,)))
+            .and_then(|encoded| encoded.extract())
+            .map_err(|err| CoreError::Routing(format!("serializing model_list failed: {err}")))?;
+
+        let deployments: Vec<Deployment> = serde_json::from_str(&model_list_json)
+            .map_err(|err| CoreError::Routing(format!("parsing model_list failed: {err}")))?;
+
+        Ok(Router::new(deployments))
+    })
+}
--- a/litellm-rust/crates/ai-gateway/src/python/mod.rs
+++ b/litellm-rust/crates/ai-gateway/src/python/mod.rs
@ -0,0 +1,4 @@
+//! Python interop for the gateway. See `AGENTS.md`: **load-time / non-hot-path
+//! only.** Compiled only under the `python-config` feature.
+
+pub mod config;
--- a/litellm-rust/crates/ai-gateway/src/realtime/mod.rs
+++ b/litellm-rust/crates/ai-gateway/src/realtime/mod.rs
@ -0,0 +1,4 @@
+//! Realtime logging collector. Observes the realtime event stream and emits a
+//! `StandardLoggingPayload` to the registered callbacks on session close.
+
+pub mod streaming;
--- a/litellm-rust/crates/ai-gateway/src/realtime/streaming.rs
+++ b/litellm-rust/crates/ai-gateway/src/realtime/streaming.rs
@ -0,0 +1,352 @@
+//! `RealTimeStreaming` — the realtime logging collector.
+//!
+//! Mirrors Python `litellm.realtime_api.main.RealTimeStreaming`: it observes the
+//! event stream in O(1) (never buffering frames), accumulating just the fields
+//! the spend log needs (model, id, cumulative usage), then on session close
+//! builds a `StandardLoggingPayload` and fans it out to every registered
+//! `CustomLogger`.
+
+use std::sync::Arc;
+use std::time::{SystemTime, UNIX_EPOCH};
+
+use litellm_core::realtime::types::RealtimeEvent;
+use serde_json::Value;
+
+use crate::constants::DEFAULT_PROVIDER;
+use crate::integrations::custom_logger::CustomLogger;
+use crate::integrations::types::{
+    RequestMetadata, StandardLoggingMetadata, StandardLoggingPayload, Usage,
+};
+
+/// Current wall-clock time as epoch seconds (float), matching the Python
+/// `startTime`/`endTime` contract.
+fn epoch_seconds() -> f64 {
+    SystemTime::now()
+        .duration_since(UNIX_EPOCH)
+        .map(|d| d.as_secs_f64())
+        .unwrap_or(0.0)
+}
+
+/// Status of a finished realtime session, mapped to the callback record status.
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub enum SessionStatus {
+    Success,
+    Failure,
+}
+
+/// Accumulates realtime session state and emits a logging payload on close.
+pub struct RealTimeStreaming {
+    callbacks: Vec<Arc<dyn CustomLogger>>,
+    /// REQUEST-ID RULE: the SpendLogs `request_id` == the OpenAI realtime session
+    /// id (`sess_…`), captured from `session.created`. Both `id` and
+    /// `litellm_call_id` are set to that value so the Python writer logs the same
+    /// id regardless of which field it reads. The gateway-generated `rt-…` id
+    /// (the constructor seed) is only a fallback for sessions that fail before
+    /// `session.created` arrives.
+    litellm_call_id: String,
+    /// See the request-id rule above — mirrors `litellm_call_id`.
+    id: String,
+    model: String,
+    custom_llm_provider: String,
+    usage: Usage,
+    response_cost: f64,
+    start_time: f64,
+    end_time: f64,
+    metadata: RequestMetadata,
+    /// Count of logging callbacks that failed to enqueue (non-fatal).
+    dropped: u64,
+}
+
+impl RealTimeStreaming {
+    /// Create a collector for one session. `litellm_call_id` is the gateway's
+    /// per-connection id; `model` is the requested model (a sane default until
+    /// `session.created` reports the upstream model).
+    pub fn new(
+        callbacks: Vec<Arc<dyn CustomLogger>>,
+        litellm_call_id: String,
+        model: String,
+        metadata: RequestMetadata,
+    ) -> Self {
+        let now = epoch_seconds();
+        Self {
+            callbacks,
+            id: litellm_call_id.clone(),
+            litellm_call_id,
+            model,
+            custom_llm_provider: DEFAULT_PROVIDER.to_string(),
+            usage: Usage::default(),
+            response_cost: 0.0,
+            start_time: now,
+            end_time: now,
+            metadata,
+            dropped: 0,
+        }
+    }
+
+    /// Number of logging callbacks that failed to enqueue so far (test/observ.).
+    #[allow(dead_code)]
+    pub fn dropped(&self) -> u64 {
+        self.dropped
+    }
+
+    /// Observe one realtime event. O(1): updates accumulated state only; never
+    /// buffers frames. Safe to call on every event in either direction.
+    pub fn observe(&mut self, event: &RealtimeEvent) {
+        match event.event_type.as_str() {
+            "session.created" | "session.updated" => self.on_session(event),
+            "response.done" => self.on_response_done(event),
+            _ => {}
+        }
+    }
+
+    /// `session.created` / `session.updated` → capture upstream id + model.
+    /// Per the request-id rule, the OpenAI session id becomes BOTH `id` and
+    /// `litellm_call_id`, replacing the gateway-generated fallback.
+    fn on_session(&mut self, event: &RealtimeEvent) {
+        let session = event.data.get("session").and_then(Value::as_object);
+        if let Some(id) = session.and_then(|s| s.get("id")).and_then(Value::as_str) {
+            if !id.is_empty() {
+                self.id = id.to_string();
+                self.litellm_call_id = id.to_string();
+            }
+        }
+        if let Some(model) = session.and_then(|s| s.get("model")).and_then(Value::as_str) {
+            if !model.is_empty() {
+                self.model = model.to_string();
+            }
+        }
+    }
+
+    /// `response.done` → add this response's usage to the cumulative totals.
+    fn on_response_done(&mut self, event: &RealtimeEvent) {
+        let usage = event
+            .data
+            .get("response")
+            .and_then(Value::as_object)
+            .and_then(|r| r.get("usage"))
+            .and_then(Value::as_object);
+        let Some(usage) = usage else { return };
+
+        let input = usage.get("input_tokens").and_then(Value::as_u64);
+        let output = usage.get("output_tokens").and_then(Value::as_u64);
+        let total = usage.get("total_tokens").and_then(Value::as_u64);
+
+        if let Some(input) = input {
+            self.usage.prompt_tokens += input;
+        }
+        if let Some(output) = output {
+            self.usage.completion_tokens += output;
+        }
+        // Prefer the upstream-reported total; otherwise derive it.
+        match total {
+            Some(total) => self.usage.total_tokens += total,
+            None => {
+                self.usage.total_tokens += input.unwrap_or(0) + output.unwrap_or(0);
+            }
+        }
+    }
+
+    /// Set the per-session response cost ($). Cost computation is Python-side in
+    /// the proxy; the gateway forwards 0.0 by default and lets the proxy price.
+    /// Public API (exercised in tests) for the future path where the gateway
+    /// prices realtime sessions itself.
+    #[allow(dead_code)]
+    pub fn set_response_cost(&mut self, cost: f64) {
+        self.response_cost = cost;
+    }
+
+    /// Build the `StandardLoggingPayload` from accumulated state.
+    pub fn build_payload(&self) -> StandardLoggingPayload {
+        StandardLoggingPayload {
+            id: self.id.clone(),
+            litellm_call_id: self.litellm_call_id.clone(),
+            call_type: "realtime".to_string(),
+            model: self.model.clone(),
+            custom_llm_provider: self.custom_llm_provider.clone(),
+            response_cost: self.response_cost,
+            prompt_tokens: self.usage.prompt_tokens,
+            completion_tokens: self.usage.completion_tokens,
+            total_tokens: self.usage.total_tokens,
+            start_time: self.start_time,
+            end_time: self.end_time,
+            stream: true,
+            metadata: StandardLoggingMetadata {
+                user_api_key_hash: self.metadata.user_api_key_hash.clone(),
+                user_api_key_user_id: self.metadata.user_api_key_user_id.clone(),
+                user_api_key_team_id: self.metadata.user_api_key_team_id.clone(),
+                ..Default::default()
+            },
+            messages: None,
+        }
+    }
+
+    /// Finish the session: stamp the end time and fan the payload out to every
+    /// callback. On a logger enqueue error we bump a non-fatal counter (the
+    /// realtime session has already ended; a dropped log must never propagate).
+    pub fn log_messages(&mut self, status: SessionStatus) {
+        self.end_time = epoch_seconds();
+        let payload = self.build_payload();
+
+        match status {
+            SessionStatus::Success => {
+                for callback in &self.callbacks {
+                    if let Err(err) = callback.log_success_event(&payload) {
+                        self.dropped += 1;
+                        eprintln!("litellm-ai-gateway: log_success_event dropped: {err}");
+                    }
+                }
+            }
+            SessionStatus::Failure => {
+                let error = crate::integrations::types::LoggingError {
+                    message: "realtime session ended in failure".to_string(),
+                    kind: "RealtimeSessionError".to_string(),
+                };
+                for callback in &self.callbacks {
+                    if let Err(err) = callback.log_failure_event(&payload, &error) {
+                        self.dropped += 1;
+                        eprintln!("litellm-ai-gateway: log_failure_event dropped: {err}");
+                    }
+                }
+            }
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::integrations::types::{LogError, LoggingError};
+    use std::sync::atomic::{AtomicU64, Ordering};
+
+    fn event(raw: &str) -> RealtimeEvent {
+        serde_json::from_str(raw).expect("valid event json")
+    }
+
+    /// A test logger that records the last payload it saw.
+    #[derive(Default)]
+    struct CapturingLogger {
+        calls: AtomicU64,
+        last_model: std::sync::Mutex<Option<String>>,
+        last_total_tokens: AtomicU64,
+    }
+
+    impl CustomLogger for CapturingLogger {
+        fn log_success_event(&self, payload: &StandardLoggingPayload) -> Result<(), LogError> {
+            self.calls.fetch_add(1, Ordering::SeqCst);
+            *self.last_model.lock().unwrap() = Some(payload.model.clone());
+            self.last_total_tokens
+                .store(payload.total_tokens, Ordering::SeqCst);
+            Ok(())
+        }
+    }
+
+    #[test]
+    fn observe_accumulates_model_and_tokens_then_logs() {
+        let logger = Arc::new(CapturingLogger::default());
+        let callbacks: Vec<Arc<dyn CustomLogger>> = vec![logger.clone()];
+        let mut streaming = RealTimeStreaming::new(
+            callbacks,
+            "call_abc".to_string(),
+            "gpt-realtime".to_string(),
+            RequestMetadata {
+                user_api_key_hash: Some("hash123".to_string()),
+                user_api_key_user_id: Some("user-1".to_string()),
+                user_api_key_team_id: Some("team-1".to_string()),
+            },
+        );
+
+        streaming.observe(&event(
+            r#"{"type":"session.created","session":{"id":"sess_001","model":"gpt-realtime-2025"}}"#,
+        ));
+        streaming.observe(&event(
+            r#"{"type":"response.done","response":{"usage":{"input_tokens":10,"output_tokens":5,"total_tokens":15}}}"#,
+        ));
+        // A second response.done accumulates.
+        streaming.observe(&event(
+            r#"{"type":"response.done","response":{"usage":{"input_tokens":3,"output_tokens":2,"total_tokens":5}}}"#,
+        ));
+
+        let payload = streaming.build_payload();
+        assert_eq!(payload.model, "gpt-realtime-2025");
+        // Request-id rule: session.created's id becomes BOTH id and
+        // litellm_call_id (replacing the "call_abc" gateway fallback), so the
+        // SpendLogs request_id is always the OpenAI session id.
+        assert_eq!(payload.id, "sess_001");
+        assert_eq!(payload.litellm_call_id, "sess_001");
+        assert_eq!(payload.prompt_tokens, 13);
+        assert_eq!(payload.completion_tokens, 7);
+        assert_eq!(payload.total_tokens, 20);
+        assert_eq!(payload.response_cost, 0.0);
+        assert_eq!(payload.call_type, "realtime");
+        assert_eq!(payload.custom_llm_provider, "openai");
+        assert_eq!(
+            payload.metadata.user_api_key_hash.as_deref(),
+            Some("hash123")
+        );
+
+        streaming.log_messages(SessionStatus::Success);
+        assert_eq!(logger.calls.load(Ordering::SeqCst), 1);
+        assert_eq!(
+            logger.last_model.lock().unwrap().as_deref(),
+            Some("gpt-realtime-2025")
+        );
+        assert_eq!(logger.last_total_tokens.load(Ordering::SeqCst), 20);
+        assert_eq!(streaming.dropped(), 0);
+    }
+
+    #[test]
+    fn payload_serializes_with_camelcase_times_and_realtime_call_type() {
+        let mut streaming = RealTimeStreaming::new(
+            Vec::new(),
+            "call_xyz".to_string(),
+            "gpt-realtime".to_string(),
+            RequestMetadata::default(),
+        );
+        streaming.observe(&event(
+            r#"{"type":"response.done","response":{"usage":{"input_tokens":1,"output_tokens":1,"total_tokens":2}}}"#,
+        ));
+        streaming.set_response_cost(0.0042);
+        let payload = streaming.build_payload();
+        let json = serde_json::to_string(&payload).expect("serialize payload");
+
+        assert!(json.contains("\"startTime\""), "missing startTime: {json}");
+        assert!(json.contains("\"endTime\""), "missing endTime: {json}");
+        assert!(
+            json.contains("\"call_type\":\"realtime\""),
+            "missing call_type realtime: {json}"
+        );
+        assert!(
+            json.contains("\"response_cost\""),
+            "missing response_cost: {json}"
+        );
+        assert_eq!(payload.response_cost, 0.0042);
+    }
+
+    /// A logger whose enqueue always fails should bump the dropped counter, not
+    /// panic or propagate.
+    #[test]
+    fn failing_logger_bumps_dropped_counter() {
+        struct FailingLogger;
+        impl CustomLogger for FailingLogger {
+            fn log_success_event(&self, _p: &StandardLoggingPayload) -> Result<(), LogError> {
+                Err(LogError::channel_full())
+            }
+            fn log_failure_event(
+                &self,
+                _p: &StandardLoggingPayload,
+                _e: &LoggingError,
+            ) -> Result<(), LogError> {
+                Err(LogError::channel_closed())
+            }
+        }
+        let callbacks: Vec<Arc<dyn CustomLogger>> = vec![Arc::new(FailingLogger)];
+        let mut streaming = RealTimeStreaming::new(
+            callbacks,
+            "call_1".to_string(),
+            "gpt-realtime".to_string(),
+            RequestMetadata::default(),
+        );
+        streaming.log_messages(SessionStatus::Success);
+        assert_eq!(streaming.dropped(), 1);
+    }
+}
--- a/litellm-rust/crates/ai-gateway/src/routes/AGENTS.md
+++ b/litellm-rust/crates/ai-gateway/src/routes/AGENTS.md
@ -0,0 +1,38 @@
+# routes/ — the route template
+
+Every route follows the **same shape** so the layout is predictable. The rule:
+
+> **Each route module exposes `pub fn router() -> Router<AppState>`.**
+> `routes/mod.rs::app` merges them all and applies state once. Adding a route is:
+> create the module, then add one `.merge(<name>::router())` line.
+
+## Default: one file
+A route is a single file containing `router()` + its handler(s) (handlers stay
+private). This is the norm — don't split until it hurts.
+```
+pub fn router() -> Router<AppState> { Router::new().route(PATH, get(handle)) }
+async fn handle(...) -> impl IntoResponse { ... }
+```
+`health.rs` and `gil.rs` are examples.
+
+## Split out `service` when there's real logic
+When a route has business logic worth testing without axum, put it in a sibling
+`service` (a file, or a folder if the route grows). The route file stays the
+**axum surface** (router + handler + any socket/SSE adapter); `service` is plain
+Rust with **no axum types**. `realtime/` is the example:
+```
+realtime/
+  mod.rs       # axum surface: router() + handler + the WS<->events adapter
+  service.rs   # pure logic: select deployment + call provider (no axum) — testable
+```
+Split `service` further (or add `transport`, `repo`, …) only once a single file
+genuinely gets hard to read.
+
+## Invariants
+- **Auth is an extractor, not a manual call.** A handler requires auth by adding
+  `crate::auth::RequireMasterKey` to its arguments; it runs during extraction.
+  Never re-implement the check per route.
+- **Handlers contain no business logic; `service` contains no axum types.**
+- A route owns its paths in its own `router()`; `mod.rs` only merges.
+- Cross-cutting concerns (logging, CORS, timeouts) → Tower layers in `mod.rs`,
+  not duplicated in handlers.
--- a/litellm-rust/crates/ai-gateway/src/routes/gil.rs
+++ b/litellm-rust/crates/ai-gateway/src/routes/gil.rs
@ -0,0 +1,30 @@
+//! `GET /health/gil` — poll to confirm Python is only touched at load time.
+//! Simple-route template: a `router()` plus its handler, in one file.
+
+use axum::routing::get;
+use axum::{Json, Router};
+use serde::Serialize;
+
+use crate::gil;
+use crate::state::AppState;
+
+/// This route's contribution to the app router.
+pub fn router() -> Router<AppState> {
+    Router::new().route("/health/gil", get(status))
+}
+
+#[derive(Debug, Serialize)]
+struct GilStatusResponse {
+    gil_acquired_last_30s: bool,
+    total_acquisitions: u64,
+    seconds_since_last: Option<u64>,
+}
+
+async fn status() -> Json<GilStatusResponse> {
+    let snapshot = gil::snapshot();
+    Json(GilStatusResponse {
+        gil_acquired_last_30s: snapshot.acquired_last_30s,
+        total_acquisitions: snapshot.total_acquisitions,
+        seconds_since_last: snapshot.seconds_since_last,
+    })
+}
--- a/litellm-rust/crates/ai-gateway/src/routes/health.rs
+++ b/litellm-rust/crates/ai-gateway/src/routes/health.rs
@ -0,0 +1,24 @@
+//! Health probes. Simple-route template: a `router()` plus its handlers, in one file.
+
+use axum::http::StatusCode;
+use axum::routing::get;
+use axum::Router;
+
+use crate::state::AppState;
+
+/// This route's contribution to the app router.
+pub fn router() -> Router<AppState> {
+    Router::new()
+        .route("/health/liveness", get(liveness))
+        .route("/health/readiness", get(readiness))
+}
+
+/// The process is up.
+async fn liveness() -> StatusCode {
+    StatusCode::OK
+}
+
+/// The server is ready to accept traffic.
+async fn readiness() -> StatusCode {
+    StatusCode::OK
+}
--- a/litellm-rust/crates/ai-gateway/src/routes/mod.rs
+++ b/litellm-rust/crates/ai-gateway/src/routes/mod.rs
@ -0,0 +1,23 @@
+//! HTTP routes.
+//!
+//! **Template:** every route module exposes `pub fn router() -> Router<AppState>`
+//! that mounts its own paths; [`app`] merges them. A trivial route is a single
+//! file (`health.rs`, `gil.rs`); a non-trivial one is a folder (`realtime/`) with
+//! `handler` (entry) + `service` (logic) + `transport` (adapters). See AGENTS.md.
+
+pub mod gil;
+pub mod health;
+pub mod realtime;
+
+use axum::Router;
+
+use crate::state::AppState;
+
+/// Assemble the application router by merging every route module's `router()`.
+pub fn app(state: AppState) -> Router {
+    Router::new()
+        .merge(health::router())
+        .merge(gil::router())
+        .merge(realtime::router())
+        .with_state(state)
+}
--- a/litellm-rust/crates/ai-gateway/src/routes/realtime/README.md
+++ b/litellm-rust/crates/ai-gateway/src/routes/realtime/README.md
@ -0,0 +1,87 @@
+# Realtime route (`GET /v1/realtime`)
+
+Proxies OpenAI's realtime WebSocket. `mod.rs` is the axum surface (handler +
+socket↔events adapter); `service.rs` is the pure logic (select a deployment, then
+splice client ↔ upstream). The pool itself lives in
+`crates/providers/src/realtime_pool.rs`.
+
+## Connection pooling
+
+### The problem
+
+The gateway's realtime overhead lives **entirely in session establishment**. On each
+client connect it dials a *fresh* upstream WS to OpenAI and waits for
+`session.created` before it can serve. Measured at 5000 calls / 500 concurrency, the
+fresh-dial session phase is **~360 ms** vs **~7 ms** direct; dial, first-audio, and
+streaming add ~0. So the one lever is removing that per-connect handshake from the
+critical path.
+
+### The idea
+
+Keep a few upstream OpenAI sockets **already connected and already past
+`session.created`** (buffered). On a client connect, hand off a warm socket — relay
+its buffered `session.created` instantly (a local `Vec::pop`, sub-millisecond) and
+splice exactly as a fresh dial would. A background task keeps the pool topped up. On
+a miss or dead socket we fall back to fresh-dial: the pool is a latency optimization,
+never a correctness dependency.
+
+```
+                          ┌───────────────────────────────────────┐
+   client connect ──────► │ routes/realtime → service::run         │
+                          │   pool.take(key)                        │
+                          │     hit  → relay buffered               │
+                          │            session.created, then splice │
+                          │     miss → fresh dial (original path)    │
+                          └───────────────┬───────────────────────┘
+                                          │ replenish (async, concurrent)
+                          ┌───────────────▼───────────────────────┐
+   background task ─────► │ RealtimePool: per-key warm sockets      │
+                          │   each = { ws, buffered session.created}│
+                          │   liveness-checked before handoff       │
+                          └─────────────────────────────────────────┘
+```
+
+A warm session is indistinguishable from a fresh one: OpenAI sends `session.created`
+unprompted on connect, we pre-read exactly that one frame and relay it on handoff,
+and we send nothing else on the socket before a client exists — so the client's first
+`session.update` behaves identically either way.
+
+### Sizing
+
+Each warm socket serves **exactly one** session (realtime isn't multiplexed), so the
+pool is sized to the **peak concurrent connects per instance**, not total live
+connections:
+
+```
+REALTIME_POOL_SIZE ≈ peak_concurrency / instance_count
+```
+
+e.g. 500 concurrency over 10 instances → ~50–64 per instance. The replenisher dials
+the missing sockets **concurrently**, so a drained pool refills in ~one handshake
+window and keeps supply close to the connect rate. Over-provisioning just burns idle
+upstream sockets, which is why warm sockets are short-lived
+(`REALTIME_POOL_MAX_IDLE_SECS`).
+
+### Config
+
+| env                           | default | meaning                                                         |
+| ----------------------------- | ------- | --------------------------------------------------------------- |
+| `REALTIME_POOL_SIZE`          | `4`     | target warm sockets per key. `0` disables pooling (fresh-dial). |
+| `REALTIME_POOL_MAX_IDLE_SECS` | `30`    | max time a warm socket sits before it's closed and replaced.    |
+
+### Notes
+
+- **Miss / dead socket → fresh dial.** Burst beyond warm supply, or a socket that
+  died, never blocks or fails — it falls back to the original path. The pool can only
+  make a connect faster, never slower or more fragile.
+- **Auth scope.** The pool key includes `api_key`, so a warm socket is only handed to
+  a request resolving to the same key — no cross-tenant reuse.
+- **Idle billing.** Warm sockets are liveness-checked at handoff and capped at
+  `REALTIME_POOL_MAX_IDLE_SECS` to bound idle billing and dodge OpenAI's idle timeout.
+- **Replenish backoff.** If a key's warm-up dials all fail (invalid credentials, an
+  unreachable upstream), the replenisher puts that key into exponential backoff
+  (500 ms → 30 s cap) instead of re-dialing it every tick. This bounds connection
+  attempts against a broken key so it can't exhaust upstream rate limits and degrade
+  valid cold-path traffic; the backoff resets the moment a dial succeeds.
+
+Benchmarks and repro: `../../benchmarks/realtime/README.md`.
--- a/litellm-rust/crates/ai-gateway/src/routes/realtime/mod.rs
+++ b/litellm-rust/crates/ai-gateway/src/routes/realtime/mod.rs
@ -0,0 +1,166 @@
+//! `GET /v1/realtime` (WebSocket).
+//!
+//! This file is the **axum surface**: `router()`, the handler, and the small
+//! socket↔events adapter. The pure logic (no axum) lives in [`service`]. Auth is
+//! the `RequireMasterKey` extractor, so the handler stays thin.
+
+mod service;
+
+use std::sync::atomic::{AtomicU64, Ordering};
+use std::sync::Arc;
+use std::time::{SystemTime, UNIX_EPOCH};
+
+use crate::io::realtime_pool::RealtimePool;
+use axum::extract::ws::{Message, WebSocket, WebSocketUpgrade};
+use axum::extract::{Query, State};
+use axum::http::StatusCode;
+use axum::response::Response;
+use axum::routing::get;
+use axum::Router;
+use futures_util::{SinkExt, StreamExt};
+use litellm_core::realtime::types::RealtimeEvent;
+use litellm_core::router::Router as ModelRouter;
+use serde::Deserialize;
+
+use crate::auth::RequireMasterKey;
+use crate::integrations::custom_logger::CustomLogger;
+use crate::integrations::types::RequestMetadata;
+use crate::realtime::streaming::{RealTimeStreaming, SessionStatus};
+use crate::state::AppState;
+
+/// Process-local monotonic counter, mixed into the per-session call id so two
+/// sessions opened in the same nanosecond still get distinct ids.
+static CALL_SEQ: AtomicU64 = AtomicU64::new(0);
+
+/// Generate a per-connection `litellm_call_id`. No external uuid dep: epoch
+/// nanos + a process-local sequence is unique enough for log correlation.
+fn new_call_id() -> String {
+    let nanos = SystemTime::now()
+        .duration_since(UNIX_EPOCH)
+        .map(|d| d.as_nanos())
+        .unwrap_or(0);
+    let seq = CALL_SEQ.fetch_add(1, Ordering::Relaxed);
+    format!("rt-{nanos:x}-{seq:x}")
+}
+
+/// This route's contribution to the app router.
+pub fn router() -> Router<AppState> {
+    Router::new().route("/v1/realtime", get(handle))
+}
+
+#[derive(Debug, Deserialize)]
+struct RealtimeQuery {
+    model: String,
+}
+
+/// Auth runs via the `RequireMasterKey` extractor. We validate the model BEFORE
+/// the upgrade so failures are clean HTTP (400/404), not a socket that opens then
+/// closes, then hand the socket to `bridge`.
+async fn handle(
+    _auth: RequireMasterKey,
+    ws: WebSocketUpgrade,
+    State(state): State<AppState>,
+    Query(query): Query<RealtimeQuery>,
+) -> Result<Response, (StatusCode, String)> {
+    if query.model.trim().is_empty() {
+        return Err((
+            StatusCode::BAD_REQUEST,
+            "missing 'model' query param".to_string(),
+        ));
+    }
+    if !state.router.has_deployment(&query.model) {
+        return Err((
+            StatusCode::NOT_FOUND,
+            format!("no deployment for model '{}'", query.model),
+        ));
+    }
+
+    let router = state.router.clone();
+    let pool = state.realtime_pool.clone();
+    let loggers = state.loggers.clone();
+    let master_key = state.master_key.clone();
+    let model = query.model;
+    Ok(ws.on_upgrade(move |socket| bridge(socket, router, pool, loggers, master_key, model)))
+}
+
+/// Adapt the axum socket (text frames) to the typed-event `Stream`/`Sink` the
+/// service wants, keeping axum types out of `service`.
+///
+/// This is also the realtime-logging seam: every upstream→client event (the
+/// direction carrying `session.created` and `response.done` with usage) is fed
+/// to a [`RealTimeStreaming`] collector via the splice's `observe` callback. The
+/// observe is O(1) and never buffers frames. When the splice returns (any of the
+/// three break paths — client disconnect, upstream close, idle timeout), we flush
+/// one logging payload to the registered callbacks.
+async fn bridge(
+    socket: WebSocket,
+    router: Arc<ModelRouter>,
+    pool: Arc<RealtimePool>,
+    loggers: Arc<Vec<Arc<dyn CustomLogger>>>,
+    master_key: Option<Arc<str>>,
+    model: String,
+) {
+    let (ws_sink, ws_stream) = socket.split();
+
+    // Attribute the spend log to the key that authenticated this session (the
+    // master key — the gateway is master-key auth). A non-null user_api_key_hash
+    // is required for the Python spend logger to write a SpendLogs row.
+    //
+    // SECURITY: hash the key — never send the raw credential. This field fans out
+    // to spend logs and every callback integration; the SHA-256 (matching the
+    // proxy's hash_token) keeps the plaintext master key out of all of them while
+    // still matching the key's hash in LiteLLM_SpendLogs.
+    let metadata = RequestMetadata {
+        user_api_key_hash: master_key.as_deref().map(crate::auth::hash_token),
+        ..RequestMetadata::default()
+    };
+
+    // Owned by THIS task only. The splice observes it via a synchronous `&mut`
+    // callback (below), so there is no Arc/Mutex/atomic on the per-frame hot
+    // path — just a monomorphized FnMut mutating stack-local fields. This is
+    // what lets observe scale: 10K concurrent sessions = 10K independent
+    // collectors, zero cross-task synchronization.
+    let mut collector = RealTimeStreaming::new(
+        loggers.as_ref().clone(),
+        new_call_id(),
+        model.clone(),
+        metadata,
+    );
+
+    let client_in = ws_stream.filter_map(|message| async move {
+        match message {
+            Ok(Message::Text(text)) => serde_json::from_str::<RealtimeEvent>(&text).ok(),
+            _ => None,
+        }
+    });
+    // Plain forwarding sink — no observe here anymore.
+    let client_out = ws_sink.with(|event: RealtimeEvent| async move {
+        Ok::<Message, axum::Error>(Message::Text(
+            serde_json::to_string(&event).unwrap_or_default(),
+        ))
+    });
+
+    futures_util::pin_mut!(client_in, client_out);
+
+    // The observe closure borrows `&mut collector` for the duration of the
+    // splice; the borrow ends when `run` returns, freeing the collector for the
+    // single post-session `log_messages` flush. `run` picks a pooled (warm) or
+    // fresh upstream — observe fires on the upstream arm either way.
+    let result = service::run(
+        &router,
+        &pool,
+        &model,
+        None,
+        |event: &RealtimeEvent| collector.observe(event),
+        client_in,
+        client_out,
+    )
+    .await;
+
+    let status = if result.is_ok() {
+        SessionStatus::Success
+    } else {
+        SessionStatus::Failure
+    };
+    collector.log_messages(status);
+}
--- a/litellm-rust/crates/ai-gateway/src/routes/realtime/service.rs
+++ b/litellm-rust/crates/ai-gateway/src/routes/realtime/service.rs
@ -0,0 +1,79 @@
+//! Business logic: select a deployment with the (pure) core router, then call the
+//! provider splice. The seam between `core::router` (selection only) and
+//! `io` (the actual WebSocket I/O).
+//!
+//! On connect we try a pre-warmed upstream from the pool (handshake already paid,
+//! `session.created` buffered) and relay it instantly. On a pool miss or dead warm
+//! socket we fresh-dial exactly as before — the pool is never on the critical path
+//! for correctness, only latency.
+
+use std::time::Duration;
+
+use crate::io::realtime_pool::{upstream_key, RealtimePool};
+use futures_util::{Sink, Stream};
+use litellm_core::error::CoreError;
+use litellm_core::realtime::types::RealtimeEvent;
+use litellm_core::router::Router;
+use litellm_core::CoreResult;
+
+/// Select a deployment for `model` and splice the client stream to the provider.
+///
+/// `pool` supplies a pre-warmed upstream when one is available; otherwise we
+/// fresh-dial. A disabled pool always misses, so this collapses to the original
+/// fresh-dial behavior.
+pub async fn run<In, Out>(
+    router: &Router,
+    pool: &RealtimePool,
+    model: &str,
+    idle_timeout: Option<Duration>,
+    observe: impl FnMut(&RealtimeEvent) + Send,
+    client_in: In,
+    client_out: Out,
+) -> CoreResult<()>
+where
+    In: Stream<Item = RealtimeEvent> + Unpin + Send,
+    Out: Sink<RealtimeEvent> + Unpin + Send,
+    <Out as Sink<RealtimeEvent>>::Error: std::fmt::Display,
+{
+    let deployment = router.get_available_deployment(model).ok_or_else(|| {
+        CoreError::Routing(format!("no deployment available for model '{model}'"))
+    })?;
+    let params = &deployment.litellm_params;
+    // Strip a leading `openai/` so the OpenAI-only realtime fn gets the bare model.
+    let provider_model = params
+        .model
+        .strip_prefix("openai/")
+        .unwrap_or(&params.model);
+
+    // Warm path: take a pooled upstream (handshake already paid) and relay its
+    // buffered session.created immediately. On miss/dead socket fall through.
+    if let Some(key) = upstream_key(
+        provider_model,
+        params.api_key.as_deref(),
+        params.api_base.as_deref(),
+    ) {
+        if let Some(handoff) = pool.take(&key) {
+            return crate::io::realtime::realtime_warm(
+                provider_model,
+                handoff,
+                idle_timeout,
+                observe,
+                client_in,
+                client_out,
+            )
+            .await;
+        }
+    }
+
+    // Cold path: fresh dial (the original behavior).
+    crate::io::realtime::realtime(
+        provider_model,
+        params.api_key.as_deref(),
+        params.api_base.as_deref(),
+        idle_timeout,
+        observe,
+        client_in,
+        client_out,
+    )
+    .await
+}
--- a/litellm-rust/crates/ai-gateway/src/state.rs
+++ b/litellm-rust/crates/ai-gateway/src/state.rs
@ -0,0 +1,21 @@
+use std::sync::Arc;
+
+use crate::io::realtime_pool::RealtimePool;
+use litellm_core::router::Router;
+
+use crate::integrations::custom_logger::CustomLogger;
+
+/// Shared application state handed to every route handler.
+#[derive(Clone)]
+pub struct AppState {
+    pub router: Arc<Router>,
+    /// The gateway master key. Any caller presenting it as a bearer token may
+    /// invoke the gateway. `None` → auth not configured (routes fail closed).
+    pub master_key: Option<Arc<str>>,
+    /// Logging callbacks fanned out at the end of each realtime session.
+    pub loggers: Arc<Vec<Arc<dyn CustomLogger>>>,
+    /// Pre-warmed upstream realtime connection pool. Disabled
+    /// (`RealtimePool::disabled()`) when `REALTIME_POOL_SIZE=0`, in which case
+    /// every realtime connect fresh-dials exactly as before.
+    pub realtime_pool: Arc<RealtimePool>,
+}
--- a/litellm-rust/crates/core/AGENTS.md
+++ b/litellm-rust/crates/core/AGENTS.md
@ -0,0 +1,3 @@
+litellm-core is the PURE translation layer — types, route contracts (traits), provider transforms (modules under `providers/`), and the router. No network, no I/O, no env reads.
+
+Routes (ocr, realtime) and providers (mistral, openai) are modules, not crates.
--- a/litellm-rust/crates/core/CLAUDE.md
+++ b/litellm-rust/crates/core/CLAUDE.md
@ -0,0 +1,47 @@
+# CLAUDE.md
+
+Rules for `litellm-rust/crates/core`.
+
+## Responsibility
+
+`core` owns shared data types, typed errors, and deterministic helper contracts.
+It must stay pure and host-independent.
+
+Allowed:
+- Shared request/response structs.
+- Typed errors with stable, non-sensitive messages.
+- Deterministic validation helpers.
+- Serialization helpers that intentionally mirror Python output shape.
+- Route templates that match Python base config responsibilities, such as
+  `ocr::transformation::OcrProviderConfig`.
+
+Not allowed:
+- Network, filesystem, database, cache, or environment access.
+- Secret reads or auth/header construction.
+- Logging callbacks, tracing spans, spend writes, or customer callbacks.
+- Provider-specific branching that belongs in `providers`.
+- Panics for user/provider-controlled input.
+
+## Typed Contracts (core rule)
+
+Trait and function boundaries MUST be strongly typed. No stringly-typed JSON
+(`&str` / `String` / `Vec<String>` / bare `serde_json::Value`) as a transform
+input or output. Parse wire bytes into typed structs/enums at the host edge;
+`core` and `providers` operate only on those types (e.g. `RealtimeEvent`,
+`RealtimeTransformResult`, `OcrRequestData`). A `type`-style discriminator is a
+typed field on a struct, not a raw string threaded through the API.
+
+## Structure
+
+Use route names directly under `src/`: `ocr`, future `messages`,
+`chat_completions`, `embeddings`, and similar top-level LiteLLM calls. Do not
+invent broad names like `engine` for route contracts.
+
+## Parity Rules
+
+- Every shared type used by a provider transform needs unit tests for
+  serialization shape.
+- If Python parity requires always emitting a `null` field instead of omitting
+  it, document that in code and pin it with a test.
+- Error enums should preserve enough detail for Python/HTTP hosts to map errors
+  consistently without exposing document contents or upstream bodies.
--- a/litellm-rust/crates/core/Cargo.toml
+++ b/litellm-rust/crates/core/Cargo.toml
@ -0,0 +1,12 @@
+[package]
+name = "litellm-core"
+version = "0.1.0"
+edition.workspace = true
+license.workspace = true
+repository.workspace = true
+
+[dependencies]
+rand.workspace = true
+serde.workspace = true
+serde_json.workspace = true
+thiserror.workspace = true
--- a/litellm-rust/crates/core/src/error.rs
+++ b/litellm-rust/crates/core/src/error.rs
@ -0,0 +1,35 @@
+use thiserror::Error;
+
+pub type CoreResult<T> = Result<T, CoreError>;
+
+#[derive(Debug, Error, PartialEq, Eq)]
+pub enum CoreError {
+    #[error("expected {expected}, got {actual}")]
+    InvalidType {
+        expected: &'static str,
+        actual: &'static str,
+    },
+    #[error("missing required field: {0}")]
+    MissingField(&'static str),
+    #[error("invalid response: {0}")]
+    InvalidResponse(String),
+    #[error("{0}")]
+    Auth(String),
+    #[error("OCR request failed with status {status}: {body}")]
+    Http { status: u16, body: String },
+    #[error("OCR network error: {0}")]
+    Network(String),
+    #[error("routing error: {0}")]
+    Routing(String),
+}
+
+pub fn json_type_name(value: &serde_json::Value) -> &'static str {
+    match value {
+        serde_json::Value::Null => "null",
+        serde_json::Value::Bool(_) => "bool",
+        serde_json::Value::Number(_) => "number",
+        serde_json::Value::String(_) => "string",
+        serde_json::Value::Array(_) => "array",
+        serde_json::Value::Object(_) => "object",
+    }
+}
--- a/litellm-rust/crates/core/src/lib.rs
+++ b/litellm-rust/crates/core/src/lib.rs
@ -0,0 +1,7 @@
+pub mod error;
+pub mod ocr;
+pub mod providers;
+pub mod realtime;
+pub mod router;
+
+pub use error::{CoreError, CoreResult};
--- a/litellm-rust/crates/core/src/ocr/mod.rs
+++ b/litellm-rust/crates/core/src/ocr/mod.rs
@ -0,0 +1,2 @@
+pub mod transformation;
+pub mod types;
--- a/litellm-rust/crates/core/src/ocr/transformation.rs
+++ b/litellm-rust/crates/core/src/ocr/transformation.rs
@ -0,0 +1,32 @@
+use serde_json::{Map, Value};
+
+use crate::CoreResult;
+
+use super::types::{OcrRequestData, OcrResponseData};
+
+pub trait OcrProviderConfig {
+    fn supported_ocr_params(&self) -> &'static [&'static str];
+
+    fn map_ocr_params(&self, non_default_params: &Map<String, Value>) -> Map<String, Value> {
+        let mut mapped_params = Map::new();
+        for (param, value) in non_default_params {
+            if self.supported_ocr_params().contains(&param.as_str()) {
+                mapped_params.insert(param.clone(), value.clone());
+            }
+        }
+        mapped_params
+    }
+
+    fn transform_ocr_request(
+        &self,
+        model: &str,
+        document: Value,
+        optional_params: Map<String, Value>,
+    ) -> CoreResult<OcrRequestData>;
+
+    fn transform_ocr_response(
+        &self,
+        model: &str,
+        response_json: Value,
+    ) -> CoreResult<OcrResponseData>;
+}
--- a/litellm-rust/crates/core/src/ocr/types.rs
+++ b/litellm-rust/crates/core/src/ocr/types.rs
@ -0,0 +1,29 @@
+use serde::{Deserialize, Serialize};
+use serde_json::Value;
+
+#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
+pub struct OcrRequestData {
+    pub data: Value,
+    pub files: Option<Value>,
+}
+
+#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
+pub struct OcrResponseData {
+    pub pages: Vec<Value>,
+    pub model: String,
+    pub document_annotation: Option<Value>,
+    pub usage_info: Option<Value>,
+    pub object: String,
+}
+
+impl OcrResponseData {
+    pub fn into_json(self) -> Value {
+        serde_json::json!({
+            "pages": self.pages,
+            "model": self.model,
+            "document_annotation": self.document_annotation,
+            "usage_info": self.usage_info,
+            "object": self.object,
+        })
+    }
+}
--- a/litellm-rust/crates/core/src/providers/mistral/mod.rs
+++ b/litellm-rust/crates/core/src/providers/mistral/mod.rs
@ -0,0 +1 @@
+pub mod ocr;
--- a/litellm-rust/crates/core/src/providers/mistral/ocr/mod.rs
+++ b/litellm-rust/crates/core/src/providers/mistral/ocr/mod.rs
@ -0,0 +1 @@
+pub mod transformation;
--- a/litellm-rust/crates/core/src/providers/mistral/ocr/transformation.rs
+++ b/litellm-rust/crates/core/src/providers/mistral/ocr/transformation.rs
@ -0,0 +1,292 @@
+use crate::error::{json_type_name, CoreError, CoreResult};
+use crate::ocr::transformation::OcrProviderConfig;
+use crate::ocr::types::{OcrRequestData, OcrResponseData};
+use serde_json::{Map, Value};
+
+const SUPPORTED_OCR_PARAMS: &[&str] = &[
+    "pages",
+    "include_image_base64",
+    "image_limit",
+    "image_min_size",
+    "bbox_annotation_format",
+    "document_annotation_format",
+    "document_annotation_prompt",
+    "extract_header",
+    "extract_footer",
+    "table_format",
+    "confidence_scores_granularity",
+    "id",
+];
+
+/// Default Mistral API base, used when the caller does not override `api_base`.
+pub const MISTRAL_DEFAULT_API_BASE: &str = "https://api.mistral.ai/v1";
+
+/// Environment variable holding the Mistral API key.
+pub const MISTRAL_API_KEY_ENV: &str = "MISTRAL_API_KEY";
+
+/// Error message raised when no Mistral API key can be resolved.
+pub const MISSING_KEY_MESSAGE: &str = "Missing Mistral API Key - A call is being made to Mistral but no key is set either in the environment variables or via params";
+
+/// Build the complete OCR endpoint URL, de-duplicating a trailing `/v1`.
+///
+/// Blank/whitespace `api_base` is treated as absent (guard at resolution time).
+pub fn complete_url(api_base: Option<&str>) -> String {
+    let base = api_base
+        .map(str::trim)
+        .filter(|base| !base.is_empty())
+        .unwrap_or(MISTRAL_DEFAULT_API_BASE)
+        .trim_end_matches('/');
+
+    if base.ends_with("/v1") {
+        format!("{base}/ocr")
+    } else {
+        format!("{base}/v1/ocr")
+    }
+}
+
+/// Resolve the Mistral API key from the explicit param or the environment.
+///
+/// Blank/whitespace values are treated as absent. Returns `CoreError::Auth`
+/// when no usable key is available.
+///
+/// Note: the env fallback only reads the process environment. Secret-manager
+/// backends (AWS/Azure/GCP/Vault) are resolved on the Python side and passed in
+/// via `api_key`; this fallback is a last resort for direct/standalone use.
+pub fn resolve_api_key(
+    api_key: Option<&str>,
+    env_lookup: &dyn Fn(&str) -> Option<String>,
+) -> CoreResult<String> {
+    api_key
+        .map(str::trim)
+        .filter(|key| !key.is_empty())
+        .map(str::to_string)
+        .or_else(|| env_lookup(MISTRAL_API_KEY_ENV).filter(|key| !key.trim().is_empty()))
+        .ok_or_else(|| CoreError::Auth(MISSING_KEY_MESSAGE.to_string()))
+}
+
+pub struct MistralOcrConfig;
+
+pub const MISTRAL_OCR_CONFIG: MistralOcrConfig = MistralOcrConfig;
+
+impl OcrProviderConfig for MistralOcrConfig {
+    fn supported_ocr_params(&self) -> &'static [&'static str] {
+        SUPPORTED_OCR_PARAMS
+    }
+
+    fn transform_ocr_request(
+        &self,
+        model: &str,
+        document: Value,
+        optional_params: Map<String, Value>,
+    ) -> CoreResult<OcrRequestData> {
+        if !document.is_object() {
+            return Err(CoreError::InvalidType {
+                expected: "object",
+                actual: json_type_name(&document),
+            });
+        }
+
+        let mut data = Map::new();
+        data.insert("model".to_string(), Value::String(model.to_string()));
+        data.insert("document".to_string(), document);
+        for (param, value) in optional_params {
+            data.insert(param, value);
+        }
+
+        Ok(OcrRequestData {
+            data: Value::Object(data),
+            files: None,
+        })
+    }
+
+    fn transform_ocr_response(
+        &self,
+        model: &str,
+        response_json: Value,
+    ) -> CoreResult<OcrResponseData> {
+        let response_object = response_json
+            .as_object()
+            .ok_or_else(|| CoreError::InvalidType {
+                expected: "object",
+                actual: json_type_name(&response_json),
+            })?;
+
+        let pages = response_object
+            .get("pages")
+            .and_then(Value::as_array)
+            .cloned()
+            .unwrap_or_default();
+        let model = response_object
+            .get("model")
+            .and_then(Value::as_str)
+            .unwrap_or(model)
+            .to_string();
+        let document_annotation = response_object.get("document_annotation").cloned();
+        let usage_info = response_object.get("usage_info").cloned();
+
+        Ok(OcrResponseData {
+            pages,
+            model,
+            document_annotation,
+            usage_info,
+            object: "ocr".to_string(),
+        })
+    }
+}
+
+pub fn supported_ocr_params() -> &'static [&'static str] {
+    MISTRAL_OCR_CONFIG.supported_ocr_params()
+}
+
+pub fn map_ocr_params(non_default_params: &Map<String, Value>) -> Map<String, Value> {
+    MISTRAL_OCR_CONFIG.map_ocr_params(non_default_params)
+}
+
+pub fn transform_ocr_request(
+    model: &str,
+    document: Value,
+    optional_params: Map<String, Value>,
+) -> CoreResult<OcrRequestData> {
+    MISTRAL_OCR_CONFIG.transform_ocr_request(model, document, optional_params)
+}
+
+pub fn transform_ocr_response(model: &str, response_json: Value) -> CoreResult<OcrResponseData> {
+    MISTRAL_OCR_CONFIG.transform_ocr_response(model, response_json)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use serde_json::json;
+
+    #[test]
+    fn supported_params_match_python_mistral_ocr_config() {
+        assert_eq!(
+            supported_ocr_params(),
+            &[
+                "pages",
+                "include_image_base64",
+                "image_limit",
+                "image_min_size",
+                "bbox_annotation_format",
+                "document_annotation_format",
+                "document_annotation_prompt",
+                "extract_header",
+                "extract_footer",
+                "table_format",
+                "confidence_scores_granularity",
+                "id",
+            ]
+        );
+    }
+
+    #[test]
+    fn map_ocr_params_drops_unknown_params() {
+        let params = json!({
+            "extract_header": true,
+            "unsupported_param": "value",
+            "pages": [0, 1]
+        });
+        let mapped = map_ocr_params(params.as_object().unwrap());
+
+        assert_eq!(mapped.get("extract_header"), Some(&json!(true)));
+        assert_eq!(mapped.get("pages"), Some(&json!([0, 1])));
+        assert!(!mapped.contains_key("unsupported_param"));
+    }
+
+    #[test]
+    fn transform_ocr_request_builds_mistral_body() {
+        let document = json!({
+            "type": "document_url",
+            "document_url": "https://example.com/doc.pdf"
+        });
+        let optional_params = json!({
+            "include_image_base64": true,
+            "table_format": "html"
+        })
+        .as_object()
+        .unwrap()
+        .clone();
+
+        let result = transform_ocr_request("mistral-ocr-latest", document.clone(), optional_params)
+            .expect("request should transform");
+
+        assert_eq!(
+            result.data,
+            json!({
+                "model": "mistral-ocr-latest",
+                "document": document,
+                "include_image_base64": true,
+                "table_format": "html"
+            })
+        );
+        assert_eq!(result.files, None);
+    }
+
+    #[test]
+    fn transform_ocr_request_rejects_non_object_document() {
+        let err = transform_ocr_request("mistral-ocr-latest", json!("bad"), Map::new())
+            .expect_err("string document should be rejected");
+
+        assert_eq!(
+            err,
+            CoreError::InvalidType {
+                expected: "object",
+                actual: "string",
+            }
+        );
+    }
+
+    #[test]
+    fn transform_ocr_response_normalizes_mistral_json() {
+        let response = json!({
+            "pages": [{"index": 0, "markdown": "hello"}],
+            "model": "mistral-ocr-2505-completion",
+            "document_annotation": null,
+            "usage_info": {"pages_processed": 1}
+        });
+
+        let result = transform_ocr_response("mistral-ocr-latest", response)
+            .expect("response should transform");
+
+        assert_eq!(result.pages, vec![json!({"index": 0, "markdown": "hello"})]);
+        assert_eq!(result.model, "mistral-ocr-2505-completion");
+        assert_eq!(result.document_annotation, Some(Value::Null));
+        assert_eq!(result.usage_info, Some(json!({"pages_processed": 1})));
+        assert_eq!(result.object, "ocr");
+    }
+
+    #[test]
+    fn complete_url_defaults_and_dedupes_v1() {
+        assert_eq!(complete_url(None), "https://api.mistral.ai/v1/ocr");
+        assert_eq!(complete_url(Some("   ")), "https://api.mistral.ai/v1/ocr");
+        assert_eq!(
+            complete_url(Some("https://proxy.internal")),
+            "https://proxy.internal/v1/ocr"
+        );
+        assert_eq!(
+            complete_url(Some("https://proxy.internal/v1/")),
+            "https://proxy.internal/v1/ocr"
+        );
+    }
+
+    #[test]
+    fn resolve_api_key_prefers_param_then_env() {
+        let no_env = |_: &str| None;
+        assert_eq!(
+            resolve_api_key(Some("sk-param"), &no_env).unwrap(),
+            "sk-param"
+        );
+
+        let with_env = |key: &str| (key == MISTRAL_API_KEY_ENV).then(|| "sk-env".to_string());
+        assert_eq!(resolve_api_key(None, &with_env).unwrap(), "sk-env");
+        // Blank param falls through to the environment.
+        assert_eq!(resolve_api_key(Some("  "), &with_env).unwrap(), "sk-env");
+    }
+
+    #[test]
+    fn resolve_api_key_errors_when_absent() {
+        let err = resolve_api_key(None, &|_| None).expect_err("missing key should error");
+        assert_eq!(err, CoreError::Auth(MISSING_KEY_MESSAGE.to_string()));
+    }
+}
--- a/litellm-rust/crates/core/src/providers/mod.rs
+++ b/litellm-rust/crates/core/src/providers/mod.rs
@ -0,0 +1,2 @@
+pub mod mistral;
+pub mod openai;
--- a/litellm-rust/crates/core/src/providers/openai/mod.rs
+++ b/litellm-rust/crates/core/src/providers/openai/mod.rs
@ -0,0 +1 @@
+pub mod realtime;
--- a/litellm-rust/crates/core/src/providers/openai/realtime/mod.rs
+++ b/litellm-rust/crates/core/src/providers/openai/realtime/mod.rs
@ -0,0 +1 @@
+pub mod transformation;
--- a/litellm-rust/crates/core/src/providers/openai/realtime/transformation.rs
+++ b/litellm-rust/crates/core/src/providers/openai/realtime/transformation.rs
@ -0,0 +1,189 @@
+use crate::realtime::transformation::RealtimeProviderConfig;
+use crate::realtime::types::{RealtimeEvent, RealtimeTransformResult};
+use crate::CoreResult;
+
+/// Default OpenAI API base, used when the caller does not override `api_base`.
+pub const OPENAI_REALTIME_DEFAULT_API_BASE: &str = "https://api.openai.com";
+
+/// Path appended to the resolved host base to reach the realtime endpoint.
+pub const OPENAI_REALTIME_PATH: &str = "/v1/realtime";
+
+/// Percent-encode a query value, escaping any char outside the RFC 3986
+/// unreserved set (`A-Za-z0-9-._~`). Keeps us dependency-free; common realtime
+/// model slugs have no special chars, but this stays correct for the rest.
+fn percent_encode(value: &str) -> String {
+    let mut encoded = String::with_capacity(value.len());
+    for byte in value.bytes() {
+        let unreserved = byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'.' | b'_' | b'~');
+        if unreserved {
+            encoded.push(byte as char);
+        } else {
+            encoded.push('%');
+            encoded.push_str(&format!("{byte:02X}"));
+        }
+    }
+    encoded
+}
+
+/// Build the realtime WebSocket URL, porting Python's `OpenAIRealtime._construct_url`.
+///
+/// Blank/whitespace `api_base` is treated as absent (guard at resolution time),
+/// falling back to the default. The scheme is swapped to its WebSocket
+/// equivalent (`https://`→`wss://`, `http://`→`ws://`); bases already using
+/// `ws`/`wss` are left untouched. A bare host or unrecognized scheme defaults to
+/// secure `wss://` so we never hand a scheme-less URL to the connector (this is
+/// a deliberate hardening over Python's `_construct_url`, which would emit a
+/// scheme-less URL here). A trailing `/` is trimmed before the path and
+/// `?model=<encoded>` are appended.
+pub fn complete_url(api_base: Option<&str>, model: &str) -> String {
+    let base = api_base
+        .map(str::trim)
+        .filter(|base| !base.is_empty())
+        .unwrap_or(OPENAI_REALTIME_DEFAULT_API_BASE);
+
+    let base = if let Some(rest) = base.strip_prefix("https://") {
+        format!("wss://{rest}")
+    } else if let Some(rest) = base.strip_prefix("http://") {
+        format!("ws://{rest}")
+    } else if base.starts_with("wss://") || base.starts_with("ws://") {
+        base.to_string()
+    } else {
+        format!("wss://{base}")
+    };
+
+    let base = base.trim_end_matches('/');
+
+    format!(
+        "{base}{OPENAI_REALTIME_PATH}?model={}",
+        percent_encode(model)
+    )
+}
+
+pub struct OpenAiRealtimeConfig;
+
+pub const OPENAI_REALTIME_CONFIG: OpenAiRealtimeConfig = OpenAiRealtimeConfig;
+
+impl RealtimeProviderConfig for OpenAiRealtimeConfig {
+    fn complete_url(&self, api_base: Option<&str>, model: &str) -> String {
+        complete_url(api_base, model)
+    }
+
+    fn transform_realtime_request(
+        &self,
+        event: &RealtimeEvent,
+        _model: &str,
+    ) -> CoreResult<RealtimeTransformResult> {
+        Ok(RealtimeTransformResult::passthrough(event.clone()))
+    }
+
+    fn transform_realtime_response(
+        &self,
+        event: &RealtimeEvent,
+        _model: &str,
+    ) -> CoreResult<RealtimeTransformResult> {
+        Ok(RealtimeTransformResult::passthrough(event.clone()))
+    }
+}
+
+pub fn transform_realtime_request(
+    event: &RealtimeEvent,
+    model: &str,
+) -> CoreResult<RealtimeTransformResult> {
+    OPENAI_REALTIME_CONFIG.transform_realtime_request(event, model)
+}
+
+pub fn transform_realtime_response(
+    event: &RealtimeEvent,
+    model: &str,
+) -> CoreResult<RealtimeTransformResult> {
+    OPENAI_REALTIME_CONFIG.transform_realtime_response(event, model)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn complete_url_defaults_to_openai_wss() {
+        assert_eq!(
+            complete_url(None, "gpt-4o-realtime-preview"),
+            "wss://api.openai.com/v1/realtime?model=gpt-4o-realtime-preview"
+        );
+    }
+
+    #[test]
+    fn complete_url_blank_base_uses_default() {
+        assert_eq!(
+            complete_url(Some("   "), "gpt-4o-realtime-preview"),
+            "wss://api.openai.com/v1/realtime?model=gpt-4o-realtime-preview"
+        );
+    }
+
+    #[test]
+    fn complete_url_swaps_http_to_ws() {
+        assert_eq!(
+            complete_url(Some("http://localhost:8080"), "gpt-4o-realtime-preview"),
+            "ws://localhost:8080/v1/realtime?model=gpt-4o-realtime-preview"
+        );
+    }
+
+    #[test]
+    fn complete_url_dedupes_trailing_slash() {
+        assert_eq!(
+            complete_url(Some("https://api.openai.com/"), "gpt-4o-realtime-preview"),
+            "wss://api.openai.com/v1/realtime?model=gpt-4o-realtime-preview"
+        );
+    }
+
+    #[test]
+    fn complete_url_custom_base() {
+        assert_eq!(
+            complete_url(Some("https://oai.azure.example"), "gpt-4o-realtime-preview"),
+            "wss://oai.azure.example/v1/realtime?model=gpt-4o-realtime-preview"
+        );
+    }
+
+    #[test]
+    fn complete_url_preserves_existing_wss_scheme() {
+        assert_eq!(
+            complete_url(Some("wss://api.openai.com"), "gpt-realtime"),
+            "wss://api.openai.com/v1/realtime?model=gpt-realtime"
+        );
+    }
+
+    #[test]
+    fn complete_url_bare_host_defaults_to_wss() {
+        assert_eq!(
+            complete_url(Some("api.openai.com"), "gpt-realtime"),
+            "wss://api.openai.com/v1/realtime?model=gpt-realtime"
+        );
+    }
+
+    #[test]
+    fn complete_url_percent_encodes_model_space() {
+        assert_eq!(
+            complete_url(None, "gpt 4o"),
+            "wss://api.openai.com/v1/realtime?model=gpt%204o"
+        );
+    }
+
+    #[test]
+    fn transform_realtime_request_passthrough_preserves_event() {
+        let event: RealtimeEvent =
+            serde_json::from_str(r#"{"type":"session.update","session":{"voice":"alloy"}}"#)
+                .expect("valid event");
+        let result =
+            transform_realtime_request(&event, "gpt-realtime").expect("passthrough is infallible");
+        assert_eq!(result.events, vec![event]);
+    }
+
+    #[test]
+    fn transform_realtime_response_passthrough_preserves_event() {
+        let event: RealtimeEvent =
+            serde_json::from_str(r#"{"type":"response.output_audio.delta","delta":"abc=="}"#)
+                .expect("valid event");
+        let result =
+            transform_realtime_response(&event, "gpt-realtime").expect("passthrough is infallible");
+        assert_eq!(result.events, vec![event]);
+    }
+}
--- a/litellm-rust/crates/core/src/realtime/mod.rs
+++ b/litellm-rust/crates/core/src/realtime/mod.rs
@ -0,0 +1,2 @@
+pub mod transformation;
+pub mod types;
--- a/litellm-rust/crates/core/src/realtime/transformation.rs
+++ b/litellm-rust/crates/core/src/realtime/transformation.rs
@ -0,0 +1,22 @@
+use crate::realtime::types::{RealtimeEvent, RealtimeTransformResult};
+use crate::CoreResult;
+
+pub trait RealtimeProviderConfig {
+    /// Build the upstream WebSocket URL (e.g. `wss://api.openai.com/v1/realtime?model=…`).
+    /// Pure string construction only — no network, no env.
+    fn complete_url(&self, api_base: Option<&str>, model: &str) -> String;
+
+    /// Transform a client → backend event before it is forwarded upstream.
+    fn transform_realtime_request(
+        &self,
+        event: &RealtimeEvent,
+        model: &str,
+    ) -> CoreResult<RealtimeTransformResult>;
+
+    /// Transform a backend → client event before it is forwarded downstream.
+    fn transform_realtime_response(
+        &self,
+        event: &RealtimeEvent,
+        model: &str,
+    ) -> CoreResult<RealtimeTransformResult>;
+}
--- a/litellm-rust/crates/core/src/realtime/types.rs
+++ b/litellm-rust/crates/core/src/realtime/types.rs
@ -0,0 +1,60 @@
+use serde::{Deserialize, Serialize};
+use serde_json::{Map, Value};
+
+/// A single realtime event exchanged over the WebSocket.
+///
+/// The `type` discriminator is a typed field; the remaining fields are
+/// preserved losslessly in `data` so a transform can pass an event through, or
+/// inspect/modify specific fields, without enumerating every event variant.
+/// Wire (de)serialization happens at the host edge — `core`/`providers` operate
+/// only on this typed form.
+#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
+pub struct RealtimeEvent {
+    #[serde(rename = "type")]
+    pub event_type: String,
+    #[serde(flatten)]
+    pub data: Map<String, Value>,
+}
+
+/// One or more typed events produced by a realtime transform.
+#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
+pub struct RealtimeTransformResult {
+    pub events: Vec<RealtimeEvent>,
+}
+
+impl RealtimeTransformResult {
+    /// Forward a single event unchanged (the OpenAI baseline).
+    pub fn passthrough(event: RealtimeEvent) -> Self {
+        Self {
+            events: vec![event],
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn event(raw: &str) -> RealtimeEvent {
+        serde_json::from_str(raw).expect("valid event json")
+    }
+
+    #[test]
+    fn realtime_event_round_trips_type_and_extra_fields() {
+        let raw = r#"{"type":"response.output_text.delta","delta":"hi","response_id":"r1"}"#;
+        let parsed = event(raw);
+        assert_eq!(parsed.event_type, "response.output_text.delta");
+        assert_eq!(parsed.data.get("delta"), Some(&Value::String("hi".into())));
+        // Re-serializing yields a semantically-equal event (key order may differ).
+        let reparsed: RealtimeEvent =
+            serde_json::from_str(&serde_json::to_string(&parsed).unwrap()).unwrap();
+        assert_eq!(parsed, reparsed);
+    }
+
+    #[test]
+    fn passthrough_produces_single_element_vec() {
+        let parsed = event(r#"{"type":"session.update"}"#);
+        let result = RealtimeTransformResult::passthrough(parsed.clone());
+        assert_eq!(result.events, vec![parsed]);
+    }
+}
--- a/litellm-rust/crates/core/src/router/deployment.rs
+++ b/litellm-rust/crates/core/src/router/deployment.rs
@ -0,0 +1,44 @@
+//! `model_list` data types, mirroring Python's deployment dict. Deserialize-ready
+//! so a deployment can be loaded straight from the proxy config's `model_list`.
+
+use serde::Deserialize;
+
+/// Per-deployment call parameters, mirroring Python's `litellm_params`.
+#[derive(Clone, Debug, Deserialize)]
+pub struct LiteLLMParams {
+    /// Provider model, e.g. `gpt-realtime` or `openai/gpt-realtime`.
+    pub model: String,
+    #[serde(default)]
+    pub api_key: Option<String>,
+    #[serde(default)]
+    pub api_base: Option<String>,
+}
+
+/// One entry of the `model_list`, mirroring Python's deployment dict.
+#[derive(Clone, Debug, Deserialize)]
+pub struct Deployment {
+    /// Public alias clients request, e.g. `gpt-realtime`.
+    pub model_name: String,
+    pub litellm_params: LiteLLMParams,
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn deserializes_from_model_list_entry() {
+        let entry = r#"{
+            "model_name": "gpt-realtime",
+            "litellm_params": {"model": "openai/gpt-realtime", "api_base": "https://x"}
+        }"#;
+        let deployment: Deployment = serde_json::from_str(entry).expect("valid entry");
+        assert_eq!(deployment.model_name, "gpt-realtime");
+        assert_eq!(deployment.litellm_params.model, "openai/gpt-realtime");
+        assert_eq!(deployment.litellm_params.api_key, None);
+        assert_eq!(
+            deployment.litellm_params.api_base.as_deref(),
+            Some("https://x")
+        );
+    }
+}
--- a/litellm-rust/crates/core/src/router/mod.rs
+++ b/litellm-rust/crates/core/src/router/mod.rs
@ -0,0 +1,93 @@
+//! Minimal Rust port of LiteLLM's `router.py` deployment selection.
+//!
+//! A [`Router`] is built from a `model_list` of [`Deployment`]s
+//! (`{ model_name, litellm_params: { model, api_key, api_base } }`) and selects
+//! one per request via a [`RoutingStrategy`]. For now the only strategy is
+//! `simple-shuffle` — a uniform random pick within a `model_name` group.
+//!
+//! This stays pure (no I/O): it only *chooses* a deployment. The host (the
+//! gateway) takes the chosen deployment and performs the actual provider call.
+//!
+//! - [`deployment`] — the `model_list` data types.
+//! - [`strategy`] — how a deployment is chosen.
+
+mod deployment;
+mod strategy;
+
+pub use deployment::{Deployment, LiteLLMParams};
+pub use strategy::RoutingStrategy;
+
+/// Load-balancing router over a `model_list`.
+#[derive(Clone, Debug, Default)]
+pub struct Router {
+    model_list: Vec<Deployment>,
+    routing_strategy: RoutingStrategy,
+}
+
+impl Router {
+    /// Build a router from a `model_list` using the default `simple-shuffle` strategy.
+    pub fn new(model_list: Vec<Deployment>) -> Self {
+        Self {
+            model_list,
+            routing_strategy: RoutingStrategy::SimpleShuffle,
+        }
+    }
+
+    /// All deployments in the `model_list`. Read-only; used by the host to
+    /// enumerate upstreams (e.g. to pre-warm a connection pool per deployment).
+    pub fn deployments(&self) -> &[Deployment] {
+        &self.model_list
+    }
+
+    /// Whether any deployment is registered under `model`.
+    pub fn has_deployment(&self, model: &str) -> bool {
+        self.model_list
+            .iter()
+            .any(|deployment| deployment.model_name == model)
+    }
+
+    /// Pick a deployment for `model` per the routing strategy. Returns `None`
+    /// when no deployment is registered under that `model_name`.
+    pub fn get_available_deployment(&self, model: &str) -> Option<&Deployment> {
+        let candidates: Vec<&Deployment> = self
+            .model_list
+            .iter()
+            .filter(|deployment| deployment.model_name == model)
+            .collect();
+        self.routing_strategy.select(&candidates)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn deployment(name: &str, model: &str) -> Deployment {
+        Deployment {
+            model_name: name.to_string(),
+            litellm_params: LiteLLMParams {
+                model: model.to_string(),
+                api_key: None,
+                api_base: None,
+            },
+        }
+    }
+
+    #[test]
+    fn selects_a_matching_deployment() {
+        let router = Router::new(vec![
+            deployment("gpt-realtime", "gpt-realtime"),
+            deployment("other", "other-model"),
+        ]);
+        let chosen = router
+            .get_available_deployment("gpt-realtime")
+            .expect("a deployment should match");
+        assert_eq!(chosen.model_name, "gpt-realtime");
+    }
+
+    #[test]
+    fn unknown_model_returns_none() {
+        let router = Router::new(vec![deployment("gpt-realtime", "gpt-realtime")]);
+        assert!(router.get_available_deployment("missing").is_none());
+    }
+}
--- a/litellm-rust/crates/core/src/router/strategy/mod.rs
+++ b/litellm-rust/crates/core/src/router/strategy/mod.rs
@ -0,0 +1,26 @@
+//! Routing policy: how the router picks one deployment from a model group.
+//!
+//! One module per strategy; [`RoutingStrategy::select`] dispatches to it. New
+//! strategies (least-busy, latency-based, …) get their own file here.
+
+mod simple_shuffle;
+
+use super::Deployment;
+
+/// How the router chooses among the deployments sharing a `model_name`.
+#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
+pub enum RoutingStrategy {
+    /// Uniform random pick among the matching deployments.
+    #[default]
+    SimpleShuffle,
+}
+
+impl RoutingStrategy {
+    /// Choose one deployment from `candidates` (all sharing the requested
+    /// `model_name`). Returns `None` when there are no candidates.
+    pub fn select<'a>(&self, candidates: &[&'a Deployment]) -> Option<&'a Deployment> {
+        match self {
+            RoutingStrategy::SimpleShuffle => simple_shuffle::select(candidates),
+        }
+    }
+}
--- a/Show more
+++ b/Show more