Merge branch 'main' into fix/prisma-engine-reap

Signed-off-by: Ian Alloway <200005377+ianalloway@users.noreply.github.com>

.agents/skills/database-access/SKILL.md

@@ -1,6 +1,8 @@
 ---
 name: database-access
 description: Add database queries, extract existing Prisma calls into repositories, or explicitly port queries to Rust and SQLx without expanding migration scope
+metadata:
+  internal: true
 ---
 
 # Database access

.github/CODEOWNERS

@@ -1,4 +1,6 @@
 /model_prices_and_context_window.json @mateo-berri @ryan-crabbe-berri @kerry-berri @joshua-berri
 /litellm/model_prices_and_context_window_backup.json @mateo-berri @ryan-crabbe-berri @kerry-berri @joshua-berri
+/ci_cd/cost_map_pins.json @mateo-berri @ryan-crabbe-berri @kerry-berri @joshua-berri
 /helm/ @yassin-berriai
 /terraform/ @yassin-berriai
+/.github/workflows/zizmor.yml @yuneng-berri

.github/PULL_REQUEST_TEMPLATE/general.md

@@ -84,9 +84,10 @@ If you're seeing a delay in your PR being merged, ping the LiteLLM Team on [Slac
 ## Screenshots / Proof of Fix
 
 <!-- Include screenshots, screen recordings, or command (e.g., curl) + output demonstrating that your changes work as expected
-     The proof must be completely e2e with no mocks, using actual LLM calls costing real $$$ if applicable. `pytest` commands are not enough
-     Show ONLY the latest run: capture Before at the merge base and After at the PR's current tip, and when new commits change behavior, replace this whole section with the fresh run instead of stacking it on top of older ones. The run must be up to date. As soon as a new commit is made and it makes this PR description's after sha stale (it's no longer tip of PR), you must re-run the QA
-     Structure the section exactly as below: Before and After one heading level below this section, each naming the commit hash it was captured at, one lower-level heading per case inside each, the same case names in the same order on both sides, and numbered steps (command, observed output) under every case, never loose prose; shared setup (config, payloads) goes above Before, and with a single case, drop the case headings and number the steps directly
+     Simple PR exception (2026-10-28, #49030). A mechanical change fully proven by focused checks may skip before/after e2e. Follow AGENTS.md's eligibility rule, trace callers, and verify bug-fix regression sensitivity. Record E2E exception at <tip SHA>, scope, rationale, commands and results here. Tests/static checks suffice only for this exception. Keep CI and bot gates, reassess changed diffs, and honor explicit e2e requests. Omit the Before/After scaffold for an exception
+     Otherwise proof must be fully e2e, no mocks, with real provider calls where applicable. Pytest alone is insufficient. The following format applies to live proof
+     Show only the latest proof. Before uses the merge base and After the tested tip. Rerun affected QA after behavior changes. Verified neutral diffs may carry proof with the tested SHA, current tip, equivalence reason and focused checks, following AGENTS.md and /qa
+     Use matching Before/After case headings with numbered commands and observed output. Put shared setup above them. For one case, omit case headings
 
 ### Before (<hash>)
 

.github/scripts/smoke_test_native_wheel.py

@@ -7,6 +7,8 @@ import zipfile
 from pathlib import Path
 from typing import Final
 
+COST_MAP: Final = Path(__file__).resolve().parents[2] / "model_prices_and_context_window.json"
+
 CHILD_SCRIPT: Final = """
 from importlib.util import module_from_spec, spec_from_file_location
 from pathlib import Path
@@ -34,6 +36,9 @@ else:
 after = module.gil_stats()
 if not isinstance(after.get("releases"), int):
     raise AssertionError(f"native module unusable after panic: {after!r}")
+
+if module.bundled_model_cost_map() != Path(sys.argv[2]).read_bytes():
+    raise AssertionError("the wheel embeds a different model_prices_and_context_window.json than this checkout")
 """
 
 
@@ -55,7 +60,9 @@ def main() -> int:
 
         native_path: Final = Path(temporary_directory) / Path(native_members[0].filename).name
         native_path.write_bytes(archive.read(native_members[0]))
-        result: Final = subprocess.run((sys.executable, "-c", CHILD_SCRIPT, str(native_path)), check=False)
+        result: Final = subprocess.run(
+            (sys.executable, "-c", CHILD_SCRIPT, str(native_path), str(COST_MAP)), check=False
+        )
 
     if result.returncode != 0:
         sys.stderr.write(f"native wheel smoke test exited with status {result.returncode}\n")

.github/workflows/cost-map-guard.yml

@@ -1,7 +1,7 @@
 name: Cost map guard
 
-on: # zizmor: ignore[dangerous-triggers] runs the base branch's code only; the PR's cost map files are read as data and never executed
-  pull_request_target:
+on:
+  pull_request:
     branches:
       - main
       - "litellm_**"
@@ -19,17 +19,20 @@ jobs:
     steps:
       - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
         with:
+          ref: ${{ github.event.pull_request.base.sha }}
           persist-credentials: false
-      - name: Fetch the pull request head and its merge base
+      - name: Fetch the pull request head, its merge base, and the base branch tip
         id: revisions
         env:
           GH_TOKEN: ${{ github.token }}
-          BASE_SHA: ${{ github.event.pull_request.base.sha }}
+          BASE_REF: ${{ github.event.pull_request.base.ref }}
           HEAD_SHA: ${{ github.event.pull_request.head.sha }}
         run: |
-          merge_base="$(gh api "repos/${GITHUB_REPOSITORY}/compare/${BASE_SHA}...${HEAD_SHA}" --jq '.merge_base_commit.sha')"
-          git fetch --no-tags --depth=1 origin "$merge_base" "$HEAD_SHA"
+          target="$(gh api "repos/${GITHUB_REPOSITORY}/branches/${BASE_REF}" --jq '.commit.sha')"
+          merge_base="$(gh api "repos/${GITHUB_REPOSITORY}/compare/${target}...${HEAD_SHA}" --jq '.merge_base_commit.sha')"
+          git fetch --no-tags --depth=1 origin "$merge_base" "$HEAD_SHA" "$target"
           echo "merge_base=$merge_base" >> "$GITHUB_OUTPUT"
+          echo "target=$target" >> "$GITHUB_OUTPUT"
       - name: Set up uv
         uses: ./.github/actions/setup-uv-with-retries
         with:
@@ -37,7 +40,30 @@ jobs:
       - name: Run the guard
         env:
           MERGE_BASE: ${{ steps.revisions.outputs.merge_base }}
+          COST_MAP_GUARD_TARGET: ${{ steps.revisions.outputs.target }}
+          COST_MAP_GUARD_AUTHOR_TYPE: ${{ github.event.pull_request.user.type }}
           HEAD_SHA: ${{ github.event.pull_request.head.sha }}
           HEAD_REF: ${{ github.event.pull_request.head.ref }}
         run: |
           uv run --no-project python ci_cd/cost_map_guard.py --base "$MERGE_BASE" --head "$HEAD_SHA" --head-ref "$HEAD_REF"
+      - name: Check whether the pull request changes the cost map
+        id: changed
+        env:
+          MERGE_BASE: ${{ steps.revisions.outputs.merge_base }}
+          HEAD_SHA: ${{ github.event.pull_request.head.sha }}
+        run: |
+          if git diff --quiet "$MERGE_BASE" "$HEAD_SHA" -- model_prices_and_context_window.json; then
+            echo "cost_map=false" >> "$GITHUB_OUTPUT"
+          else
+            echo "cost_map=true" >> "$GITHUB_OUTPUT"
+          fi
+      - if: steps.changed.outputs.cost_map == 'true'
+        run: rustup toolchain install --no-self-update
+      - name: Load the cost map through model-catalog
+        if: steps.changed.outputs.cost_map == 'true'
+        env:
+          HEAD_SHA: ${{ github.event.pull_request.head.sha }}
+        run: |
+          git show "${HEAD_SHA}:model_prices_and_context_window.json" > "$RUNNER_TEMP/model_prices_and_context_window.json"
+          cargo run --locked --quiet --manifest-path litellm-rust/Cargo.toml -p litellm-model-catalog --bin check-registry \
+            -- "$RUNNER_TEMP/model_prices_and_context_window.json"

.github/workflows/cost-map-pins-recheck.yml

@@ -0,0 +1,91 @@
+name: Cost map pins recheck
+
+on:
+  push:
+    branches:
+      - main
+      - "litellm_**"
+    paths:
+      - model_prices_and_context_window.json
+      - litellm/model_prices_and_context_window_backup.json
+      - ci_cd/cost_map_pins.json
+
+permissions: {}
+
+jobs:
+  rerun-cost-map-guard:
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    permissions:
+      actions: write
+      contents: read
+      pull-requests: read
+    steps:
+      - name: Re-run the cost map guard on open pull requests the push could break
+        env:
+          GH_TOKEN: ${{ github.token }}
+          BASE_REF: ${{ github.ref_name }}
+          BEFORE_SHA: ${{ github.event.before }}
+          AFTER_SHA: ${{ github.event.after }}
+          FORCED: ${{ github.event.forced }}
+          COST_MAP_PATHS: '["model_prices_and_context_window.json", "litellm/model_prices_and_context_window_backup.json", "ci_cd/cost_map_pins.json"]'
+          PINS_PATHS: '["ci_cd/cost_map_pins.json"]'
+        run: |
+          if [[ "$BEFORE_SHA" =~ ^0+$ || "$FORCED" == true ]] \
+            || ! gh api "repos/${GITHUB_REPOSITORY}/compare/${BEFORE_SHA}...${AFTER_SHA}?per_page=1" \
+              --jq '(env.PINS_PATHS | fromjson) as $pins | (.files | length) < 300 and all(.files[]; [.filename, .previous_filename] | any(IN($pins[])) | not)' \
+              | grep -qx true; then
+            export RECHECK_PATHS="$COST_MAP_PATHS"
+          else
+            export RECHECK_PATHS="$PINS_PATHS"
+          fi
+          rerun_cutoff="$(date -u -d '30 days ago' +%Y-%m-%dT%H:%M:%SZ)"
+          candidates="$(gh api graphql --paginate -F owner="${GITHUB_REPOSITORY_OWNER}" -F name="${GITHUB_REPOSITORY#*/}" -F base="$BASE_REF" \
+            -f query='query($owner: String!, $name: String!, $base: String!, $endCursor: String) {
+              repository(owner: $owner, name: $name) {
+                pullRequests(states: OPEN, baseRefName: $base, first: 25, after: $endCursor) {
+                  nodes { number headRefOid changedFiles files(first: 100) { nodes { path } } }
+                  pageInfo { hasNextPage endCursor }
+                }
+              }
+            }' \
+            --jq '(env.RECHECK_PATHS | fromjson) as $paths | .data.repository.pullRequests.nodes[]
+              | {number, headRefOid, listed: any(.files.nodes[].path; IN($paths[])), unlisted: (.changedFiles > 100)}
+              | select(.listed or .unlisted) | "\(.number) \(.headRefOid) \(.listed)"')"
+          failed=0
+          declare -A rerun_heads
+          while read -r number head_sha listed; do
+            [ -n "$number" ] || continue
+            if [ "$listed" = false ]; then
+              if ! changed="$(gh api --paginate "repos/${GITHUB_REPOSITORY}/pulls/${number}/files?per_page=100" \
+                --jq '(env.RECHECK_PATHS | fromjson) as $paths | .[].filename | select(IN($paths[]))')"; then
+                echo "::error::could not list the files of #$number"
+                failed=1
+                continue
+              fi
+              [ -n "$changed" ] || continue
+            fi
+            [ -z "${rerun_heads[$head_sha]:-}" ] || continue
+            rerun_heads[$head_sha]=1
+            if ! runs="$(gh run list --repo "$GITHUB_REPOSITORY" --workflow cost-map-guard.yml --commit "$head_sha" --limit 100 --json databaseId,createdAt --jq '.[] | "\(.databaseId) \(.createdAt)"')"; then
+              echo "::error::could not list the cost map guard runs of #$number"
+              failed=1
+              continue
+            fi
+            if [ -z "$runs" ]; then
+              echo "::warning::no cost map guard run found for #$number"
+              continue
+            fi
+            while read -r run_id created_at; do
+              if [[ "$created_at" < "$rerun_cutoff" ]]; then
+                echo "::warning::cost map guard run $run_id for #$number is over 30 days old, so GitHub cannot re-run it; #$number gets the pin checks on its next push"
+                continue
+              fi
+              gh run watch "$run_id" --repo "$GITHUB_REPOSITORY" > /dev/null || true
+              if ! gh run rerun "$run_id" --repo "$GITHUB_REPOSITORY"; then
+                echo "::error::could not re-run cost map guard run $run_id for #$number"
+                failed=1
+              fi
+            done <<< "$runs"
+          done <<< "$candidates"
+          exit "$failed"

.github/workflows/image-scan.yml

@@ -13,8 +13,10 @@ on:
       - migrations/Dockerfile
       - migrations/run.py
       - gateway/Dockerfile
+      - gateway/launch.py
       - gateway/main.py
       - gateway/routes/allowlist.py
+      - litellm_cli/**
       - backend/Dockerfile
       - backend/main.py
       - litellm-rust/**

.github/workflows/interactions-spec-canary.yml

@@ -19,7 +19,7 @@ jobs:
     name: interactions-spec-canary
     if: github.repository == 'BerriAI/litellm'
     runs-on: ubuntu-latest
-    timeout-minutes: 15
+    timeout-minutes: 30
     permissions:
       contents: read
     outputs:
@@ -39,8 +39,11 @@ jobs:
         with:
           version: "0.10.9"
 
+      - name: Set up the Rust bridge
+        uses: ./.github/actions/rust-bridge
+
       - name: Install dependencies
-        run: .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra proxy --no-install-project
+        run: .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra proxy
 
       - name: Run the compliance tests against Google's live spec
         id: compliance

.github/workflows/spend-logs-guard.yml

@@ -1,13 +1,9 @@
 name: Spend logs guard
 
-on: # zizmor: ignore[dangerous-triggers] runs the base branch's code only; the PR's diff is read as data and never executed
-  pull_request_target:
+on:
+  pull_request:
     branches:
       - main
-  pull_request_review:
-    types:
-      - submitted
-      - dismissed
 
 permissions:
   contents: read
@@ -23,7 +19,7 @@ jobs:
     steps:
       - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
         with:
-          ref: ${{ github.event.pull_request.base.ref }}
+          ref: ${{ github.event.pull_request.base.sha }}
           persist-credentials: false
       - name: Fetch the pull request head, its merge base and its reviews
         id: revisions

.github/workflows/test-linting.yml

@@ -376,10 +376,10 @@ jobs:
             exit 0
           fi
           files="$(git diff --name-only "$base" HEAD)"
-          if grep -Eq '^(litellm/(proxy|types)/|ui/litellm-dashboard/(src/lib/http/schema\.d\.ts|scripts/gen-api-types\.mjs|package(-lock)?\.json)$|\.github/workflows/test-linting\.yml$)' <<< "$files"; then
+          if grep -Eq '^(litellm/|enterprise/|(pyproject\.toml|uv\.lock)$|ui/litellm-dashboard/(src/lib/http/schema\.d\.ts|scripts/gen-api-types\.mjs|package(-lock)?\.json)$|\.github/workflows/test-linting\.yml$)' <<< "$files"; then
             echo "relevant=true" >> "$GITHUB_OUTPUT"
           else
-            echo "No proxy, types or generator changes in this pull request, nothing to verify."
+            echo "No litellm, enterprise, dependency or generator changes in this pull request, nothing to verify."
             echo "relevant=false" >> "$GITHUB_OUTPUT"
           fi
 
@@ -561,6 +561,9 @@ jobs:
       - name: router_code_coverage
         run: uv run --no-sync python ./tests/code_coverage_tests/router_code_coverage.py
 
+      - name: router_surface_check
+        run: uv run --no-sync python ./tests/code_coverage_tests/router_surface_check.py
+
       - name: test_chat_completion_imports
         run: uv run --no-sync python ./tests/code_coverage_tests/test_chat_completion_imports.py
 

.github/workflows/test-merge-smoke.yml

@@ -132,7 +132,7 @@ jobs:
   windows-sdk:
     name: windows-sdk
     runs-on: windows-2025
-    timeout-minutes: 10
+    timeout-minutes: 30
     env:
       LITELLM_LOCAL_MODEL_COST_MAP: "True"
     defaults:
@@ -154,8 +154,11 @@ jobs:
         with:
           version: "0.10.9"
 
+      - name: Set up the Rust bridge
+        uses: ./.github/actions/rust-bridge
+
       - name: Install dependencies
-        run: .github/scripts/uv_sync_with_retries.sh --frozen --no-install-workspace --group dev --python 3.11
+        run: .github/scripts/uv_sync_with_retries.sh --frozen --group dev --python 3.11
 
       - name: Run the Windows tests
         run: uv run --no-sync python -m pytest --tb=short -v tests/windows_tests

.github/workflows/test-rust.yml

@@ -4,6 +4,7 @@ on:
   push:
     paths:
       - "litellm-rust/**"
+      - "model_prices_and_context_window.json"
       - "litellm/rust_bridge/**"
       - "scripts/generate_trace_types.py"
       - "scripts/trace_codegen/**"
@@ -38,6 +39,7 @@ on:
       - "litellm_**"
     paths:
       - "litellm-rust/**"
+      - "model_prices_and_context_window.json"
       - "litellm/rust_bridge/**"
       - "scripts/generate_trace_types.py"
       - "scripts/trace_codegen/**"

.github/workflows/test-terraform-provider.yml

@@ -12,6 +12,7 @@ on:
     paths:
       - "terraform/provider/**"
       - "litellm/proxy/**"
+      - "enterprise/**"
       - ".github/workflows/test-terraform-provider.yml"
 
 permissions:

.github/workflows/test-unit.yml

@@ -42,7 +42,7 @@ jobs:
         id: rust-bridge-key
         if: steps.changes.outputs.decision != 'skip'
         env:
-          RUST_BRIDGE_CACHE_KEY: ${{ runner.os }}-rust-bridge-editable-${{ hashFiles('litellm-rust/Cargo.lock', 'litellm-rust/Cargo.toml', 'litellm-rust/crates/**', 'pyproject.toml') }}
+          RUST_BRIDGE_CACHE_KEY: ${{ runner.os }}-rust-bridge-editable-${{ hashFiles('litellm-rust/Cargo.lock', 'litellm-rust/Cargo.toml', 'litellm-rust/crates/**', 'model_prices_and_context_window.json', 'pyproject.toml') }}
         run: echo "key=$RUST_BRIDGE_CACHE_KEY" >> "$GITHUB_OUTPUT"
 
       - name: Set up Python
@@ -137,6 +137,7 @@ jobs:
         include:
           - shard: core-utils
             test-path: |-
+              tests/unit/accounting
               tests/unit/batch_completion
               tests/unit/decisions
               tests/unit/extraction
@@ -235,6 +236,7 @@ jobs:
               tests/unit/llms/azure_ai
               tests/unit/llms/base_llm
               tests/unit/llms/baseten
+              tests/unit/llms/bigmodel
               tests/unit/llms/black_forest_labs
               tests/unit/llms/brave
               tests/unit/llms/bytez
@@ -244,6 +246,7 @@ jobs:
               tests/unit/llms/clarifai
               tests/unit/llms/claude_code
               tests/unit/llms/cloudflare
+              tests/unit/llms/codestral
               tests/unit/llms/codex
               tests/unit/llms/cohere
               tests/unit/llms/cometapi
@@ -319,6 +322,7 @@ jobs:
               tests/unit/llms/openai_like
               tests/unit/llms/opencode
               tests/unit/llms/opensandbox
+              tests/unit/llms/opensearch
               tests/unit/llms/openrouter
               tests/unit/llms/ovhcloud
               tests/unit/llms/parallel_ai
@@ -1306,6 +1310,8 @@ jobs:
           done
   openai-sdk-compat:
     name: OpenAI ${{ matrix.openai-version }} compatibility
+    needs: rust-bridge
+    if: ${{ !cancelled() }}
     permissions:
       contents: read
       pull-requests: read
@@ -1342,10 +1348,18 @@ jobs:
         with:
           version: "0.10.9"
 
+      - name: Set up the Rust bridge
+        if: steps.changes.outputs.decision != 'skip'
+        timeout-minutes: 5
+        uses: ./.github/actions/rust-bridge
+        with:
+          artifact: ${{ needs.rust-bridge.outputs.artifact }}
+
       - name: Install dependencies
         if: steps.changes.outputs.decision != 'skip'
         run: |
           .github/scripts/uv_sync_with_retries.sh --frozen --group dev --group proxy-dev --extra proxy --no-install-workspace
+          cp rust-bridge-dist/litellm/rust_bridge/_native.* litellm/rust_bridge/
           uv pip install "openai==${OPENAI_VERSION:?}" --exclude-newer-package openai=2026-10-07T00:00:00Z
           uv run --no-sync python -c 'import os, openai; assert openai.__version__ == os.environ["OPENAI_VERSION"], openai.__version__'
 

.github/workflows/zizmor.yml

@@ -27,10 +27,15 @@ jobs:
         with:
           persist-credentials: false
 
+      - name: Install zizmor
+        run: |
+          curl -fsSL --retry 3 -o "$RUNNER_TEMP/zizmor.tar.gz" \
+            https://github.com/zizmorcore/zizmor/releases/download/v1.31.1/zizmor-x86_64-unknown-linux-gnu.tar.gz
+          echo "d185f726bdaf92df78273217dfdd24ebcd6b0ef48cf3ce776ff5762b6e138c99  $RUNNER_TEMP/zizmor.tar.gz" | sha256sum -c -
+          tar xzf "$RUNNER_TEMP/zizmor.tar.gz" -C "$RUNNER_TEMP" zizmor
+
       - name: Run zizmor
-        uses: zizmorcore/zizmor-action@5f14fd08f7cf1cb1609c1e344975f152c7ee938d # v0.5.6
-        with:
-          version: "1.24.1"
-          min-severity: medium
-          advanced-security: false
-          annotations: true
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          "$RUNNER_TEMP/zizmor" --no-ignores --no-config --min-severity=medium --format=github .

.grype.yaml

@@ -11,3 +11,12 @@ ignore:
     package:
       name: zlib
       type: apk
+  - vulnerability: GHSA-v4x8-3wr5-3wq8
+    reason: >-
+      filelock ReadWriteLock advisory, and no package in the image takes ReadWriteLock;
+      the fix (4.2.1) is blocked by litellm's own filelock<4.0 cap until #48635 or #48802 lifts it,
+      and the image then moves off 3.32.7, which retires this version-scoped rule
+    package:
+      name: filelock
+      version: 3.32.7
+      type: python

AGENTS.md

@@ -31,6 +31,8 @@ A test must only fail when litellm code changes. Never pin facts we don't own (a
 
 End-to-end tests belong in `tests/e2e/`. A rule about one directory's files, like its harness or which CI runs it, goes in that directory's `AGENTS.md`, never only in a README or here. Read a directory's `AGENTS.md` before editing files in it
 
+Before you build a new way to move data between the endpoints, the provider transformations, and cost tracking or logging, read the architecture decision records in `docs/adrs/`, which say why those layers are shaped the way they are
+
 When creating PRs, target the repository's current default branch for both internal and external / OSS contributions. Check it with `python3 scripts/default_branch.py --branch` instead of assuming a branch name or relying on cached `origin/HEAD`
 
 When writing a PR body, treat the comments and imperative instructions inside .github/PULL_REQUEST_TEMPLATE/general.md as rules to follow, not just layout. Agent harnesses may strip HTML comments from copies of that file injected into context, so read .github/PULL_REQUEST_TEMPLATE/general.md from disk before writing a PR body to make sure you see every comment rule. A section you have nothing to put in (Relevant issues, Affected release, Linear ticket, Caveats, QA runbook, and so on) is removed entirely, heading included, never left as an empty title
@@ -39,7 +41,9 @@ Same applies for filing bug reports and feature requests, with .github/ISSUE_TEM
 
 If you're resolving a linear ticket, in the "## Linear ticket" section of the PR, say "Resolves LIT-1234", replacing "LIT-1234" with the actual ticket id that you're resolving. If you don't have the ticket id, don't make one up or search for it. Just drop the section
 
-Never use `pytest` commands or the like as "Screenshots / Proof of Fix". We prefer curl'ing a live proxy instance running on localhost:4000 (I like to run it with `python litellm/proxy/proxy_cli.py --config litellm/proxy/dev_config.yaml --detailed_debug --reload --use_v2_migration_resolver 2>&1 | tee litellm.log`. The Admin UI dev server is `npm run dev` in `ui/litellm-dashboard`, served on port 3000) and showing both the command run and the output. Also, it should hit real LLM provider APIs, not mocks, and cost real $$$ because that is the most realistic test. The proof of fix should be exactly what the end user / customer would see / do. The run logs in PR #27703 is a prime example of how to do it (not a huge fan of using a python test script that future me and the team will have no visibility into. I prefer just curl commands or a short list of bash commands (e.g., using `for`)). If it's a UI thing, or the main use case runs through a headful agentic coding tool like Claude Code or Codex, drive that surface yourself and embed your own before and after screenshots of it in the PR (the Admin UI page, or what the coding tool shows), next to an ordered list of the URLs to go to (e.g., http://localhost:4000/ui/?page=logs), where to click, and what fields to fill out so a reviewer can reproduce it
+Simple PRs may skip before/after e2e when the full diff is mechanical and focused checks completely prove it, such as #49030 restoring an existing helper name. Trace affected callers and run relevant tests/static checks at the tip. For a bug fix, establish that the regression catches the original defect. Size alone never qualifies. New behavior, API/provider semantics, auth, billing, persistence, concurrency or uncertain integration risk need live e2e. In Screenshots / Proof of Fix, record `E2E exception at <tip SHA>`, scope, rationale, commands and results. Tests count as proof only for this exception. Reassess changed diffs, retain CI/bot gates, and honor explicit e2e requests (2026-10-28, unnecessary live QA for #49030)
+
+For other PRs, proof must show the end user's real flow before and after, with commands and output against a live proxy and real provider calls where applicable. Pytest alone is insufficient. For UI or interactive coding-tool flows, drive that surface and embed before/after screenshots, with URLs, clicks and input fields a reviewer can reproduce
 
 If you ever write any human-facing text (pull requests, issues, commit messages, discussion posts, github comments, release notes, docs, etc.), always follow these guidelines to sound less AI-y:
 - don't use emojis
@@ -105,6 +109,7 @@ Follow these coding conventions for new/updated code (a three-line fix in a lega
 - No file sprawl: deliberate file and folder structure
 - Standard over hand-rolled: use the official SDK or a library where one exists. Where none does, follow industry standards instead of inventing local conventions
 - API-fragmentation-aware: when logic must branch on which API surface produced or consumes data (e.g. chat completions vs Anthropic Messages vs Responses API shapes), proactively look for an existing shared helper (e.g. `litellm_core_utils/prompt_templates/factory.py`) before writing per-surface parsing in the new module. If none exists, add one there instead of duplicating the same format-detection logic in every new guardrail/integration
+- Current default for new model names: logic that picks behavior from a model-name table (tokenizer, message framing, supported params) gives a provider's model the table does not list yet that provider's current behavior, never its oldest, since providers ship names faster than any table is updated
 
 Follow conventional commits for commit names and PR titles
 

ARCHITECTURE.md

@@ -2,6 +2,8 @@
 
 This document helps contributors understand where to make changes in LiteLLM.
 
+For why a layer is shaped the way it is, read the architecture decision records in [`docs/adrs/`](docs/adrs/AGENTS.md) (Python SDK and proxy) and [`litellm-rust/docs/adrs/`](litellm-rust/docs/adrs/AGENTS.md) (Rust core)
+
 ---
 
 ## How It Works

CONTRIBUTING.md

@@ -8,6 +8,7 @@ Here are the core requirements for any PR submitted to LiteLLM:
 
 - [ ] **Sign the Contributor License Agreement (CLA)** - [see details](#contributor-license-agreement-cla)
 - [ ] **Keep scope isolated** - Your changes should address 1 specific problem at a time
+- [ ] **Prove it works end to end** - If your change affects what the proxy or SDK does, paste the commands you ran against a live proxy and their output in the PR - [see details](#proving-your-change-works-end-to-end)
 
 #### Proxy (Backend) PRs
 
@@ -248,6 +249,41 @@ If `make test-unit` fails:
 - **Test edge cases**: Don't just test the happy path
 - **Update documentation**: If you change APIs, update docs
 
+## Proving Your Change Works End to End
+
+Tests are required, but they do not prove that your change works for a user. A PR that changes what the proxy or SDK does shows a real run in its "Screenshots / Proof of Fix" section, meaning the exact commands you sent to a live proxy and the output that came back, against real provider APIs and not mocks. `pytest` output does not count as proof, because a passing test only shows that the code does what its mocks expect. A PR that changes only docs or CI shows the rendered page or the CI run instead
+
+A good proof looks like this:
+
+1. Install the proxy dependencies with `make install-proxy-dev`, then start the proxy from your branch with a config that has the model you changed, `uv run litellm --config your_config.yaml --detailed_debug`
+2. Send the request a user would send with `curl`, and paste both the command and the response. When cost or routing is involved, include the response headers, since `x-litellm-response-cost` is what the user sees
+3. For a bug fix, do this twice, once at the merge base with BerriAI/litellm's main (`git merge-base origin/main HEAD`, with your remote for BerriAI/litellm in place of `origin` in a fork clone) to show the bug and once at your branch's latest commit to show the fix. Name both commit hashes
+4. If your change is reachable from more than one endpoint (`/v1/chat/completions`, `/v1/responses`, `/v1/messages`), show each one
+5. If the request can stream, show a `"stream": true` run too, because streaming builds usage and cost on a different path. A stream sends its headers before its usage exists, so show the `usage.cost` on the last chunk, which `/v1/chat/completions` sends when the request has `"stream_options": {"include_usage": true}` and other endpoints send with `include_cost_in_streaming_usage: true` under `litellm_settings`, or, when the proxy has a database, the request's row on the Admin UI's Logs page or `GET /spend/logs`
+6. For UI changes, add before and after screenshots and name the page
+7. If users reach your change through a coding tool such as Claude Code or Codex, drive that tool the way a user does and add before and after screenshots of it, because a `curl` or a headless run alone does not show what the user sees
+
+```bash
+curl -sD - http://localhost:4000/v1/chat/completions \
+  -H "Authorization: Bearer $LITELLM_MASTER_KEY" \
+  -H "Content-Type: application/json" \
+  -d '{"model": "your-model", "messages": [{"role": "user", "content": "hi"}]}'
+```
+
+When the provider reports its own usage or spend numbers, compare the proxy's numbers against them in the proof
+
+## Contributing an LLM Provider Integration
+
+Provider work (a new provider, a new parameter, a new usage or cost field) is the most common kind of contribution, and it is where PRs get reworked most often. Follow these rules
+
+**Find the existing machinery before you add plumbing.** LiteLLM already has generic paths that move data between the endpoints, the provider transformations, and cost tracking. A second, hand-rolled path for the same data is a common reason a provider PR gets rewritten before merge. [`ARCHITECTURE.md`](ARCHITECTURE.md) shows where the layers live, and the decision records in [`docs/adrs/`](docs/adrs/AGENTS.md) explain why they are shaped that way. For example, [the ADR on provider usage fields](docs/adrs/adr_001_provider_usage_fields_ride_as_usage_extras.md) covers how provider-specific usage fields reach cost tracking. If you cannot find a mechanism for what you need, say so in the PR or ask in [#pr-review on Slack](https://join.slack.com/t/litellmossslack/shared_invite/zt-3o7nkuyfr-p_kbNJj8taRfXGgQI1~YyA) before you build one
+
+**Bill a new provider usage field from the usage object.** Leave the field on the usage object as an extra and read it in that provider's `cost_calculator.py`. A provider without one gets a new `cost_calculator.py` and an arm for it in `cost_per_token` (`litellm/cost_calculator.py`), which picks the calculator by provider. If the provider's transform builds `Usage` from named fields, as the Anthropic, Gemini, and Bedrock Converse ones do, pass the new field there too. Streamed requests rebuild usage in `ChunkProcessor.calculate_usage` (`litellm/litellm_core_utils/streaming_chunk_builder_utils.py`), which does not copy extras yet the way the Responses-to-chat bridge does (minus a short exclusion list), so until it does, bill the field on streams by naming it there, as Anthropic's `inference_geo` is. A provider that reports its own charge for the request, as OpenRouter does in `usage.cost`, is billed at that charge from `_hidden_params` before any calculator runs, unless the deployment sets its own prices. When the field counts a built-in tool such as web search, add the provider to `get_cost_for_web_search_request` in `litellm/llms/__init__.py`, and add a gate to `litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py` only when no existing gate fires. Then show the field's cost in your proof on every endpoint it reaches, streaming included
+
+**Do not change what a caller sees to make internals easier.** Each endpoint keeps the schema of the API it emulates. A `/v1/responses` caller gets `input_tokens` and a `/v1/chat/completions` caller gets `prompt_tokens`, whatever the provider sent and whatever cost tracking needs. A provider usage field that shows up as an extra key is fine, since it only adds a key, but renaming, removing, or reshaping a key the endpoint defines is not. Reshaping a public response to feed an internal consumer breaks every user of that endpoint
+
+If your change makes an architecture decision, such as a new cross-cutting mechanism or a deliberate difference from a provider's API shape, add an ADR to [`docs/adrs/`](docs/adrs/AGENTS.md) in the same PR
+
 ## Building and Running Locally
 
 ### LiteLLM Proxy Server

backend/Dockerfile

@@ -24,6 +24,7 @@ ENV UV_PYTHON_DOWNLOADS=0
 COPY pyproject.toml README.md rust-toolchain.toml ./
 COPY .cargo ./.cargo
 COPY litellm-rust ./litellm-rust
+COPY model_prices_and_context_window.json ./
 
 RUN --mount=type=cache,target=/root/.cache/uv \
     mkdir -p litellm/rust_bridge /wheels && \

ci_cd/cost_map_guard.py

@@ -1,37 +1,67 @@
 """Guard the cost map on pull requests.
 
-Every pull request whose diff against its merge base touches one of the two cost map files gets the file
-checks: the files parse and the backup copy matches the root file. A pull request that leaves both untouched
-skips them, since merging it keeps the base branch's copies and its head tree only carries whatever state the
-branch was cut from. Pull requests from the cost map sync bot (branches named litellm_cost_map_sync_*) always
-get the file checks and additionally may only touch those two files and may only add or update models.
+Every pull request whose diff against its merge base touches one of the two cost map files or the pins file gets the
+file checks: the files parse, the backup copy matches the root file, and every capability pinned in
+ci_cd/cost_map_pins.json (a value verified against a live provider call where the provider's model listing says
+otherwise) still holds in the tree that merging the pull request into its base branch produces, so a branch cut before
+a pin was added cannot lower it, and a pull request that deletes the pins file fails too. A pull request that leaves
+all three untouched skips them, since merging it keeps the base branch's copies and its head tree only carries
+whatever state the branch was cut from. Pull requests from the cost map sync bot (branches named
+litellm_cost_map_sync_*) always get the file checks and additionally may only touch the two cost map files and may
+only add or update models, so a sync can never lower a pinned capability or edit the pins. Price sync pull requests
+(branches named litellm-providers/price-sync*) and pull requests a bot account opened (COST_MAP_GUARD_AUTHOR_TYPE is
+Bot) get the human checks and may not touch the pins file either.
 """
 
 from __future__ import annotations
 
 import argparse
 import json
+import os
 import subprocess
 import sys
-from collections.abc import Mapping, Sequence
+from collections.abc import Iterator, Mapping, Sequence
 from dataclasses import dataclass
 from types import MappingProxyType
 from typing import Final
 
 COST_MAP_PATH: Final = "model_prices_and_context_window.json"
 BACKUP_PATH: Final = "litellm/model_prices_and_context_window_backup.json"
+PINS_PATH: Final = "ci_cd/cost_map_pins.json"
 GUARDED_PATHS: Final = (COST_MAP_PATH, BACKUP_PATH)
+CHECKED_PATHS: Final = (*GUARDED_PATHS, PINS_PATH)
 BOT_BRANCH_PREFIX: Final = "litellm_cost_map_sync_"
+PRICE_SYNC_BRANCH_PREFIX: Final = "litellm-providers/price-sync"
+TARGET_ENV: Final = "COST_MAP_GUARD_TARGET"
+AUTHOR_TYPE_ENV: Final = "COST_MAP_GUARD_AUTHOR_TYPE"
+BOT_AUTHOR_TYPE: Final = "Bot"
+MERGE_CONFLICT_EXIT_CODE: Final = 1
 SPECIAL_ROOT_KEYS: Final = frozenset({"sample_spec", "fallback_generalizations"})
 EVALUATION_OUTPUT_PRICE_SOURCES: Final[Mapping[str, str]] = MappingProxyType({})
 
 CostMap = dict[str, object]
+_MISSING: Final = object()
 
 
 @dataclass(frozen=True, slots=True)
 class Snapshot:
     cost_map: str
     backup: str
+    pins: str
+
+
+@dataclass(frozen=True, slots=True)
+class MergeFailed:
+    stderr: str
+
+
+@dataclass(frozen=True, slots=True)
+class Pin:
+    key: str
+    field: str
+    value: object
+    checked: str
+    source: str
 
 
 def _parse_object(text: str, path: str) -> CostMap | str:
@@ -91,6 +121,48 @@ def _entries(cost_map: CostMap) -> dict[str, dict[str, object]]:
     return {key: entry for key, entry in cost_map.items() if isinstance(entry, dict)}
 
 
+def _pin(key: str, field: str, spec: object) -> Pin | str:
+    malformed: Final = f"{PINS_PATH}: {key}.{field} needs value, checked, and source"
+    if not isinstance(spec, dict) or "value" not in spec:
+        return malformed
+    checked: Final = spec.get("checked")
+    source: Final = spec.get("source")
+    if not isinstance(checked, str) or not isinstance(source, str) or not checked or not source:
+        return malformed
+    return Pin(key, field, spec["value"], checked, source)
+
+
+def _pins(parsed: CostMap) -> Iterator[Pin | str]:
+    for key, fields in parsed.items():
+        if not isinstance(fields, dict):
+            yield f"{PINS_PATH}: {key} must map field names to pins"
+            continue
+        for field, spec in fields.items():
+            yield _pin(key, field, spec)
+
+
+def _pin_violations(parsed: CostMap, entries: Mapping[str, Mapping[str, object]]) -> Iterator[str]:
+    for pin in _pins(parsed):
+        if isinstance(pin, str):
+            yield pin
+        elif pin.key not in entries:
+            yield f"{PINS_PATH} pins {pin.key}.{pin.field} but {COST_MAP_PATH} has no {pin.key} entry"
+        elif entries[pin.key].get(pin.field, _MISSING) != pin.value:
+            yield (
+                f"{COST_MAP_PATH}: {pin.key}.{pin.field} must stay {json.dumps(pin.value)} "
+                f"(verified {pin.checked}: {pin.source}); change {PINS_PATH} with new evidence first"
+            )
+
+
+def pin_failures(pins_text: str, head_map: CostMap) -> tuple[str, ...]:
+    if not pins_text:
+        return (f"{PINS_PATH} is missing; restore it, its pins were verified against live provider calls",)
+    parsed: Final = _parse_object(pins_text, PINS_PATH)
+    if isinstance(parsed, str):
+        return (parsed,)
+    return tuple(_pin_violations(parsed, _entries(head_map)))
+
+
 def _bot_failures(base: Snapshot, head_map: CostMap, changed_files: Sequence[str]) -> tuple[str, ...]:
     base_map: Final = _parse_object(base.cost_map, COST_MAP_PATH)
     if isinstance(base_map, str):
@@ -120,8 +192,18 @@ def _bot_failures(base: Snapshot, head_map: CostMap, changed_files: Sequence[str
     )
 
 
+def pin_author_failures(head_ref: str, author_type: str, changed_files: Sequence[str]) -> tuple[str, ...]:
+    if PINS_PATH not in changed_files:
+        return ()
+    if head_ref.startswith(PRICE_SYNC_BRANCH_PREFIX):
+        return (f"price sync PRs may not change {PINS_PATH}; a pin moves only in a human PR with new evidence",)
+    if author_type == BOT_AUTHOR_TYPE:
+        return (f"PRs opened by a bot may not change {PINS_PATH}; a pin moves only in a human PR with new evidence",)
+    return ()
+
+
 def touches_cost_map(changed_files: Sequence[str]) -> bool:
-    return any(path in GUARDED_PATHS for path in changed_files)
+    return any(path in CHECKED_PATHS for path in changed_files)
 
 
 def contract_for(bot: bool, changed_files: Sequence[str]) -> str:
@@ -130,9 +212,21 @@ def contract_for(bot: bool, changed_files: Sequence[str]) -> str:
     return "human PR, file checks only" if touches_cost_map(changed_files) else "human PR, cost map untouched"
 
 
+def _merged_pin_failures(base: Snapshot, merged: Snapshot | None) -> tuple[str, ...]:
+    if merged is None:
+        return ("the pull request does not merge cleanly into its base branch; merge the base branch in first",)
+    if not merged.pins and not base.pins:
+        return ()
+    merged_map: Final = _parse_object(merged.cost_map, COST_MAP_PATH)
+    if isinstance(merged_map, str):
+        return (f"after merging into the base branch, {merged_map}",)
+    return pin_failures(merged.pins, merged_map)
+
+
 def guard_failures(
     base: Snapshot,
     head: Snapshot,
+    merged: Snapshot | None,
     changed_files: Sequence[str],
     bot: bool,
     *,
@@ -147,6 +241,7 @@ def guard_failures(
         *_file_failures(
             head, head_map, evaluation_output_price_sources=evaluation_output_price_sources
         ),
+        *_merged_pin_failures(base, merged),
         *(_bot_failures(base, head_map, changed_files) if bot else ()),
     )
 
@@ -157,7 +252,22 @@ def _git(*args: str) -> str | None:
 
 
 def snapshot(revision: str) -> Snapshot:
-    return Snapshot(*(_git("show", f"{revision}:{path}") or "" for path in GUARDED_PATHS))
+    return Snapshot(*(_git("show", f"{revision}:{path}") or "" for path in CHECKED_PATHS))
+
+
+def merged_tree(merge_base: str, target: str, head: str) -> str | MergeFailed | None:
+    result: Final = subprocess.run(
+        ("git", "merge-tree", "--write-tree", f"--merge-base={merge_base}", target, head),
+        check=False,
+        capture_output=True,
+        text=True,
+    )
+    tree: Final = result.stdout.split("\n", 1)[0]
+    if result.returncode == 0:
+        return tree
+    if result.returncode == MERGE_CONFLICT_EXIT_CODE and tree:
+        return None
+    return MergeFailed(result.stderr.strip())
 
 
 def changed_files(base: str, head: str) -> tuple[str, ...] | None:
@@ -169,6 +279,13 @@ def main(argv: Sequence[str]) -> int:
     parser: Final = argparse.ArgumentParser(description=__doc__)
     parser.add_argument("--base", required=True, help="merge base of the pull request")
     parser.add_argument("--head", required=True, help="head commit of the pull request")
+    env_target: Final = os.environ.get(TARGET_ENV)
+    parser.add_argument(
+        "--target",
+        default=env_target,
+        required=env_target is None,
+        help=f"tip of the base branch the pull request merges into, defaulting to ${TARGET_ENV}",
+    )
     parser.add_argument("--head-ref", required=True, help="head branch name of the pull request")
     args: Final = parser.parse_args(argv)
     bot: Final = args.head_ref.startswith(BOT_BRANCH_PREFIX)
@@ -176,7 +293,19 @@ def main(argv: Sequence[str]) -> int:
     if changed is None:
         print(f"cost map guard failed: git diff {args.base} {args.head} failed, so the changed files are unknown")
         return 1
-    failures: Final = guard_failures(snapshot(args.base), snapshot(args.head), changed, bot)
+    if not bot and not touches_cost_map(changed):
+        print(f"cost map guard passed ({contract_for(bot, changed)})")
+        return 0
+    merged: Final = merged_tree(args.base, args.target, args.head)
+    if isinstance(merged, MergeFailed):
+        print(f"cost map guard failed: git merge-tree into {args.target} failed: {merged.stderr}")
+        return 1
+    failures: Final = (
+        *guard_failures(
+            snapshot(args.base), snapshot(args.head), None if merged is None else snapshot(merged), changed, bot
+        ),
+        *pin_author_failures(args.head_ref, os.environ.get(AUTHOR_TYPE_ENV, ""), changed),
+    )
     contract: Final = contract_for(bot, changed)
     if failures:
         print(f"cost map guard failed ({contract}):")

ci_cd/cost_map_pins.json

@@ -0,0 +1,16 @@
+{
+    "fireworks_ai/accounts/fireworks/models/minimax-m3": {
+        "supports_vision": {
+            "value": true,
+            "checked": "2026-09-26",
+            "source": "POST https://api.fireworks.ai/inference/v1/chat/completions read a 512x512 PNG back as digit=7, shape=circle, color=red while GET https://api.fireworks.ai/inference/v1/models still said supports_image_input false and GET https://api.fireworks.ai/v1/serverless/models?format=nested carried no supportsImageInput on the row"
+        }
+    },
+    "fireworks_ai/minimax-m3": {
+        "supports_vision": {
+            "value": true,
+            "checked": "2026-09-26",
+            "source": "POST https://api.fireworks.ai/inference/v1/chat/completions read a 512x512 PNG back as digit=7, shape=circle, color=red while GET https://api.fireworks.ai/inference/v1/models still said supports_image_input false and GET https://api.fireworks.ai/v1/serverless/models?format=nested carried no supportsImageInput on the row"
+        }
+    }
+}

ci_cd/spend_logs_guard.py

@@ -2,9 +2,12 @@
 needs an approving review from the spend-log owner before it can merge.
 
 CODEOWNERS can only match file paths, and every schema change edits the same schema.prisma copies,
-so a path rule would put every migration on the owner. This gate reads the diff instead: when no
-added line in the watched files mentions the table it passes at once, otherwise it passes only once
-the owner's latest review on the pull request is an approval. The workflow re-runs on every review.
+so a path rule would put every migration on the owner. This gate reads the diff instead: a schema.prisma
+hunk counts when it adds or removes a line inside the `model LiteLLM_SpendLogs { ... }` block, a migration
+hunk counts when an added line names the table. Without such a hunk it passes at once, otherwise it passes
+only once the owner's latest review on the pull request is an approval that still covers every spend-log line
+in the head (approvals survive pushes, so a later commit may not add spend-log lines the owner never saw).
+Reviews do not trigger the workflow, so the check is re-run by hand after approving.
 """
 
 from __future__ import annotations
@@ -13,8 +16,10 @@ import argparse
 import re
 import subprocess
 import sys
-from collections.abc import Sequence
+from collections.abc import Callable, Mapping, Sequence
+from dataclasses import dataclass
 from pathlib import Path
+from types import MappingProxyType
 from typing import Final
 
 from pydantic import BaseModel, ConfigDict, TypeAdapter
@@ -25,7 +30,10 @@ WATCHED: Final = (
     "litellm-proxy-extras/litellm_proxy_extras/schema.prisma",
 )
 MIGRATIONS_PREFIX: Final = "litellm-proxy-extras/litellm_proxy_extras/migrations/"
-TABLE: Final = re.compile(r"(?<![A-Za-z0-9])LiteLLM_SpendLogs", re.IGNORECASE)
+TABLE: Final = re.compile(r"(?<![A-Za-z0-9])LiteLLM_SpendLogs(?![A-Za-z0-9])", re.IGNORECASE)
+DECIDING_STATES: Final = frozenset({"APPROVED", "CHANGES_REQUESTED", "DISMISSED"})
+MODEL_START: Final = re.compile(r"^model\s+LiteLLM_SpendLogs\s*\{")
+HUNK: Final = re.compile(r"^@@ -(\d+)(?:,\d+)? \+(\d+)(?:,\d+)? @@[^\n]*\n", re.MULTILINE)
 
 
 class Reviewer(BaseModel):
@@ -37,31 +45,77 @@ class Review(BaseModel):
     model_config = ConfigDict(frozen=True, extra="ignore")
     state: str
     user: Reviewer
+    commit_id: str | None = None
 
 
 _REVIEWS: Final = TypeAdapter(list[Review])
 
 
+@dataclass(frozen=True, slots=True)
+class Block:
+    """Lines of `model LiteLLM_SpendLogs { ... }` in one schema.prisma copy at the merge base and at the head."""
+
+    base: range
+    head: range
+
+
+@dataclass(frozen=True, slots=True)
+class Hunk:
+    removed: tuple[tuple[int, str], ...]
+    added: tuple[tuple[int, str], ...]
+
+
 def watched(path: str) -> bool:
     return path in WATCHED or (path.startswith(MIGRATIONS_PREFIX) and path.endswith("/migration.sql"))
 
 
-def _added_lines(section: str) -> tuple[str, ...]:
-    body: Final = section.split("\n@@", 1)
-    return tuple(line[1:] for line in body[1].splitlines() if line.startswith("+")) if len(body) == 2 else ()
+def model_block(schema: str) -> range:
+    """1-based line range of `model LiteLLM_SpendLogs { ... }` including both braces, empty when the schema has none."""
+    lines: Final = schema.splitlines()
+    starts: Final = tuple(number for number, line in enumerate(lines, 1) if MODEL_START.match(line))
+    if not starts:
+        return range(0)
+    ends: Final = tuple(number for number, line in enumerate(lines, 1) if number > starts[0] and line.startswith("}"))
+    return range(starts[0], (ends[0] if ends else len(lines)) + 1)
 
 
-def spend_log_hits(diff: str) -> tuple[str, ...]:
-    """`path: added line` for every added line in a watched file that names the table."""
+def _hunks(section: str) -> tuple[Hunk, ...]:
+    headers: Final = tuple(HUNK.finditer(section))
+    ends: Final = tuple(header.start() for header in headers[1:]) + (len(section),)
+    return tuple(_hunk(header, section[header.end() : end]) for header, end in zip(headers, ends))
+
+
+def _hunk(header: re.Match[str], body: str) -> Hunk:
+    lines: Final = body.splitlines()
+    return Hunk(
+        removed=tuple(enumerate((line[1:] for line in lines if line.startswith("-")), int(header.group(1)))),
+        added=tuple(enumerate((line[1:] for line in lines if line.startswith("+")), int(header.group(2)))),
+    )
+
+
+def spend_log_hits(diff: str, blocks: Mapping[str, Block]) -> tuple[str, ...]:
+    """`path: +line` or `path: -line` for every changed line that touches the table.
+
+    A schema.prisma line counts when its number falls inside that copy's `blocks` entry (head for added lines,
+    merge base for removed ones), a migration line when it is added and names the table."""
     sections: Final = re.split(r"^diff --git [^\n]*\n", diff, flags=re.MULTILINE)
-    return tuple(f"{path}: {line.strip()}" for path, line in _watched_added_lines(sections) if TABLE.search(line))
-
-
-def _watched_added_lines(sections: Sequence[str]) -> tuple[tuple[str, str], ...]:
-    pairs: Final = tuple((_path_of(section), section) for section in sections)
+    paths: Final = tuple((_path_of(section), section) for section in sections)
     return tuple(
-        (path, line) for path, section in pairs if watched(path) for line in _added_lines(section)
-    )  # comprehension-ok: path then its lines
+        hit for path, section in paths if watched(path) for hit in _section_hits(path, _hunks(section), blocks)
+    )  # comprehension-ok: path then its hits
+
+
+def _section_hits(path: str, hunks: Sequence[Hunk], blocks: Mapping[str, Block]) -> tuple[str, ...]:
+    if path not in blocks:
+        return tuple(f"{path}: +{line.strip()}" for hunk in hunks for _, line in hunk.added if TABLE.search(line))
+    block: Final = blocks[path]
+    removed: Final = tuple(
+        f"{path}: -{line.strip()}" for hunk in hunks for number, line in hunk.removed if number in block.base
+    )  # comprehension-ok: hunk then its lines
+    added: Final = tuple(
+        f"{path}: +{line.strip()}" for hunk in hunks for number, line in hunk.added if number in block.head
+    )  # comprehension-ok: hunk then its lines
+    return removed + added
 
 
 def _path_of(section: str) -> str:
@@ -70,11 +124,33 @@ def _path_of(section: str) -> str:
 
 
 def approved_by(reviews: Sequence[Review], login: str) -> bool:
-    """True when `login`'s most recent review on the pull request is an approval."""
-    states: Final = tuple(review.state for review in reviews if review.user.login == login)
+    """True when `login`'s most recent review that changes GitHub's approval state is an approval.
+
+    COMMENTED and PENDING reviews leave an earlier approval in force on GitHub, so they are skipped here too.
+    """
+    own: Final = tuple(review.state for review in reviews if review.user.login == login)
+    states: Final = tuple(state for state in own if state in DECIDING_STATES)
     return bool(states) and states[-1] == "APPROVED"
 
 
+def latest_approval(reviews: Sequence[Review], login: str) -> Review | None:
+    """`login`'s most recent APPROVED review on the pull request, the one GitHub keeps in force after pushes."""
+    own: Final = tuple(review for review in reviews if review.user.login == login and review.state == "APPROVED")
+    return own[-1] if own else None
+
+
+def approval_covers(
+    approval: Review, head: str, hits: Sequence[str], hits_at: Callable[[str], Sequence[str] | None]
+) -> bool:
+    """True when the approval is on the head, or the head adds no spend-log line the approved commit did not."""
+    if approval.commit_id is None:
+        return False
+    if approval.commit_id == head:
+        return True
+    reviewed: Final = hits_at(approval.commit_id)
+    return reviewed is not None and set(hits) <= set(reviewed)
+
+
 def added_diff(base: str, head: str) -> str:
     result: Final = subprocess.run(
         ("git", "diff", "--unified=0", "--no-color", f"{base}..{head}", "--", *WATCHED, MIGRATIONS_PREFIX),
@@ -85,6 +161,27 @@ def added_diff(base: str, head: str) -> str:
     return result.stdout
 
 
+def file_at(revision: str, path: str) -> str:
+    """Contents of `path` at `revision`, empty when the file does not exist there."""
+    result: Final = subprocess.run(("git", "show", f"{revision}:{path}"), check=False, capture_output=True, text=True)
+    return result.stdout if result.returncode == 0 else ""
+
+
+def schema_blocks(base: str, head: str) -> Mapping[str, Block]:
+    return MappingProxyType(
+        {path: Block(base=model_block(file_at(base, path)), head=model_block(file_at(head, path))) for path in WATCHED}
+    )
+
+
+def fetched(commit: str) -> bool:
+    result: Final = subprocess.run(("git", "fetch", "--no-tags", "--depth=1", "origin", commit), check=False)
+    return result.returncode == 0
+
+
+def reviewed_hits(base: str, commit: str) -> tuple[str, ...] | None:
+    return spend_log_hits(added_diff(base, commit), schema_blocks(base, commit)) if fetched(commit) else None
+
+
 class Arguments(BaseModel):
     model_config = ConfigDict(frozen=True)
     base: str
@@ -100,18 +197,28 @@ def main(argv: Sequence[str]) -> int:
     parser.add_argument("--approver", required=True, help="GitHub login whose approval unlocks the merge")
     parser.add_argument("--reviews", required=True, help="JSON file with the pull request's reviews")
     args: Final = Arguments.model_validate(vars(parser.parse_args(argv)))
-    hits: Final = spend_log_hits(added_diff(args.base, args.head))
+    hits: Final = spend_log_hits(added_diff(args.base, args.head), schema_blocks(args.base, args.head))
     if not hits:
-        sys.stdout.write("No added line in schema.prisma or a migration names LiteLLM_SpendLogs; nothing to review.\n")
+        sys.stdout.write(
+            "No change inside model LiteLLM_SpendLogs in schema.prisma and no added migration line names it; "
+            "nothing to review.\n"
+        )
         return 0
-    if approved_by(_REVIEWS.validate_json(args.reviews.read_bytes()), args.approver):
+    reviews: Final = _REVIEWS.validate_json(args.reviews.read_bytes())
+    approval: Final = latest_approval(reviews, args.approver) if approved_by(reviews, args.approver) else None
+    if approval is not None and approval_covers(
+        approval, args.head, hits, lambda commit: reviewed_hits(args.base, commit)
+    ):
         sys.stdout.write(f"LiteLLM_SpendLogs change approved by {args.approver}: {', '.join(hits)}\n")
         return 0
     listed: Final = "".join(f"  {hit}\n" for hit in hits)
-    sys.stdout.write(
+    reason: Final = (
         f"This pull request changes LiteLLM_SpendLogs and needs an approving review from @{args.approver}:\n"
-        f"{listed}The check re-runs when a review is submitted.\n"
+        if approval is None
+        else f"LiteLLM_SpendLogs lines changed after @{args.approver} approved {approval.commit_id}, "
+        f"a new approving review is needed:\n"
     )
+    sys.stdout.write(f"{reason}{listed}After approving, re-run this job.\n")
     return 1
 
 

codecov.yaml

@@ -15,9 +15,10 @@ ignore:
 flag_management:
   default_rules:
     carryforward: true
-  # Dead flags no CI job uploads anymore: their carried-forward sessions were
-  # measured against old revisions, and the stale line maps mark comment lines
-  # of since-edited files as missed, sinking patch coverage on unrelated PRs.
+  # Dead flags no CI job uploads anymore, and flags whose job is path-gated on
+  # PRs: their carried-forward sessions were measured against old revisions, and
+  # the stale line maps mark comment lines of since-edited files as missed,
+  # sinking patch coverage on unrelated PRs.
   individual_flags:
     - name: proxy-mgmt-behavior
       carryforward: false
@@ -99,6 +100,10 @@ flag_management:
       carryforward: false
     - name: proxy-db-endpoints-and-responses
       carryforward: false
+    - name: roi-postgres
+      carryforward: false
+    - name: redis-compat
+      carryforward: false
 
 component_management:
   individual_components:

docs/adrs/AGENTS.md

@@ -0,0 +1,5 @@
+# Architecture decision records for the Python SDK and proxy
+
+This folder records decisions about the Python SDK and proxy the way [`litellm-rust/docs/adrs/`](../../litellm-rust/docs/adrs/AGENTS.md) records them for the Rust core, and that file's process applies here unchanged: one short `adr_NNN_short_title.md` per decision that is hard to reverse or that a new contributor would reasonably question, with Context, Decision, Alternatives Considered, and Consequences, superseded by a new ADR instead of edited once accepted. Each folder numbers its own ADRs, so a reference to an ADR in the other folder names its path
+
+Read the ADRs here before you build a new way to move data between the endpoints, the provider transformations, and cost tracking or logging. A PR that makes a decision of that kind adds its ADR in the same PR

docs/adrs/adr_001_provider_usage_fields_ride_as_usage_extras.md

@@ -0,0 +1,39 @@
+# ADR 001: Provider usage fields ride as extras on the normalized usage object
+
+Status: Accepted
+
+## Context
+
+Providers report usage fields that no other provider has, such as xAI's `usage.server_side_tool_usage_details.web_search_calls` or Anthropic's `usage.inference_geo`, and cost tracking has to read them or that spend never reaches `LiteLLM_SpendLogs`. One provider response can be served on `/v1/chat/completions`, `/v1/responses`, and `/v1/messages`, streaming or not, so a field that only one of those paths carries is billed wrong on the others
+
+[#30817](https://github.com/BerriAI/litellm/pull/30817) fixed unbilled xAI web searches. One draft did the copy in the xAI Responses transform to keep provider code out of shared files, and that swapped `/v1/responses` usage to the chat shape, so callers of that endpoint lost the Responses schema. The merged version keeps the schema, copies unknown usage fields in the shared Responses-to-chat bridge for every provider, and gates on the reported field without naming the provider
+
+In October 2026, `Usage` and `ResponseAPIUsage` kept unknown usage keys as pydantic extras, and that bridge copied them. Paths that build usage from named fields kept only the fields they name: the Anthropic, Gemini, and Bedrock Converse transforms, the stream builder, and the bridges from chat usage to `/v1/responses` and `/v1/messages`
+
+## Decision
+
+Provider-only usage fields stay on the usage object as pydantic extras, and cost tracking reads them there, in that provider's cost calculator or in a shared gate that checks the field instead of the provider. An extra reaches callers as an added usage key. No transform renames, removes, or reshapes a usage field the endpoint's schema defines to feed cost tracking, though it may fill one LiteLLM's usage object already defines, such as `server_tool_use`
+
+A shared path that rebuilds usage, such as the stream builder or a bridge between endpoint shapes, copies the extras instead of naming provider fields. A provider transform that builds usage from named fields names the new field. Neither is bypassed with a side channel. Older code that differs, such as the Azure branches in the shared tracker and the Gemini pricing tier read from `_hidden_params`, predates this ADR
+
+## Alternatives Considered
+
+**Reshape usage in the provider transform**, as a draft of #30817 did. It changes the schema that callers of that endpoint rely on, and every provider and endpoint needs its own copy
+
+**A typed field on `ResponseAPIUsage` and `Usage` for each provider's quirk**. The types grow without bound and every bridge needs an update for each one. A field several providers report, like `server_tool_use`, does get one
+
+**A side channel such as `_hidden_params`**. Cost calculators receive `Usage`, so a side channel makes a second source of truth for the same number
+
+**Provider branches or provider field names in shared code**. They put one provider's logic in a file every provider shares, which is why #30817 replaced its xAI-only gate with one that checks the reported field
+
+## Consequences
+
+Benefits:
+
+- Billing a new provider usage field touches that provider's calculator (plus a `cost_per_token` arm when the calculator is new), its transform when that names fields, and for a built-in tool at most one gate and one `get_cost_for_web_search_request` arm
+
+Costs:
+
+- Extras are untyped, so a reader validates them before use, as `_usage_reports_server_side_web_search_calls` does
+- The stream builder and the bridges from chat usage did not copy extras when this was written, so until they do, a field billed on those paths is named in them, and tests and proof runs cover each endpoint and `"stream": true`
+- This covers usage fields LiteLLM prices, not request-side params or a provider's own charge for the request, which OpenRouter and opencode Go report and `response_cost_calculator` reads from `_hidden_params` first

enterprise/enterprise_hooks/google_text_moderation.py

@@ -15,7 +15,10 @@ import litellm
 from litellm._logging import verbose_proxy_logger
 from litellm.integrations.custom_logger import CustomLogger
 from litellm.proxy._types import UserAPIKeyAuth
-from litellm.proxy.guardrails._content_utils import iter_message_text
+from litellm.proxy.guardrails._content_utils import (
+    is_non_conversational_request,
+    iter_message_text,
+)
 from litellm.types.utils import CallTypesLiteral
 
 _THRESHOLD_VALUE: Final = TypeAdapter(object)
@@ -94,7 +97,7 @@ class ENTERPRISE_GoogleTextModeration(CustomLogger):
 
     async def async_moderation_hook(
         self,
-        data: dict,
+        data: dict[str, object],
         user_api_key_dict: UserAPIKeyAuth,
         call_type: CallTypesLiteral,
     ):
@@ -102,6 +105,9 @@ class ENTERPRISE_GoogleTextModeration(CustomLogger):
         - Calls Google's Text Moderation API
         - Rejects request if it fails safety check
         """
+        if is_non_conversational_request(call_type, data):
+            verbose_proxy_logger.debug("Google text moderation: skipping non-conversational call type %s", call_type)
+            return
         # Covers multimodal list content + Responses-API input.
         text = "".join(iter_message_text(data))
         if text:

enterprise/litellm_enterprise/enterprise_callbacks/callback_controls.py

@@ -33,9 +33,8 @@ class EnterpriseCallbackControls:
 
             try:
                 disabled_callbacks = EnterpriseCallbackControls.get_disabled_callbacks(litellm_params, standard_callback_dynamic_params)
-                verbose_logger.debug(f"Dynamically disabled callbacks from {X_LITELLM_DISABLE_CALLBACKS}: {disabled_callbacks}")
-                verbose_logger.debug(f"Checking if {callback} is disabled via headers. Disable callbacks from headers: {disabled_callbacks}")
                 if disabled_callbacks is not None:
+                    verbose_logger.debug(f"Checking if {callback} is in dynamically disabled callbacks: {disabled_callbacks}")
                     #########################################################
                     # premium user check
                     #########################################################

enterprise/litellm_enterprise/enterprise_callbacks/secret_detection.py

@@ -32,6 +32,7 @@ from litellm.integrations.custom_guardrail import (
     CustomGuardrail,
     log_guardrail_information,
 )
+from litellm.llms.base_llm.guardrail_translation.utils import effective_skip_tool_message_for_guardrail
 from litellm.proxy._types import UserAPIKeyAuth
 from litellm.proxy.guardrails._content_utils import iter_tool_json_fields, map_tool_json_text, walk_user_text
 from litellm.types.guardrails import GuardrailEventHooks
@@ -753,7 +754,12 @@ class _ENTERPRISE_SecretDetection(CustomGuardrail):
         _masked_entity_count.set({})
 
         # Covers multimodal list content + Responses-API input.
-        inspected = walk_user_text(data, self.redact_text, self.redact_tool_payload) + self._redact_prompt(data)
+        inspected = walk_user_text(
+            data,
+            self.redact_text,
+            self.redact_tool_payload,
+            skip_tool_message=effective_skip_tool_message_for_guardrail(self),
+        ) + self._redact_prompt(data)
 
         if inspected == 0:
             # Image-only, empty-text, and unsupported payloads inspected

enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py

@@ -398,7 +398,7 @@ class CheckBatchCost:
         deployment. Once that deployment leaves the router, default fallbacks can
         silently send the retrieve to a provider that never saw the batch, so its
         404 must not retire a potentially recoverable row."""
-        return self.llm_router.get_deployment(model_id=model_id) is not None
+        return self.llm_router.registry().deployment(model_id) is not None
 
     @staticmethod
     def _is_output_file_gone_at_provider(error: Exception, output_file_id: Optional[str]) -> bool:
@@ -564,7 +564,7 @@ class CheckBatchCost:
     def _get_deployment_id_for_bare_model(
         self, bare_model_name: str, llm_provider: str
     ) -> Optional[str]:
-        model_group = self.llm_router.resolve_model_name_from_model_id(bare_model_name)
+        model_group = self.llm_router.registry().resolve(bare_model_name, kind="from_id")
         deployment_id = (
             self._get_deployment_id_for_provider(model_group, llm_provider) if model_group else None
         )
@@ -580,7 +580,7 @@ class CheckBatchCost:
     ) -> Optional[str]:
         from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
 
-        for deployment in self.llm_router.get_model_list(model_name=None) or []:
+        for deployment in self.llm_router.registry().deployments() or []:
             litellm_params = deployment.get("litellm_params") or {}
             actual_model = litellm_params.get("model")
             if not isinstance(actual_model, str):
@@ -623,8 +623,9 @@ class CheckBatchCost:
         """
         from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
 
-        for deployment_id in self.llm_router.get_model_ids(model_name=model_group):
-            deployment_info = self.llm_router.get_deployment(model_id=deployment_id)
+        registry = self.llm_router.registry()
+        for deployment_id in registry.names(model_group, kind="ids"):
+            deployment_info = registry.deployment(deployment_id)
             if deployment_info is None:
                 continue
             try:
@@ -743,7 +744,7 @@ class CheckBatchCost:
             except (IndexError, AttributeError):
                 pass
 
-        credentials: Final = self.llm_router.get_deployment_credentials_with_provider(model_id) or {}
+        credentials: Final = self.llm_router.registry().credentials(model_id, form="with_provider") or {}
         _file_content = await afile_content(
             file_id=cast(str, raw_output_file_id),
             _litellm_internal_model_credentials=MappingProxyType(dict(credentials)),
@@ -766,7 +767,7 @@ class CheckBatchCost:
             except Exception:
                 pass
 
-        deployment_info = self.llm_router.get_deployment(model_id=model_id)
+        deployment_info = self.llm_router.registry().deployment(model_id)
         if deployment_info is None:
             verbose_proxy_logger.info(
                 f"Skipping job {job.unified_object_id} because it is not a valid deployment info"
@@ -996,7 +997,7 @@ class CheckBatchCost:
         )
 
         try:
-            deployment_info: Final = self.llm_router.get_deployment(model_id=model_id)
+            deployment_info: Final = self.llm_router.registry().deployment(model_id)
             if deployment_info is None:
                 verbose_proxy_logger.error(
                     f"CheckBatchCost: deployment {model_id} is no longer in the router, so the line items of batch "
@@ -1009,7 +1010,7 @@ class CheckBatchCost:
                 custom_llm_provider=deployment_info.litellm_params.custom_llm_provider,
             )
             cost_provider: Final = cast(BatchFileProvider, llm_provider)
-            credentials: Final = self.llm_router.get_deployment_credentials_with_provider(model_id) or {}
+            credentials: Final = self.llm_router.registry().credentials(model_id, form="with_provider") or {}
             file_access: Final = {
                 **credentials,
                 "_litellm_internal_model_credentials": MappingProxyType(dict(credentials)),

enterprise/litellm_enterprise/proxy/common_utils/check_responses_cost.py

@@ -30,7 +30,7 @@ from collections.abc import Awaitable, Callable, Mapping
 from dataclasses import dataclass
 from datetime import datetime, timedelta, timezone
 from types import MappingProxyType
-from typing import TYPE_CHECKING, Final, cast
+from typing import TYPE_CHECKING, Final
 
 from litellm_enterprise.proxy.common_utils.managed_object_attribution import (
     ManagedObjectCreatorAttribution,
@@ -273,7 +273,7 @@ class CheckResponsesCost:
 
     def _resolve_deployment(self, response_id: str) -> Deployment | None:
         model_id: str | None = ResponsesAPIRequestUtils.get_model_id_from_response_id(response_id)
-        return self.llm_router.get_deployment(model_id=model_id) if model_id is not None else None
+        return self.llm_router.registry().deployment(model_id) if model_id is not None else None
 
     async def _get_response(
         self,
@@ -290,12 +290,14 @@ class CheckResponsesCost:
         sees provider env vars, so it fails for every deployment whose credentials
         live in the config; the row then never leaves ``queued``.
         """
-        if not via_router:
-            return await litellm.aget_responses(response_id=response_id, litellm_metadata=litellm_metadata)
-        router_response = await self.llm_router.aget_responses(
-            response_id=response_id, litellm_metadata=litellm_metadata
+        retrieved: Final[object] = (
+            await self.llm_router.aget_responses(response_id=response_id, litellm_metadata=litellm_metadata)
+            if via_router
+            else await litellm.aget_responses(response_id=response_id, litellm_metadata=litellm_metadata)
         )
-        return cast(ResponsesAPIResponse, router_response)
+        if not isinstance(retrieved, ResponsesAPIResponse):
+            raise TypeError(f"Retrieving {response_id} did not return a response object")
+        return retrieved
 
     async def _fetch_unbilled(self, target: _PollTarget) -> ResponsesAPIResponse:
         return await self._get_response(
@@ -413,7 +415,7 @@ class CheckResponsesCost:
         if target.deployment is None or not isinstance(requested, str):
             return MappingProxyType({})
         routed: Final = _ROUTED_DEPLOYMENTS.validate_python(
-            self.llm_router.get_model_list(model_name=requested, team_id=team_id) or ()
+            self.llm_router.registry().deployments(requested, team_id=team_id) or ()
         )
         serving_id: Final = target.deployment.model_info.id
         return MappingProxyType(

enterprise/litellm_enterprise/proxy/hooks/managed_files.py

@@ -28,7 +28,7 @@ import httpx
 from fastapi import HTTPException
 from openai import APIConnectionError, AsyncOpenAI
 from openai.types.file_deleted import FileDeleted
-from pydantic import BaseModel, JsonValue, TypeAdapter, ValidationError
+from pydantic import BaseModel, ConfigDict, JsonValue, TypeAdapter, ValidationError
 from typing_extensions import ReadOnly, Unpack
 
 import litellm
@@ -53,7 +53,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
     extract_file_metadata,
 )
 from litellm.llms.base_llm.files.storage_backend_factory import get_storage_backend
-from litellm.llms.base_llm.files.transformation import BaseFileEndpoints
+from litellm.llms.base_llm.files.transformation import BaseFileEndpoints, FileListModelScope
 from litellm.llms.base_llm.managed_resources.isolation import (
     build_list_page,
     build_owner_filter,
@@ -245,7 +245,7 @@ def _provider_file_retrieve_credentials(
     if llm_router is None or model_id is None:
         return None
     try:
-        credentials: Final = llm_router.get_deployment_credentials_with_provider(model_id)
+        credentials: Final = llm_router.registry().credentials(model_id, form="with_provider")
     except Exception as error:
         verbose_logger.warning(
             "Failed to retrieve credentials for provider file "
@@ -309,6 +309,7 @@ def _is_xai_batch_output(
 class _ManagedFileRow(Protocol):
     unified_file_id: str
     file_object: OpenAIFileObject
+    model_mappings: Mapping[str, str]
     flat_model_file_ids: Sequence[str]
     storage_backend: Optional[str]
     storage_url: Optional[str]
@@ -362,6 +363,8 @@ class _ManagedResourceDatabase(Protocol):
     @property
     def litellm_managedfiletable(self) -> _ManagedFileTableActions: ...
 
+    async def query_raw(self, query: str, *args: object) -> object: ...
+
     @property
     def litellm_managedobjecttable(self) -> _ManagedObjectTableActions: ...
 
@@ -385,10 +388,83 @@ class _RouterFileCallKwargs(TypedDict, total=False):
     custom_llm_provider: ReadOnly[str | None]
 
 
+def _managed_database(prisma_client: _ManagedResourcePrismaClient) -> _ManagedResourceDatabase:
+    return prisma_client.db
+
+
 def _managed_file_table(prisma_client: _ManagedResourcePrismaClient) -> _ManagedFileTableActions:
     return prisma_client.db.litellm_managedfiletable
 
 
+class _ListedFileRow(Protocol):
+    @property
+    def unified_file_id(self) -> str: ...
+
+    @property
+    def file_object(self) -> object: ...
+
+    @property
+    def model_mappings(self) -> Mapping[str, object]: ...
+
+    @property
+    def storage_backend(self) -> str | None: ...
+
+
+class _ScopedFileRow(BaseModel):
+    model_config = ConfigDict(frozen=True)
+
+    unified_file_id: str
+    file_object: JsonValue = None
+    model_mappings: Mapping[str, JsonValue]
+    storage_backend: str | None = None
+
+
+_SCOPED_FILE_ROWS: Final = TypeAdapter(tuple[_ScopedFileRow, ...])
+
+
+def _owner_condition(owner_filter: Mapping[str, object]) -> tuple[str, tuple[str, ...]]:
+    match owner_filter:
+        case {"OR": [{"created_by": str() as user_id}, {"team_id": str() as team_id}]}:
+            return '("created_by" = $1 OR "team_id" = $2)', (user_id, team_id)
+        case {"created_by": str() as user_id}:
+            return '"created_by" = $1', (user_id,)
+        case {"team_id": str() as team_id}:
+            return '"team_id" = $1', (team_id,)
+        case _:
+            return ("FALSE", ()) if owner_filter else ("TRUE", ())
+
+
+async def _scoped_file_rows(
+    database: _ManagedResourceDatabase,
+    owner_filter: Mapping[str, object],
+    model_mapping_keys: frozenset[str],
+    cursor_id: str | None,
+    direction: Literal["asc", "desc"],
+    take: int,
+) -> tuple[_ScopedFileRow, ...]:
+    owner_condition, owner_params = _owner_condition(owner_filter)
+    keys_param: Final = len(owner_params) + 1
+    comparison: Final = "<" if direction == "desc" else ">"
+    cursor_condition: Final = (
+        f'AND ("created_at", "unified_file_id") {comparison} (SELECT "created_at", "unified_file_id" '
+        f'FROM "LiteLLM_ManagedFileTable" WHERE "unified_file_id" = ${keys_param + 1}) '
+        if cursor_id
+        else ""
+    )
+    order: Final = direction.upper()
+    rows: Final = await database.query_raw(
+        'SELECT "unified_file_id", "file_object", "model_mappings", "storage_backend" FROM "LiteLLM_ManagedFileTable" '
+        f'WHERE {owner_condition} AND ("model_mappings" ?| ${keys_param}::text[] OR "storage_backend" IS NOT NULL) '
+        f'{cursor_condition}ORDER BY "created_at" {order}, "unified_file_id" {order} '
+        f"LIMIT ${keys_param + (2 if cursor_id else 1)}",
+        *owner_params,
+        sorted(model_mapping_keys),
+        *((cursor_id,) if cursor_id else ()),
+        take,
+    )
+    return _SCOPED_FILE_ROWS.validate_python(rows)
+
+
 def _iter_provider_file_id_pairs(
     rows: Sequence[_ManagedFileRow],
     requested_provider_file_ids: frozenset[str],
@@ -2265,14 +2341,23 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
         user_api_key_dict: UserAPIKeyAuth,
         limit: Optional[int] = None,
         after: Optional[str] = None,
+        model_scope: Optional[FileListModelScope] = None,
+        before: Optional[str] = None,
         **data: Dict,
     ) -> FileListPage:
         """List the managed files the caller owns, newest first.
 
         Pagination is keyset based on ``unified_file_id`` so a key that owns
         every file on the proxy still reads one bounded page at a time.
-        ``purpose`` is applied after parsing, because the managed file table
-        keeps it inside the ``file_object`` blob instead of a column, and rows
+        ``after`` pages toward older files and ``before`` toward newer ones,
+        each page still ordered newest first.
+        ``model_scope`` is applied in the query, so only rows whose
+        ``model_mappings`` name one of its deployment ids or model names, and
+        storage-backed rows, leave the database; a storage-backed row then
+        stays only when its model name is one the router sends to the listed
+        provider. ``purpose`` is applied after parsing, because the managed
+        file table keeps it inside the ``file_object`` blob instead of a
+        column. Rows
         whose blob will not parse drop out there too, so a chunk of rows can
         yield fewer matches than the page holds. Successive chunks are read
         until the page is full or the caller's rows run out, which keeps
@@ -2281,9 +2366,8 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
         scan has to continue past it, the chunk widens to
         ``FILE_LIST_CONTINUATION_CHUNK_SIZE``, so the walk costs one query per
         that many rows instead of one per page. That bound is per query, not
-        per request: the work is still linear in the rows the caller owns, and
-        a filter matching nothing reads every one of them, with no index
-        covering either the owner filter or the sort.
+        per request: a ``purpose`` filter matching nothing still reads every
+        row that passes the query.
         """
         validate_file_list_limit(limit)
         validate_file_list_purpose(purpose)
@@ -2292,36 +2376,60 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
         if owner_filter is None:
             return FileListPage.model_validate(build_list_page([]))
 
-        if after:
-            cursor_row = await _managed_file_table(self.prisma_client).find_first(
-                where={**owner_filter, "unified_file_id": after}
+        if after and before:
+            raise ProxyException(
+                message="Pass either 'after' (or 'after_id') or 'before_id' when listing files, not both.",
+                type="invalid_request_error",
+                param="before_id",
+                code=400,
+                openai_code="invalid_value",
+            )
+        cursor_param: Final = "before_id" if before else "after"
+        cursor: Final = before or after
+        database: Final = _managed_database(self.prisma_client)
+        if cursor:
+            cursor_row = await database.litellm_managedfiletable.find_first(
+                where={**owner_filter, "unified_file_id": cursor}
             )
             if cursor_row is None:
                 raise ProxyException(
-                    message=f"Invalid 'after' cursor: no file found with id '{after}'.",
+                    message=f"Invalid '{cursor_param}' cursor: no file found with id '{cursor}'.",
                     type="invalid_request_error",
-                    param="after",
+                    param=cursor_param,
                     code=400,
                     openai_code="invalid_value",
                 )
 
+        if model_scope is not None and not model_scope.model_mapping_keys:
+            return FileListPage.model_validate(build_list_page([]))
+
         page_size: Final = min(limit or MAX_FILE_LIST_LIMIT, MAX_FILE_LIST_LIMIT)
         matches: Final[List[OpenAIFileObject]] = []
-        cursor_id = after
+        cursor_id = cursor
         chunk_size = page_size + 1
+        direction: Final[Literal["asc", "desc"]] = "asc" if before else "desc"
 
         while len(matches) <= page_size:
             cursor_args: _CursorPageArgs = {"cursor": {"unified_file_id": cursor_id}, "skip": 1} if cursor_id else {}
-            chunk = await _managed_file_table(self.prisma_client).find_many(
-                where=owner_filter,
-                take=chunk_size,
-                order=[{"created_at": "desc"}, {"unified_file_id": "desc"}],
-                **cursor_args,
+            chunk: Sequence[_ListedFileRow] = (
+                await _scoped_file_rows(
+                    database, owner_filter, model_scope.model_mapping_keys, cursor_id, direction, chunk_size
+                )
+                if model_scope is not None
+                else await database.litellm_managedfiletable.find_many(
+                    where=owner_filter,
+                    take=chunk_size,
+                    order=[{"created_at": direction}, {"unified_file_id": direction}],
+                    **cursor_args,
+                )
             )
             matches.extend(
                 _public_file_object(parsed_file_object, row.unified_file_id)
                 for row in chunk
-                if (parsed_file_object := _parse_managed_file_object(row.file_object, row.unified_file_id)) is not None
+                if (
+                    model_scope is None or model_scope.holds(row.model_mappings, stored=row.storage_backend is not None)
+                )
+                and (parsed_file_object := _parse_managed_file_object(row.file_object, row.unified_file_id)) is not None
                 and (purpose is None or parsed_file_object.purpose == purpose)
             )
             if len(chunk) < chunk_size:
@@ -2329,7 +2437,10 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
             cursor_id = chunk[-1].unified_file_id
             chunk_size = max(chunk_size, FILE_LIST_CONTINUATION_CHUNK_SIZE)
 
-        return FileListPage.model_validate(build_list_page(matches[:page_size], has_more=len(matches) > page_size))
+        page: Final = matches[:page_size]
+        return FileListPage.model_validate(
+            build_list_page(page[::-1] if before else page, has_more=len(matches) > page_size)
+        )
 
     def _is_batch_polling_enabled(self) -> bool:
         """
@@ -2539,7 +2650,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
             k: v for k, v in data.items() if k not in ("model", "file_id", "_litellm_internal_model_credentials")
         }
         for model_id, model_file_id in specific_model_file_id_mapping.items():
-            credentials = llm_router.get_deployment_credentials_with_provider(model_id=model_id)
+            credentials = llm_router.registry().credentials(model_id, form="with_provider")
             delete_data = {
                 **filtered_data,
                 **(
@@ -2583,7 +2694,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
                     # against the deployment's configured bucket, which they only
                     # trust from this immutable server-side snapshot, never from
                     # request params.
-                    credentials = llm_router.get_deployment_credentials_with_provider(model_id=model_id)
+                    credentials = llm_router.registry().credentials(model_id, form="with_provider")
                     if credentials is not None:
                         data["_litellm_internal_model_credentials"] = cast(Dict, MappingProxyType(dict(credentials)))
                     else:

enterprise/litellm_enterprise/proxy/management_endpoints/project_endpoints.py

@@ -290,7 +290,7 @@ def _check_team_project_limits(
 
 
 def _router_access_group_names(llm_router: "Router | None") -> frozenset[str]:
-    return frozenset(llm_router.get_model_access_groups()) if llm_router is not None else frozenset()
+    return frozenset(llm_router.registry().access_groups()) if llm_router is not None else frozenset()
 
 
 def _project_models_expanding_at_request_time(

examples/bigmodel.md

@@ -0,0 +1,63 @@
+# BigModel (Zhipu AI)
+
+Use `bigmodel/` for the domestic BigModel platform and `zai/` for international Z.AI. Each platform uses its own credentials and default URL
+
+```yaml
+model_list:
+  - model_name: glm
+    litellm_params:
+      model: bigmodel/glm-5.3
+      api_key: os.environ/BIGMODEL_API_KEY
+      api_base: https://open.bigmodel.cn
+```
+
+One model configuration supports all three native APIs. The provider selects the upstream endpoint from the requested API
+
+| Proxy endpoint | BigModel endpoint |
+| --- | --- |
+| `/v1/chat/completions` | `/api/paas/v4/chat/completions` |
+| `/v1/responses` | `/api/v1/responses` |
+| `/v1/messages` | `/api/anthropic/v1/messages` |
+
+`api_base` is optional and defaults to `https://open.bigmodel.cn`. Set `BIGMODEL_API_BASE` to override the default, or pass an explicit `api_base` to override the environment value. Existing API-specific base URLs and complete endpoint URLs are normalized, preserving a custom proxy's path prefix
+
+A GLM Coding Plan key uses `https://open.bigmodel.cn/api/coding/paas/v4` as its `api_base`, since Coding Plan and standard keys do not work on each other's endpoints. Chat Completions then calls the Coding Plan endpoint, and Responses and Messages keep the endpoints in the table above
+
+The Admin UI lists BigModel (Zhipu AI), displays its official logo, and provides a required password API Key field and an optional API Base field with the default URL. Authentication uses an explicit `api_key` or `BIGMODEL_API_KEY`
+
+```python
+import litellm
+
+chat = litellm.completion(
+    model="bigmodel/glm-5.3",
+    messages=[{"role": "user", "content": "Hello"}],
+    thinking={"type": "enabled"},
+    reasoning_effort="low",
+)
+
+response = litellm.responses(
+    model="bigmodel/glm-5.3",
+    input="Hello",
+    reasoning={"effort": "low"},
+    store=True,
+)
+
+async def messages():
+    return await litellm.anthropic.messages.acreate(
+        model="bigmodel/glm-5.3",
+        max_tokens=1024,
+        messages=[{"role": "user", "content": "Hello"}],
+        thinking={"type": "adaptive"},
+        output_config={"effort": "low"},
+    )
+```
+
+Each API retains its native response format and streaming events. The native Messages SDK entry point is asynchronous, as is the proxy's `/v1/messages` route
+
+GLM-5.3, GLM-5.3-Flash, and GLM-5.3-FlashX use always-on thinking. GLM-5.3 supports `low`, `high`, and `max` reasoning effort. Messages preserves native `thinking`, `output_config`, and `reasoning_effort` parameters
+
+Set `store=True` when creating a Responses request if you need continuation, retrieval, or deletion. Use the response ID returned by LiteLLM. BigModel documents a seven-day retention period and does not support cancel, compact, native WebSocket, or file_search
+
+The model catalog and packaged backup include GLM-5.3, GLM-5.3-Flash, and GLM-5.3-FlashX. Prices are expressed in USD/token using a static 7.0 CNY/USD conversion, with the source and verification date in each entry's metadata. Use existing `litellm_params` cost fields for contract-specific deployment prices
+
+API references: [Chat Completions](https://docs.bigmodel.cn/cn/guide/develop/openai/introduction), [Responses](https://docs.bigmodel.cn/cn/guide/develop/responses/introduction), [Messages](https://docs.bigmodel.cn/cn/guide/develop/claude/introduction), and [pricing](https://docs.bigmodel.cn/cn/guide/start/pricing), checked on 2026-10-27

gateway/Dockerfile

@@ -40,6 +40,7 @@ ENV UV_PYTHON_DOWNLOADS=0
 COPY pyproject.toml README.md rust-toolchain.toml ./
 COPY .cargo ./.cargo
 COPY litellm-rust ./litellm-rust
+COPY model_prices_and_context_window.json ./
 
 RUN --mount=type=cache,target=/root/.cache/uv \
     mkdir -p litellm/rust_bridge /wheels && \
@@ -144,6 +145,7 @@ COPY --from=builder --chown=nonroot:nonroot /app/.venv /app/.venv
 COPY --from=builder --chown=nonroot:nonroot /app/litellm /app/litellm
 COPY --from=builder --chown=nonroot:nonroot /app/enterprise /app/enterprise
 COPY --from=builder --chown=nonroot:nonroot /app/gateway /app/gateway
+COPY --from=builder --chown=nonroot:nonroot /app/litellm_cli /app/litellm_cli
 COPY --from=builder --chown=nonroot:nonroot /app/docker/component_entrypoint.sh /app/docker/component_entrypoint.sh
 COPY --from=builder --chown=nonroot:nonroot /app/schema.prisma /app/schema.prisma
 COPY --from=builder /opt/prisma /opt/prisma
@@ -153,7 +155,7 @@ RUN find /app/.venv -type f -path "*/tornado/test/*" -delete && \
     find /app/.venv -type d -path "*/tornado/test" -delete && \
     chmod -R a+rX /opt/prisma && \
     python -c "from prisma.client import BINARY_PATHS; paths = list(BINARY_PATHS.query_engine.values()); assert paths and all(p.startswith('/opt/prisma/') for p in paths), paths" && \
-    python -c "import litellm; from litellm.rust_bridge.loader import native_bridge_available; assert litellm.__file__ == '/app/litellm/__init__.py', litellm.__file__; assert native_bridge_available()"
+    python -c "import litellm, litellm.rust_bridge._native; assert litellm.__file__ == '/app/litellm/__init__.py', litellm.__file__"
 
 USER nonroot
 

gateway/launch.py

@@ -8,6 +8,11 @@ handed the loopback URL it listens on. A pre-existing ``DATABASE_URL`` wins in
 wins under token auth too, so exporting it here is enough for every worker to
 pick the pooled URL up unchanged.
 
+LiteLLM imports wait for ``main``: every uvicorn worker imports this module again
+before it can answer its first 5 s health ping, so a slow host would kill each one.
+Workers then load the gateway app through ``worker_app`` without garbage collection
+pauses, which hold the GIL long enough to miss a ping.
+
 Run with:
     python -m gateway.launch --workers 4 --host 0.0.0.0 --port 4000
 """
@@ -15,19 +20,19 @@ Run with:
 import os
 import sys
 from collections.abc import Callable, Mapping, Sequence
-from typing import Final
+from typing import TYPE_CHECKING, Final
 
 from uvicorn.main import main as uvicorn_main
 
-from litellm.proxy.db.db_url_settings import DatabaseURLSettings
-from litellm.proxy.db.pgbouncer import (
-    PgBouncerError,
-    PgBouncerSettings,
-    export_pooled_database_url,
-    start_in_container_pgbouncer,
-)
+from litellm_cli.main import without_gc_pauses
 
-GATEWAY_APP: Final = "gateway.main:app"
+if TYPE_CHECKING:
+    from fastapi import FastAPI
+
+    from litellm.proxy.db.db_url_settings import DatabaseURLSettings
+    from litellm.proxy.db.pgbouncer import PgBouncerError, PgBouncerSettings
+
+GATEWAY_APP: Final = "gateway.launch:worker_app"
 KEEPALIVE_FLAG: Final = "--timeout-keep-alive"
 
 
@@ -35,21 +40,23 @@ def uvicorn_argv(argv: Sequence[str], environ: Mapping[str, str]) -> tuple[str,
     """Honor ``KEEPALIVE_TIMEOUT`` like ``proxy_cli.py`` does, unless the flag was passed explicitly."""
     keepalive: Final = environ.get("KEEPALIVE_TIMEOUT")
     if keepalive is None or any(arg == KEEPALIVE_FLAG or arg.startswith(f"{KEEPALIVE_FLAG}=") for arg in argv):
-        return (GATEWAY_APP, *argv)
-    return (GATEWAY_APP, *argv, KEEPALIVE_FLAG, keepalive)
+        return (GATEWAY_APP, "--factory", *argv)
+    return (GATEWAY_APP, "--factory", *argv, KEEPALIVE_FLAG, keepalive)
 
 
 def pool_database_url(
-    settings: DatabaseURLSettings,
-    pgbouncer: PgBouncerSettings,
+    settings: "DatabaseURLSettings",
+    pgbouncer: "PgBouncerSettings",
     environ: Mapping[str, str],
-) -> str | PgBouncerError | None:
+) -> "str | PgBouncerError | None":
     """Start the in-container PgBouncer and return its loopback URL, or None when ``pgbouncer.enabled`` is off.
 
     The upstream URL is whatever ``apply_to_env`` assembled from the discrete
     ``DATABASE_*`` vars (or an operator-pinned ``DATABASE_URL``). Under token
     auth the pooler mints and renews the upstream token itself.
     """
+    from litellm.proxy.db.pgbouncer import PgBouncerError, start_in_container_pgbouncer
+
     if not pgbouncer.enabled:
         return None
     upstream_url: Final = environ.get("DATABASE_URL")
@@ -58,11 +65,20 @@ def pool_database_url(
     return start_in_container_pgbouncer(pgbouncer, upstream_url, token_auth=settings.token_auth())
 
 
+def worker_app() -> "FastAPI":
+    with without_gc_pauses():
+        from gateway.main import app
+    return app
+
+
 def _serve(argv: Sequence[str]) -> None:
     uvicorn_main(tuple(argv), prog_name="uvicorn")
 
 
 def main(argv: Sequence[str], serve: Callable[[Sequence[str]], None] = _serve) -> None:
+    from litellm.proxy.db.db_url_settings import DatabaseURLSettings
+    from litellm.proxy.db.pgbouncer import PgBouncerError, PgBouncerSettings, export_pooled_database_url
+
     settings: Final = DatabaseURLSettings.from_env()
     settings.apply_to_env()
     pooled_url: Final = pool_database_url(settings, PgBouncerSettings(), os.environ)

helm/litellm-helm/README.md

@@ -259,3 +259,9 @@ itself.
 `lensWorker.mode` selects `disabled`, `bundled` or `external`. Bundled mode generates separate service and identity-signing credentials. External mode requires `lensWorker.gateway.secretName` and `lensWorker.serviceTokenSecret.name`; their keys default to `gateway-secret` and `service-token`. Provision distinct values matching the external Lens deployment
 
 Follow the [Lens Helm connection guide](https://github.com/BerriAI/lens/blob/main/helm/lens/README.md#connect-a-gateway) for complete values, verification, GitOps credential requirements and existing-data migration. Source-chart installation requires a built Lens image until the first signed Lens release is published
+
+## Shutdown accounting
+
+`GRACEFUL_SHUTDOWN_TIMEOUT` defaults to 30 seconds. Pending success callbacks share the time remaining since SIGTERM or the first `/health/drain` call. Callback flushing reserves the existing 10-second teardown headroom for cancellation, tracked accounting tasks and persistence. Exhausting this budget can discard callbacks, which is logged
+
+Set `terminationGracePeriodSeconds` to at least `GRACEFUL_SHUTDOWN_TIMEOUT + 10`, allowing additional time for preStop hooks and database writes. This is not a hard shutdown bound or a guarantee of persistence. Database retries retain their own limits, and cancellation-resistant callbacks can remain alive. The existing preStop hook can also be followed by Uvicorn's separate connection drain, as described in `values.yaml`

litellm-proxy-extras/litellm_proxy_extras/migration_data_checks.py

@@ -14,11 +14,9 @@ either, so the migration keeps today's behavior for it.
 """
 
 import os
-import re
-from collections.abc import Callable, Iterator
+from collections.abc import Callable
 from dataclasses import dataclass
 from functools import reduce
-from itertools import accumulate
 from pathlib import Path
 from types import MappingProxyType
 from typing import TYPE_CHECKING, Final, Literal
@@ -43,7 +41,7 @@ MIGRATION_DATA_CHECK_TIMEOUT_ENV_VAR: Final = "LITELLM_MIGRATION_DATA_CHECK_TIME
 DEFAULT_MIGRATION_DATA_CHECK_TIMEOUT_SECONDS: Final = 600.0
 REFUSAL: Final = "Existing rows would make a pending schema change fail:"
 
-Kind = Literal["too_long", "duplicates", "nulls", "any_row", "orphans", "check_violations"]
+Kind = Literal["too_long"]
 
 
 @dataclass(frozen=True, slots=True)
@@ -54,9 +52,6 @@ class Check:
     statement: str
     source: str
     index: str | None = None
-    referenced_table: str | None = None
-    referenced_columns: tuple[str, ...] = ()
-    expression: str | None = None
 
 
 @dataclass(frozen=True, slots=True)
@@ -100,33 +95,6 @@ class ColumnType:
 
 ColumnTypeLookup = Callable[[str, str], ColumnType | None]
 
-_IDENT: Final = r'"?(\w+)"?'
-_QUALIFIED: Final = r'(?:"?\w+"?\s*\.\s*)?"?(\w+)"?'
-_CREATE_INDEX: Final = re.compile(
-    rf"^\s*CREATE\s+(UNIQUE\s+)?INDEX\s+(?:CONCURRENTLY\s+)?(?:IF\s+NOT\s+EXISTS\s+)?{_IDENT}\s+ON\s+(?:ONLY\s+)?"
-    rf"{_QUALIFIED}\s*(?:USING\s+(\w+)\s*)?\((.*?)\)",
-    re.IGNORECASE | re.DOTALL,
-)
-_ALTER_TABLE: Final = re.compile(
-    rf"^\s*ALTER\s+TABLE\s+(?:IF\s+EXISTS\s+)?(?:ONLY\s+)?{_QUALIFIED}\s+(.*)$", re.IGNORECASE | re.DOTALL
-)
-_SET_NOT_NULL: Final = re.compile(rf"^ALTER\s+(?:COLUMN\s+)?{_IDENT}\s+SET\s+NOT\s+NULL$", re.IGNORECASE)
-_ADD_COLUMN: Final = re.compile(
-    rf"^ADD\s+(?:COLUMN\s+)?(?:IF\s+NOT\s+EXISTS\s+)?(?!(?:CONSTRAINT|FOREIGN|UNIQUE|CHECK|PRIMARY|EXCLUDE)\b){_IDENT}\s+(.*)$",
-    re.IGNORECASE | re.DOTALL,
-)
-_CONSTRAINT: Final = rf"^ADD\s+(?:CONSTRAINT\s+{_IDENT}\s+)?"
-_ADD_FOREIGN_KEY: Final = re.compile(
-    rf"{_CONSTRAINT}FOREIGN\s+KEY\s*\(([^)]*)\)\s*REFERENCES\s+{_QUALIFIED}\s*\(([^)]*)\)(.*)$",
-    re.IGNORECASE | re.DOTALL,
-)
-_ADD_UNIQUE: Final = re.compile(rf"{_CONSTRAINT}UNIQUE\s*\(([^)]*)\)", re.IGNORECASE | re.DOTALL)
-_ADD_CHECK: Final = re.compile(rf"{_CONSTRAINT}CHECK\s*\((.*)\)\s*(NOT\s+VALID)?\s*$", re.IGNORECASE | re.DOTALL)
-_PLAIN_COLUMN: Final = re.compile(rf"^{_IDENT}(?:\s+(?:ASC|DESC|NULLS\s+(?:FIRST|LAST)))*$", re.IGNORECASE)
-_COMMENT: Final = re.compile(r"--[^\n]*")
-_NOT_NULL: Final = re.compile(r"\bNOT\s+NULL\b", re.IGNORECASE)
-_HAS_DEFAULT: Final = re.compile(r"\bDEFAULT\b|\bGENERATED\b", re.IGNORECASE)
-
 
 def migration_data_check_timeout_seconds() -> float:
     """Seconds one count query may run before the preflight gives up on verifying it."""
@@ -145,117 +113,6 @@ def migration_data_check_timeout_seconds() -> float:
         return DEFAULT_MIGRATION_DATA_CHECK_TIMEOUT_SECONDS
 
 
-def statements(migration_sql: str) -> tuple[str, ...]:
-    """The statements of one migration file, comments stripped, empty ones dropped."""
-    parts: Final = _COMMENT.sub("", migration_sql).split(";")
-    return tuple(part.strip() for part in parts if part.strip())
-
-
-def _split_top_level(text: str) -> tuple[str, ...]:
-    """Split on the commas outside parentheses."""
-    depth: Final = accumulate((1 if char == "(" else -1 if char == ")" else 0) for char in text)
-    cuts: Final = tuple(
-        position for position, (char, level) in enumerate(zip(text, depth, strict=True)) if char == "," and level == 0
-    )
-    starts: Final = (0, *(cut + 1 for cut in cuts))
-    stops: Final = (*cuts, len(text))
-    pieces: Final = tuple(text[start:stop].strip() for start, stop in zip(starts, stops, strict=True))
-    return tuple(piece for piece in pieces if piece)
-
-
-def _plain_columns(column_list: str) -> tuple[str, ...] | None:
-    """The column names of an index or constraint column list, None when one entry is an
-    expression the derivation cannot turn into a count."""
-    matches: Final = tuple(_PLAIN_COLUMN.match(entry) for entry in _split_top_level(column_list))
-    if any(match is None for match in matches):
-        return None
-    return tuple(match.group(1) for match in matches if match is not None)
-
-
-def _index_checks(match: "re.Match[str]", statement_text: str, source: str) -> Iterator[Check]:
-    unique, name, table, method, column_list = match.groups()
-    columns: Final = _plain_columns(column_list)
-    if columns is None:
-        return
-    label: Final = f'CREATE {"UNIQUE " if unique else ""}INDEX "{name}"'
-    if method is None or method.lower() == "btree":
-        yield Check("too_long", table, columns, label, source, index=name)
-    if unique:
-        yield Check("duplicates", table, columns, label, source, index=name)
-
-
-def _alter_table_checks(table: str, action: str, source: str) -> Iterator[Check]:
-    if (set_not_null := _SET_NOT_NULL.match(action)) is not None:
-        yield Check("nulls", table, (set_not_null.group(1),), f'ALTER TABLE "{table}" {action}', source)
-        return
-    if (add_column := _ADD_COLUMN.match(action)) is not None:
-        column, body = add_column.groups()
-        if _NOT_NULL.search(body) is not None and _HAS_DEFAULT.search(body) is None:
-            yield Check(
-                "any_row", table, (column,), f'ALTER TABLE "{table}" ADD COLUMN "{column}" ... NOT NULL', source
-            )
-        return
-    if (foreign_key := _ADD_FOREIGN_KEY.match(action)) is not None:
-        constraint, columns, referenced_table, referenced_columns, tail = foreign_key.groups()
-        own: Final = _plain_columns(columns)
-        theirs: Final = _plain_columns(referenced_columns)
-        if own is None or theirs is None or len(own) != len(theirs) or "NOT VALID" in tail.upper():
-            return
-        yield Check(
-            "orphans",
-            table,
-            own,
-            _constraint_label(table, constraint, "FOREIGN KEY"),
-            source,
-            referenced_table=referenced_table,
-            referenced_columns=theirs,
-        )
-        return
-    if (unique := _ADD_UNIQUE.match(action)) is not None:
-        constraint, columns = unique.groups()
-        unique_columns: Final = _plain_columns(columns)
-        if unique_columns is not None:
-            yield Check("duplicates", table, unique_columns, _constraint_label(table, constraint, "UNIQUE"), source)
-        return
-    if (check := _ADD_CHECK.match(action)) is not None:
-        constraint, expression, not_valid = check.groups()
-        if not_valid is None:
-            yield Check(
-                "check_violations",
-                table,
-                (),
-                _constraint_label(table, constraint, "CHECK"),
-                source,
-                expression=expression,
-            )
-
-
-def _constraint_label(table: str, constraint: str | None, kind: str) -> str:
-    named: Final = f'CONSTRAINT "{constraint}" ' if constraint else ""
-    return f'ALTER TABLE "{table}" ADD {named}{kind}'
-
-
-def checks_for_statement(statement_text: str, source: str) -> tuple[Check, ...]:
-    """The count queries one SQL statement needs before it can be applied, empty for a
-    statement that cannot fail on existing rows or that the derivation cannot read."""
-    if (index := _CREATE_INDEX.match(statement_text)) is not None:
-        return tuple(_index_checks(index, statement_text, source))
-    if (alter := _ALTER_TABLE.match(statement_text)) is not None:
-        table, actions = alter.groups()
-        return tuple(
-            check for action in _split_top_level(actions) for check in _alter_table_checks(table, action, source)
-        )  # comprehension-ok: flattening the per-action checks of one statement
-    return ()
-
-
-def checks_for_migration(name: str, migration_sql: str) -> tuple[Check, ...]:
-    return tuple(
-        check
-        for statement_text in statements(migration_sql)
-        for check in checks_for_statement(statement_text, f"migration {name}")
-    )  # comprehension-ok: flattening the per-statement checks of one migration
-
-
 def checks_for_index(table: str, name: str, columns: tuple[str, ...]) -> tuple[Check, ...]:
     """The entry-size check a request-log index needs before `CREATE INDEX CONCURRENTLY`."""
     return (Check("too_long", table, columns, f'CREATE INDEX "{name}"', "request-log index", index=name),)
@@ -343,11 +200,16 @@ def btree_max_entry_bytes(block_size: int) -> int:
 
 
 def _value_bytes(column: ColumnType, identifier: "sql.Identifier") -> "sql.Composed":
+    """The bytes a value takes in the index entry: its length, or its stored size when
+    Postgres compressed it, since `index_form_tuple` keeps the compressed bytes."""
     from psycopg import sql
 
-    if column.type_name in _OCTET_LENGTH_TYPES:
-        return sql.SQL("octet_length({})").format(identifier)
-    return sql.SQL("octet_length({}::text)").format(identifier)
+    length: Final = (
+        sql.SQL("octet_length({})").format(identifier)
+        if column.type_name in _OCTET_LENGTH_TYPES
+        else sql.SQL("octet_length({}::text)").format(identifier)
+    )
+    return sql.SQL("least({}, pg_column_size({}))").format(length, identifier)
 
 
 def _aligned(offset: "sql.Composable", align: str) -> "sql.Composed":
@@ -419,42 +281,7 @@ def _entry_size_query(check: Check, schema: str, types: tuple[ColumnType, ...],
 
 
 def _count_query(check: Check, schema: str, types: tuple[ColumnType, ...], limit_bytes: int) -> "sql.Composed":
-    from psycopg import sql
-
-    table: Final = sql.Identifier(schema, check.table)
-    columns: Final = sql.SQL(", ").join(sql.Identifier(column) for column in check.columns)
-    match check.kind:
-        case "too_long":
-            return _entry_size_query(check, schema, types, limit_bytes)
-        case "duplicates":
-            not_null: Final = sql.SQL(" AND ").join(
-                sql.SQL("{} IS NOT NULL").format(sql.Identifier(column)) for column in check.columns
-            )
-            return sql.SQL(
-                "SELECT count(*) FROM (SELECT 1 FROM {t} WHERE {nn} GROUP BY {cols} HAVING count(*) > 1) AS d"
-            ).format(t=table, nn=not_null, cols=columns)
-        case "nulls":
-            return sql.SQL("SELECT count(*) FROM {t} WHERE {col} IS NULL").format(
-                t=table, col=sql.Identifier(check.columns[0])
-            )
-        case "any_row":
-            return sql.SQL("SELECT count(*) FROM {t}").format(t=table)
-        case "orphans":
-            pairs: Final = sql.SQL(" AND ").join(
-                sql.SQL("parent.{} = child.{}").format(sql.Identifier(theirs), sql.Identifier(own))
-                for own, theirs in zip(check.columns, check.referenced_columns, strict=True)
-            )
-            present: Final = sql.SQL(" AND ").join(
-                sql.SQL("child.{} IS NOT NULL").format(sql.Identifier(column)) for column in check.columns
-            )
-            return sql.SQL(
-                "SELECT count(*) FROM {t} AS child WHERE {present} AND NOT EXISTS (SELECT 1 FROM {ref} AS parent WHERE {pairs})"
-            ).format(t=table, present=present, ref=sql.Identifier(schema, check.referenced_table or ""), pairs=pairs)
-        case "check_violations":
-            return sql.SQL("SELECT count(*) FROM {t} WHERE NOT ({expr})").format(
-                t=table,
-                expr=sql.SQL(check.expression or "true"),  # pyright: ignore[reportArgumentType]  # copied from our own migration file
-            )
+    return _entry_size_query(check, schema, types, limit_bytes)
 
 
 def _applies(
@@ -469,15 +296,7 @@ def _applies(
         return False
     if check.index is not None and _index_is_valid(connection, schema, check.index):
         return False
-    if check.kind == "any_row":
-        return types is None
-    if types is None:
-        return False
-    if check.kind == "too_long":
-        return any(column.can_overflow_btree for column in types)
-    if check.kind == "orphans":
-        return check.referenced_table is not None and _table_exists(connection, schema, check.referenced_table)
-    return True
+    return types is not None and any(column.can_overflow_btree for column in types)
 
 
 def _too_long_detail(check: Check, types: tuple[ColumnType, ...], values: tuple[object, ...], limit_bytes: int) -> str:
@@ -510,21 +329,7 @@ def _run_check(
     rows: Final = _int(values[0], 0) if values else 0
     if rows == 0:
         return None
-    match check.kind:
-        case "too_long":
-            return BlockedChange(check, rows, _too_long_detail(check, types, values, limit_bytes))
-        case "duplicates":
-            return BlockedChange(check, rows, "duplicated value groups")
-        case "nulls":
-            return BlockedChange(check, rows, "NULL")
-        case "any_row":
-            return BlockedChange(
-                check, rows, "rows, and a NOT NULL column with no DEFAULT cannot be added to a table with rows"
-            )
-        case "orphans":
-            return BlockedChange(check, rows, f'with no matching row in "{check.referenced_table}"')
-        case "check_violations":
-            return BlockedChange(check, rows, f"failing CHECK ({check.expression})")
+    return BlockedChange(check, rows, _too_long_detail(check, types, values, limit_bytes))
 
 
 def find_blockers(
@@ -566,19 +371,7 @@ def find_blockers(
 
 def _remedy(check: Check) -> str:
     joined: Final = ", ".join(check.columns)
-    match check.kind:
-        case "too_long":
-            return f'Shorten or hash the long "{check.table}" ({joined}) values so every row\'s index entry fits.'
-        case "duplicates":
-            return f'Remove or merge the duplicate "{check.table}" ({joined}) values.'
-        case "nulls":
-            return f'Fill the NULL "{check.table}"."{joined}" values, or give the column a default first.'
-        case "any_row":
-            return "Give the new column a DEFAULT in the migration, or add it nullable and fill it afterwards."
-        case "orphans":
-            return f'Delete or repoint the "{check.table}" ({joined}) values with no "{check.referenced_table}" row.'
-        case "check_violations":
-            return f'Fix the "{check.table}" rows that fail the expression before adding the constraint.'
+    return f'Shorten or hash the long "{check.table}" ({joined}) values so every row\'s index entry fits.'
 
 
 def _describe(finding: Finding) -> str:
@@ -602,30 +395,4 @@ def _describe(finding: Finding) -> str:
 def report(findings: tuple[Finding, ...]) -> str:
     """The operator-facing refusal: one block per finding, then what to do next."""
     lines: Final = "\n".join(_describe(finding) for finding in findings)
-    return (
-        f"{REFUSAL}\n{lines}\n"
-        "No schema change was made. Fix the rows and start again, or run `litellm --check_migrations_against_existing_data` with the new image to check again."
-    )
-
-
-def pending_change_checks(
-    connection: "psycopg.Connection[tuple[object, ...]]",
-    schema: str,
-    migrations_dir: Path | None,
-    indexes: tuple[tuple[str, str, tuple[str, ...]], ...],
-) -> tuple[Check, ...]:
-    """Every check the pending migrations and the request-log indexes need. `migrations_dir`
-    is None when the schema is pushed rather than migrated, so only the indexes are checked."""
-    migration_checks: Final = (
-        tuple(
-            check
-            for name, migration_sql in pending_migrations(connection, schema, migrations_dir)
-            for check in checks_for_migration(name, migration_sql)
-        )  # comprehension-ok: flattening the per-migration checks
-        if migrations_dir is not None
-        else ()
-    )
-    index_checks: Final = tuple(
-        check for table, name, columns in indexes for check in checks_for_index(table, name, columns)
-    )  # comprehension-ok: flattening the per-index checks
-    return migration_checks + index_checks
+    return f"{REFUSAL}\n{lines}\nNo index was built. Fix the rows and the next index build picks it up."

litellm-proxy-extras/litellm_proxy_extras/migration_dry_run.py

@@ -0,0 +1,184 @@
+"""`litellm --check_migrations_against_existing_data`: run the pending migrations for real, each
+inside its own transaction that is always rolled back, so Postgres itself says whether this
+database's rows and schema take them. Each migration holds the locks the real upgrade will,
+and releases them before the next one runs."""
+
+from __future__ import annotations
+
+import re
+import time
+from collections.abc import Callable
+from dataclasses import dataclass
+from functools import reduce
+from pathlib import Path
+from typing import TYPE_CHECKING, Final
+
+from litellm_proxy_extras.migration_data_checks import pending_migrations
+
+if TYPE_CHECKING:
+    import psycopg
+
+_CONCURRENTLY: Final = re.compile(
+    r"\b((?:CREATE\s+(?:UNIQUE\s+)?INDEX|DROP\s+INDEX|REINDEX\s+\w+)\s+)CONCURRENTLY\s+", re.IGNORECASE
+)
+UNDEFINED_SQLSTATES: Final = frozenset({"42703", "42P01", "42704", "42883"})
+
+Migration = tuple[str, str]
+
+
+@dataclass(frozen=True, slots=True)
+class Applied:
+    name: str
+    seconds: float
+    replayed: tuple[str, ...] = ()
+
+
+@dataclass(frozen=True, slots=True)
+class Rejected:
+    name: str
+    error: str
+    seconds: float
+
+
+@dataclass(frozen=True, slots=True)
+class Baseline:
+    """A database with tables but no `_prisma_migrations`: the boot baselines it instead of replaying
+    history, so the dry run asks the same question the boot does. `refusal` is None when the boot
+    will baseline, otherwise its reason (a custom migration folder, or the schema diff)."""
+
+    refusal: str | None
+
+    @property
+    def matches(self) -> bool:
+        return self.refusal is None
+
+
+@dataclass(frozen=True, slots=True)
+class DryRun:
+    applied: tuple[Applied, ...]
+    rejected: Rejected | None
+    baseline: Baseline | None = None
+
+    @property
+    def ok(self) -> bool:
+        return self.rejected is None and (self.baseline is None or self.baseline.matches)
+
+
+def transactional(migration_sql: str) -> str:
+    """The migration as it can run inside a transaction: `CONCURRENTLY` index builds become plain ones."""
+    return _CONCURRENTLY.sub(r"\1", migration_sql)
+
+
+def _attempt(
+    connection: psycopg.Connection[tuple[object, ...]],
+    schema: str,
+    chain: tuple[Migration, ...],
+    clock: Callable[[], float],
+) -> tuple[float, psycopg.Error | None]:
+    import psycopg
+    from psycopg import sql
+
+    started: Final = clock()
+    try:
+        with connection.transaction(force_rollback=True):
+            connection.execute(sql.SQL("SET LOCAL search_path TO {}").format(sql.Identifier(schema)))
+            for _, migration_sql in chain:
+                connection.execute(transactional(migration_sql))
+    except psycopg.Error as exc:
+        return clock() - started, exc
+    return clock() - started, None
+
+
+def _run(
+    connection: psycopg.Connection[tuple[object, ...]],
+    schema: str,
+    pending: tuple[Migration, ...],
+    position: int,
+    done: DryRun,
+    clock: Callable[[], float],
+) -> DryRun:
+    """One migration in its own rolled-back transaction. A migration that fails because something
+    an earlier pending migration creates does not exist yet is rerun with those replayed in front."""
+    if done.rejected is not None:
+        return done
+    name: Final = pending[position][0]
+    seconds, error = _attempt(connection, schema, (pending[position],), clock)
+    if error is None:
+        return DryRun(done.applied + (Applied(name, seconds),), None, done.baseline)
+    if error.sqlstate not in UNDEFINED_SQLSTATES or position == 0:
+        return DryRun(done.applied, Rejected(name, str(error).strip(), seconds), done.baseline)
+    earlier: Final = pending[:position]
+    chained_seconds, chained_error = _attempt(connection, schema, earlier + (pending[position],), clock)
+    if chained_error is None:
+        replayed: Final = tuple(earlier_name for earlier_name, _ in earlier)
+        return DryRun(done.applied + (Applied(name, chained_seconds, replayed),), None, done.baseline)
+    return DryRun(done.applied, Rejected(name, str(chained_error).strip(), chained_seconds), done.baseline)
+
+
+def _has_ledger(connection: psycopg.Connection[tuple[object, ...]], schema: str) -> bool:
+    from psycopg import sql
+
+    row: Final = connection.execute(
+        "SELECT to_regclass(%s)", (sql.Identifier(schema, "_prisma_migrations").as_string(connection),)
+    ).fetchone()
+    return row is not None and row[0] is not None
+
+
+def _has_litellm_tables(connection: psycopg.Connection[tuple[object, ...]], schema: str) -> bool:
+    row: Final = connection.execute(
+        "SELECT 1 FROM information_schema.tables WHERE table_schema = %s AND table_name LIKE 'LiteLLM\\_%%' LIMIT 1",
+        (schema,),
+    ).fetchone()
+    return row is not None
+
+
+def dry_run(
+    connection: psycopg.Connection[tuple[object, ...]],
+    schema: str,
+    migrations_dir: Path,
+    baseline_check: Callable[[], str | None] | None = None,
+    clock: Callable[[], float] = time.monotonic,
+) -> DryRun:
+    """Apply every pending migration in order, each in its own transaction that is rolled back
+    before the next one starts. The first migration Postgres rejects ends the run with its error.
+    `baseline_check` (None when the boot would baseline, otherwise why it refuses) stands in for
+    the replay on a database that has tables but no migration history, which the boot baselines."""
+    if baseline_check is not None and not _has_ledger(connection, schema) and _has_litellm_tables(connection, schema):
+        return DryRun((), None, Baseline(baseline_check()))
+    pending: Final = pending_migrations(connection, schema, migrations_dir)
+    return reduce(
+        lambda done, position: _run(connection, schema, pending, position, done, clock),
+        range(len(pending)),
+        DryRun((), None),
+    )
+
+
+def _timing(applied: Applied) -> str:
+    replay: Final = f" (depends on {', '.join(applied.replayed)}, replayed in front of it)" if applied.replayed else ""
+    return f"\n  {applied.name}: {applied.seconds:.1f} s{replay}"
+
+
+def report(result: DryRun) -> str:
+    """What the operator reads: the per-migration time the real upgrade will hold its locks,
+    or the migration Postgres rejected with Postgres's own words. Nothing was applied either way."""
+    if result.baseline is not None:
+        if result.baseline.matches:
+            return (
+                "No migration history and the schema matches this build, "
+                "the proxy baselines it at boot, nothing to dry-run."
+            )
+        return f"No migration history and the proxy cannot baseline it at boot: {result.baseline.refusal}"
+    timings: Final = "".join(_timing(applied) for applied in result.applied)
+    if result.rejected is None:
+        count: Final = len(result.applied)
+        if count == 0:
+            return "No pending migrations, this database is on this version's schema."
+        return (
+            f"{count} pending migration(s) applied one at a time in a dry run, each rolled back before the next "
+            f"so its locks are released, nothing changed:{timings}"
+        )
+    return (
+        f"Migration {result.rejected.name} fails on this database after {result.rejected.seconds:.1f} s, "
+        f"nothing was applied.\n{result.rejected.error}"
+        + (f"\nApplied before it in the dry run:{timings}" if timings else "")
+    )

litellm-proxy-extras/litellm_proxy_extras/migration_recovery.py

@@ -86,6 +86,18 @@ def migration_files(directory: Path) -> tuple[tuple[str, str], ...]:
     )
 
 
+def has_packaged_history(migrations_dir: Path) -> bool:
+    """Whether `migrations_dir` carries exactly the migration files and schema.prisma this package ships,
+    the only history the boot baselines an existing database onto."""
+    packaged_dir: Final = Path(__file__).parent
+    migrations: Final = migration_files(migrations_dir)
+    return (
+        bool(migrations)
+        and migrations == migration_files(packaged_dir)
+        and (migrations_dir / "schema.prisma").read_bytes() == (packaged_dir / "schema.prisma").read_bytes()
+    )
+
+
 def baseline_current_schema(
     coordinator: MigrationCoordinator,
     schema: str,
@@ -95,14 +107,9 @@ def baseline_current_schema(
 ) -> None:
     from psycopg import sql
 
-    packaged_dir: Final = Path(__file__).parent
-    migrations: Final = migration_files(migrations_dir)
-    if (
-        not migrations
-        or migrations != migration_files(packaged_dir)
-        or (migrations_dir / "schema.prisma").read_bytes() != (packaged_dir / "schema.prisma").read_bytes()
-    ):
+    if not has_packaged_history(migrations_dir):
         raise RuntimeError("Cannot automatically baseline an existing database with custom migration history.")
+    migrations: Final = migration_files(migrations_dir)
 
     coordinator.acquire_prisma_lock()
     existing: Final = coordinator.connection.execute(

litellm-proxy-extras/litellm_proxy_extras/migrations/20261021000000_managed_file_created_by_created_at_index/migration.sql

@@ -0,0 +1,2 @@
+CREATE INDEX CONCURRENTLY IF NOT EXISTS "LiteLLM_ManagedFileTable_created_by_created_at_idx"
+ON "LiteLLM_ManagedFileTable" ("created_by", "created_at" DESC);

litellm-proxy-extras/litellm_proxy_extras/migrations/20261027000000_add_accounting_ledger_tables/migration.sql

@@ -0,0 +1,47 @@
+SET LOCAL lock_timeout = '10s';
+
+CREATE TYPE "AccountingEventKind" AS ENUM ('inbound_usage', 'outbound_usage', 'cost');
+
+CREATE TYPE "AccountingDeliveryState" AS ENUM ('pending', 'applied', 'parked');
+
+CREATE TYPE "AccountingDeliveryTarget" AS ENUM ('LiteLLM_VerificationToken', 'LiteLLM_UserTable', 'LiteLLM_TeamTable', 'LiteLLM_TeamMembership', 'LiteLLM_OrganizationTable', 'LiteLLM_OrganizationMembership', 'LiteLLM_EndUserTable', 'LiteLLM_AgentsTable', 'LiteLLM_TagTable', 'LiteLLM_ProjectTable', 'LiteLLM_ModelAccessGroupBudgetTable', 'LiteLLM_BudgetTable', 'LiteLLM_BudgetWindowSpend', 'LiteLLM_DailyUserSpend', 'LiteLLM_DailyTeamSpend', 'LiteLLM_DailyOrganizationSpend', 'LiteLLM_DailyEndUserSpend', 'LiteLLM_DailyAgentSpend', 'LiteLLM_DailyTagSpend', 'LiteLLM_DailyModelUsage', 'LiteLLM_DailyToolSpend', 'LiteLLM_AutoRouterDailySpend');
+
+-- CreateTable
+CREATE TABLE IF NOT EXISTS "LiteLLM_AccountingEvent" (
+    "event_id" TEXT NOT NULL,
+    "kind" "AccountingEventKind" NOT NULL,
+    "source_event_id" TEXT,
+    "schema_version" INTEGER NOT NULL,
+    "payload_hash" TEXT NOT NULL,
+    "occurred_at" TIMESTAMP(3) NOT NULL,
+    "recorded_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "payload" JSONB NOT NULL,
+
+    CONSTRAINT "LiteLLM_AccountingEvent_schema_version_check" CHECK ("schema_version" > 0),
+    CONSTRAINT "LiteLLM_AccountingEvent_pkey" PRIMARY KEY ("event_id")
+);
+
+-- CreateTable
+CREATE TABLE IF NOT EXISTS "LiteLLM_AccountingDelivery" (
+    "event_id" TEXT NOT NULL,
+    "target" "AccountingDeliveryTarget" NOT NULL,
+    "version" INTEGER NOT NULL,
+    "state" "AccountingDeliveryState" NOT NULL DEFAULT 'pending',
+    "attempts" INTEGER NOT NULL DEFAULT 0,
+    "next_attempt_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "applied_at" TIMESTAMP(3),
+    "last_error" TEXT,
+
+    CONSTRAINT "LiteLLM_AccountingDelivery_version_check" CHECK ("version" > 0),
+    CONSTRAINT "LiteLLM_AccountingDelivery_attempts_check" CHECK ("attempts" >= 0),
+    CONSTRAINT "LiteLLM_AccountingDelivery_pkey" PRIMARY KEY ("event_id","target","version")
+);
+
+-- CreateIndex
+CREATE INDEX IF NOT EXISTS "LiteLLM_AccountingEvent_recorded_at_idx" ON "LiteLLM_AccountingEvent"("recorded_at");
+
+-- CreateIndex
+CREATE INDEX IF NOT EXISTS "LiteLLM_AccountingEvent_kind_source_event_id_idx" ON "LiteLLM_AccountingEvent"("kind", "source_event_id");
+
+-- CreateIndex
+CREATE INDEX IF NOT EXISTS "LiteLLM_AccountingDelivery_target_state_next_attempt_at_idx" ON "LiteLLM_AccountingDelivery"("target", "state", "next_attempt_at");

litellm-proxy-extras/litellm_proxy_extras/schema.prisma

@@ -1163,6 +1163,7 @@ model LiteLLM_ManagedFileTable {
   @@index([unified_file_id])
   @@index([flat_model_file_ids], type: Gin)
   @@index([team_id, created_at(sort: Desc)])
+  @@index([created_by, created_at(sort: Desc)])
 }
 
 model LiteLLM_ManagedObjectTable { // for batches or finetuning jobs which use the
@@ -2060,3 +2061,73 @@ model LiteLLM_TelemetryReport {
   @@index([window_end])
   @@index([created_at])
 }
+
+enum AccountingEventKind {
+  inbound_usage
+  outbound_usage
+  cost
+}
+
+enum AccountingDeliveryState {
+  pending
+  applied
+  parked
+}
+
+enum AccountingDeliveryTarget {
+  LiteLLM_VerificationToken
+  LiteLLM_UserTable
+  LiteLLM_TeamTable
+  LiteLLM_TeamMembership
+  LiteLLM_OrganizationTable
+  LiteLLM_OrganizationMembership
+  LiteLLM_EndUserTable
+  LiteLLM_AgentsTable
+  LiteLLM_TagTable
+  LiteLLM_ProjectTable
+  LiteLLM_ModelAccessGroupBudgetTable
+  LiteLLM_BudgetTable
+  LiteLLM_BudgetWindowSpend
+  LiteLLM_DailyUserSpend
+  LiteLLM_DailyTeamSpend
+  LiteLLM_DailyOrganizationSpend
+  LiteLLM_DailyEndUserSpend
+  LiteLLM_DailyAgentSpend
+  LiteLLM_DailyTagSpend
+  LiteLLM_DailyModelUsage
+  LiteLLM_DailyToolSpend
+  LiteLLM_AutoRouterDailySpend
+}
+
+// Durable accounting ledger. One row per usage or cost event; payload holds the
+// private event envelope versioned by schema_version. Usage rows retain request-scoped
+// pricing context; cost rows include resolved rates and the finalized pricing snapshot.
+model LiteLLM_AccountingEvent {
+  event_id        String              @id
+  kind            AccountingEventKind
+  source_event_id String?
+  schema_version  Int
+  payload_hash    String
+  occurred_at     DateTime
+  recorded_at     DateTime            @default(now())
+  payload         Json
+
+  @@index([recorded_at])
+  @@index([kind, source_event_id])
+}
+
+// One row per (event, target table, version): pending work for the projector and,
+// once applied in the same transaction as the target write, the receipt that stops a retry.
+model LiteLLM_AccountingDelivery {
+  event_id        String
+  target          AccountingDeliveryTarget
+  version         Int
+  state           AccountingDeliveryState  @default(pending)
+  attempts        Int                      @default(0)
+  next_attempt_at DateTime                 @default(now())
+  applied_at      DateTime?
+  last_error      String?
+
+  @@id([event_id, target, version])
+  @@index([target, state, next_attempt_at])
+}

litellm-proxy-extras/litellm_proxy_extras/utils.py

@@ -11,18 +11,13 @@ from collections.abc import Callable
 from contextlib import AbstractContextManager, nullcontext
 from dataclasses import dataclass, replace
 from pathlib import Path
-from typing import TYPE_CHECKING, Final, Optional, Protocol, Union
+from typing import TYPE_CHECKING, Final, Protocol
 
 from litellm_proxy_extras import prisma_toolchain
 from litellm_proxy_extras._logging import logger
 from litellm_proxy_extras._redaction import _redact_credentials
-from litellm_proxy_extras.migration_data_checks import (
-    Finding,
-    find_blockers,
-    migration_data_check_timeout_seconds,
-    pending_change_checks,
-    report,
-)
+from litellm_proxy_extras.migration_dry_run import dry_run
+from litellm_proxy_extras.migration_dry_run import report as dry_run_report
 from litellm_proxy_extras.migration_lock import held_migration_lock
 from litellm_proxy_extras.prisma_toolchain import (
     PRISMA_COMMAND_TIMEOUT_ENV_VAR,
@@ -36,7 +31,6 @@ from litellm_proxy_extras.replica_identity import (
     apply_replica_identity_full,
 )
 from litellm_proxy_extras.request_log_indexes import (
-    REQUEST_LOG_INDEXES,
     ensure_request_log_indexes,
     filter_request_log_index_diff,
 )
@@ -52,7 +46,7 @@ class LensCheckConnect(Protocol):
     ) -> "psycopg.Connection[tuple[object, ...]]": ...
 
 
-def str_to_bool(value: Optional[str]) -> bool:
+def str_to_bool(value: str | None) -> bool:
     if value is None:
         return False
     return value.lower() in ("true", "1", "t", "y", "yes")
@@ -169,7 +163,7 @@ def _without_sql_comments(statement: str) -> str:
     ).strip()
 
 
-def _without_spend_logs_pk_clauses(statement: str) -> Optional[str]:
+def _without_spend_logs_pk_clauses(statement: str) -> str | None:
     prefix_match = _SPEND_LOGS_ALTER_RE.match(statement)
     if not prefix_match:
         return statement
@@ -217,7 +211,7 @@ def _max_migration_timestamp(names) -> int:
     return max(_migration_timestamp(n) for n in names)
 
 
-def _redacted_command(command: object) -> Union[str, tuple[str, ...], list[str]]:
+def _redacted_command(command: object) -> str | tuple[str, ...] | list[str]:
     if isinstance(command, tuple):
         return tuple(_redact_credentials(str(argument)) for argument in command)
     if isinstance(command, list):
@@ -404,7 +398,7 @@ class ProxyExtrasDBManager:
         return tuple(row) if row is not None else ()
 
     @staticmethod
-    def _failed_migration_logs(migration_name: str, started_at: str) -> Optional[str]:
+    def _failed_migration_logs(migration_name: str, started_at: str) -> str | None:
         row: Final = ProxyExtrasDBManager._read_migration_ledger(
             "SELECT logs FROM {} WHERE migration_name = %s AND started_at = %s::timestamptz "
             "AND finished_at IS NULL AND rolled_back_at IS NULL",
@@ -721,10 +715,10 @@ class ProxyExtrasDBManager:
         return row is not None
 
     @staticmethod
-    def _prisma_schema_param(url: str) -> Optional[str]:
+    def _prisma_schema_param(url: str) -> str | None:
         """The `schema` query param Prisma uses to pick its target schema,
         or None when the URL does not set one."""
-        from urllib.parse import urlparse, parse_qsl
+        from urllib.parse import parse_qsl, urlparse
 
         return next(
             (v for k, v in parse_qsl(urlparse(url).query) if k == "schema"),
@@ -1424,7 +1418,6 @@ class ProxyExtrasDBManager:
             bool: True if setup was successful, False otherwise
         """
         ensure_prisma_toolchain(prisma_command=_get_prisma_command(), prisma_env=_get_prisma_env())
-        ProxyExtrasDBManager.raise_if_rows_block_pending_changes(use_migrate=use_migrate)
         migrated = ProxyExtrasDBManager._run_migrations(use_migrate=use_migrate, use_v2_resolver=use_v2_resolver)
         if not migrated:
             return False
@@ -1433,46 +1426,69 @@ class ProxyExtrasDBManager:
         return True
 
     @staticmethod
-    def migration_data_check_findings(use_migrate: bool = True) -> tuple[Finding, ...]:
-        """What in the existing rows would make a pending migration or request-log index
-        fail, read with count queries only: no write, no table lock. Empty when nothing
-        blocks, when no database is configured, or when it cannot be reached, so the
-        migration step that follows reports the connection problem the way it does today."""
+    def _baseline_refusal(prisma_dir: Path) -> str | None:
+        """Why `baseline_current_schema` would refuse this database at boot, None when it would baseline:
+        the migrations folder must be the packaged one, then `prisma migrate diff` must find the live
+        database and this build's schema.prisma identical."""
+        from litellm_proxy_extras.migration_lock import migration_environment
+        from litellm_proxy_extras.migration_recovery import has_packaged_history
+
+        if not has_packaged_history(prisma_dir):
+            return (
+                f"{prisma_dir} is a custom migration history (LITELLM_MIGRATION_DIR), it differs from the "
+                "migrations this build ships, and the proxy only baselines onto the packaged history."
+            )
+        try:
+            prisma_toolchain.run_prisma(
+                (
+                    _get_prisma_command(),
+                    "migrate",
+                    "diff",
+                    "--from-schema-datasource",
+                    str(prisma_dir / "schema.prisma"),
+                    "--to-schema-datamodel",
+                    str(prisma_dir / "schema.prisma"),
+                    "--exit-code",
+                ),
+                timeout=prisma_toolchain.prisma_command_timeout(),
+                env=migration_environment(_get_prisma_env()),
+            )
+        except subprocess.CalledProcessError as exc:
+            diff: Final = str(exc.stdout or exc.stderr or exc).strip()
+            return f"the schema differs from this build. prisma migrate diff:\n{diff}"
+        except subprocess.TimeoutExpired as exc:
+            return f"prisma migrate diff did not finish within {exc.timeout} s"
+        return None
+
+    @staticmethod
+    def migration_dry_run_summary(use_migrate: bool = True) -> tuple[bool, str]:
+        """`--check_migrations_against_existing_data`: the pending migrations applied for real on
+        DIRECT_URL or DATABASE_URL inside one transaction that is rolled back, so Postgres reports
+        what the upgrade would hit. (True, the per-migration timings) when they all apply,
+        (False, Postgres's own error) when one does not."""
         import psycopg
 
+        if not use_migrate:
+            return (True, "The schema is pushed rather than migrated, there is nothing to dry-run.")
         database_url: Final = os.environ.get("DATABASE_URL")
         if not database_url:
-            return ()
+            return (False, "DATABASE_URL is not set.")
         direct_url: Final = ProxyExtrasDBManager._strip_prisma_query_params(
             os.environ.get("DIRECT_URL") or database_url
         )
         schema: Final = ProxyExtrasDBManager._prisma_schema_param(database_url) or "public"
-        migrations_dir: Final = Path(ProxyExtrasDBManager._get_prisma_dir()) / "migrations" if use_migrate else None
-        indexes: Final = tuple((index.table, index.name, index.columns) for index in REQUEST_LOG_INDEXES)
+        prisma_dir: Final = Path(ProxyExtrasDBManager._get_prisma_dir())
         try:
             with psycopg.connect(direct_url, connect_timeout=10, autocommit=True) as connection:
-                checks: Final = pending_change_checks(connection, schema, migrations_dir, indexes)
-                return find_blockers(connection, schema, checks, migration_data_check_timeout_seconds())
+                result: Final = dry_run(
+                    connection,
+                    schema,
+                    prisma_dir / "migrations",
+                    baseline_check=lambda: ProxyExtrasDBManager._baseline_refusal(prisma_dir),
+                )
         except psycopg.OperationalError as exc:
-            logger.warning(
-                "Could not reach the database for the pre-migration row check: %s", _redact_credentials(str(exc))
-            )
-            return ()
-
-    @staticmethod
-    def migration_data_check_report(use_migrate: bool = True) -> Optional[str]:
-        """The operator-facing refusal for `migration_data_check_findings`, None when nothing blocks."""
-        findings: Final = ProxyExtrasDBManager.migration_data_check_findings(use_migrate=use_migrate)
-        return report(findings) if findings else None
-
-    @staticmethod
-    def raise_if_rows_block_pending_changes(use_migrate: bool) -> None:
-        """Refuse to run migrations that the existing rows would make fail, before any is
-        applied. The RuntimeError carries the report and reaches the operator through the
-        proxy's "Database migration cannot proceed" exit."""
-        refusal: Final = ProxyExtrasDBManager.migration_data_check_report(use_migrate=use_migrate)
-        if refusal is not None:
-            raise RuntimeError(refusal)
+            return (False, f"Could not reach the database: {_redact_credentials(str(exc))}")
+        return (result.ok, dry_run_report(result))
 
     @staticmethod
     def build_request_log_indexes(build: Callable[[str, str], bool] = ensure_request_log_indexes) -> bool:
@@ -1724,5 +1740,4 @@ class ProxyExtrasDBManager:
                 time.sleep(random.randrange(5, 15))
             finally:
                 os.chdir(original_dir)
-                pass
         return False

litellm-rust/.agents/skills/rust-error/SKILL.md

@@ -1,6 +1,8 @@
 ---
 name: rust-error
 description: Define, wrap, convert, classify, report or test Rust error types in litellm-rust with thiserror. Use when adding an error type or variant, mapping one crate's error into another, turning an error into a gateway response or Python exception, deciding retry or fault from an error, or replacing call sites that build errors from strings
+metadata:
+  internal: true
 ---
 
 # Rust errors

litellm-rust/.agents/skills/rust-gateway/SKILL.md

@@ -1,6 +1,8 @@
 ---
 name: rust-gateway
 description: Add or change an HTTP route, extractor, response envelope, stream or router mount in the litellm-rust gateway crates (crates/gateway and crates/gateway-*). Use when adding an endpoint, parsing a request body, choosing an error envelope for an API surface, relaying SSE, or writing a router-level test
+metadata:
+  internal: true
 ---
 
 # Rust gateway

litellm-rust/.agents/skills/rust-string-enums/SKILL.md

@@ -1,6 +1,8 @@
 ---
 name: rust-string-enums
 description: Define or refactor Rust string-valued enums and their Serde adapters in litellm-rust, using Strum and serde_with while preserving parsing, wire values, and schemas
+metadata:
+  internal: true
 ---
 
 # Rust string enums

litellm-rust/.agents/skills/rust-tracing/SKILL.md

@@ -1,6 +1,8 @@
 ---
 name: rust-tracing
 description: Add or change Rust diagnostic tracing in litellm-rust, including route spans, subscriber layers, and Python logger delivery
+metadata:
+  internal: true
 ---
 
 # Rust tracing

litellm-rust/Cargo.lock

@@ -3150,6 +3150,7 @@ dependencies = [
  "defmt",
  "jiff-core",
  "jiff-static",
+ "jiff-tzdb",
  "log",
  "portable-atomic",
  "portable-atomic-util",
@@ -3178,6 +3179,12 @@ dependencies = [
  "syn 2.0.119",
 ]
 
+[[package]]
+name = "jiff-tzdb"
+version = "0.1.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fa8377070c6bae868759445e5a77f66d84f0b72f3a054bfb00e6d038b8282da7"
+
 [[package]]
 name = "jni"
 version = "0.22.4"
@@ -3729,9 +3736,12 @@ dependencies = [
 name = "litellm-cost"
 version = "0.1.0"
 dependencies = [
- "criterion",
- "proptest",
+ "jiff",
  "rstest",
+ "schemars 1.2.2",
+ "serde",
+ "serde_json",
+ "strum",
  "thiserror 2.0.19",
 ]
 
@@ -4277,17 +4287,35 @@ name = "litellm-model-catalog"
 version = "0.1.0"
 dependencies = [
  "indexmap 2.14.0",
+ "jiff",
  "jsonschema",
+ "litellm-cost",
  "litellm-error",
- "litellm-llms-types",
+ "macro_rules_attribute",
  "rstest",
  "schemars 1.2.2",
  "serde",
  "serde_json",
+ "serde_with",
+ "strum",
  "thiserror 2.0.19",
  "time",
 ]
 
+[[package]]
+name = "litellm-model-catalog-refresh"
+version = "0.1.0"
+dependencies = [
+ "litellm-error",
+ "litellm-http",
+ "litellm-model-catalog",
+ "reqwest",
+ "rstest",
+ "thiserror 2.0.19",
+ "tokio",
+ "url",
+]
+
 [[package]]
 name = "litellm-python-bridge"
 version = "0.1.0"
@@ -4317,6 +4345,7 @@ dependencies = [
  "litellm-inference-transcription",
  "litellm-llms",
  "litellm-llms-types",
+ "litellm-model-catalog",
  "litellm-router-types",
  "litellm-secrets",
  "litellm-secrets-aws",
@@ -4387,12 +4416,17 @@ name = "litellm-router"
 version = "0.1.0"
 dependencies = [
  "litellm-config",
+ "litellm-core-utils",
  "litellm-inference",
  "litellm-inference-messages",
  "litellm-router-types",
  "rstest",
 ]
 
+[[package]]
+name = "litellm-router-core"
+version = "0.1.0"
+
 [[package]]
 name = "litellm-router-types"
 version = "0.1.0"

litellm-rust/Cargo.toml

@@ -73,6 +73,9 @@ litellm-python-compat = { path = "crates/python-compat" }
 litellm-circuit-breaker = { path = "crates/circuit-breaker" }
 litellm-redis-client = { path = "crates/redis-client" }
 litellm-redis-testing = { path = "crates/redis-testing" }
+litellm-cost = { path = "crates/cost" }
+litellm-model-catalog = { path = "crates/model-catalog" }
+litellm-model-catalog-refresh = { path = "crates/model-catalog-refresh" }
 
 askama = { version = "0.16.1", default-features = false, features = ["derive", "std"] }
 tracing = "0.1"
@@ -109,6 +112,7 @@ serde = { version = "1.0", features = ["derive"] }
 serde_json = { version = "1.0", features = ["float_roundtrip"] }
 serde_with = { version = "=3.21.0", default-features = false, features = ["std", "macros"] }
 sha2 = "0.10"
+sha1 = "0.10"
 syn = { version = "2", default-features = false }
 sqlx = { version = "0.9.0", default-features = false, features = ["json", "macros", "postgres", "runtime-tokio", "chrono", "tls-rustls-ring-native-roots"] }
 chrono = { version = "0.4.45", default-features = false, features = ["std"] }

litellm-rust/crates/core-utils/src/core_helpers.rs

@@ -13,6 +13,7 @@ const FINISH_REASONS: &[(&str, &str)] = &[
     ("max_tokens", "length"),
     ("refusal", "content_filter"),
     ("compaction", "length"),
+    ("model_context_window_exceeded", "length"),
     ("guardrail_intervened", "content_filter"),
     ("content_filtered", "content_filter"),
     ("content_filter", "content_filter"),
@@ -74,6 +75,7 @@ mod tests {
     #[case::end_turn("end_turn", "stop")]
     #[case::stop_sequence("stop_sequence", "stop")]
     #[case::max_tokens("max_tokens", "length")]
+    #[case::model_context_window_exceeded("model_context_window_exceeded", "length")]
     #[case::refusal("refusal", "content_filter")]
     #[case::guardrail_intervened("guardrail_intervened", "content_filter")]
     #[case::content_filtered("content_filtered", "content_filter")]

litellm-rust/crates/core-utils/src/exception_mapping_utils/openai.rs

@@ -53,6 +53,7 @@ pub(super) const RULES: &[Rule] = &[
         |mapping| {
             mapping.error_str.contains("invalid_request_error")
                 && !mapping.error_str.contains("Incorrect API key provided")
+                && !matches!(mapping.status, Some(401 | 404))
         },
         PublicError::BadRequest,
     ),
@@ -132,6 +133,23 @@ mod tests {
         assert_eq!(classified(Some(400), text), Some(expected));
     }
 
+    #[rstest::rstest]
+    #[case::unauthorized(Some(401), None)]
+    #[case::not_found(Some(404), None)]
+    #[case::forbidden(Some(403), Some(PublicError::BadRequest))]
+    #[case::rate_limited(Some(429), Some(PublicError::BadRequest))]
+    #[case::server_error(Some(500), Some(PublicError::BadRequest))]
+    #[case::no_status(None, Some(PublicError::BadRequest))]
+    fn invalid_request_errors_leave_401_and_404_to_the_status_mapping(
+        #[case] status: Option<u16>,
+        #[case] expected: Option<PublicError>,
+    ) {
+        assert_eq!(
+            classified(status, "invalid_request_error bad field"),
+            expected
+        );
+    }
+
     #[rstest::rstest]
     #[case::rate_limit_before_context_window(
         "rate limit and This model's maximum context length is 10",

litellm-rust/crates/core-utils/src/serde_compat.rs

@@ -1,13 +1,25 @@
-pub fn parse_str_bool(value: &str) -> Option<bool> {
-    let token = value.trim_matches(|character: char| {
+fn python_strip(value: &str) -> &str {
+    value.trim_matches(|character: char| {
         character.is_whitespace() || matches!(character, '\u{1c}'..='\u{1f}')
-    });
+    })
+}
+
+pub fn parse_str_bool(value: &str) -> Option<bool> {
+    let token = python_strip(value);
     if token.eq_ignore_ascii_case("true") {
         return Some(true);
     }
     token.eq_ignore_ascii_case("false").then_some(false)
 }
 
+pub fn parse_pydantic_bool(value: &str) -> Option<bool> {
+    match python_strip(value).to_ascii_lowercase().as_str() {
+        "1" | "on" | "t" | "true" | "y" | "yes" => Some(true),
+        "0" | "off" | "f" | "false" | "n" | "no" => Some(false),
+        _ => None,
+    }
+}
+
 /// `redis-py` string Booleans: only `1`, `true`, and `yes` (case-insensitive) are true.
 pub fn parse_redis_bool(value: &str) -> bool {
     value == "1" || value.eq_ignore_ascii_case("true") || value.eq_ignore_ascii_case("yes")
@@ -34,4 +46,27 @@ mod tests {
     ) {
         assert_eq!(parse_str_bool(input), expected, "{input:?}");
     }
+
+    #[rstest]
+    #[case::yes(" YES ", Some(true))]
+    #[case::on("On", Some(true))]
+    #[case::one("1", Some(true))]
+    #[case::t("t", Some(true))]
+    #[case::y("y", Some(true))]
+    #[case::control_whitespace_true("\u{1c}TRUE\u{1f}", Some(true))]
+    #[case::no("No", Some(false))]
+    #[case::off("OFF", Some(false))]
+    #[case::zero("0", Some(false))]
+    #[case::f("F", Some(false))]
+    #[case::n("n", Some(false))]
+    #[case::false_word("false", Some(false))]
+    #[case::two("2", None)]
+    #[case::empty("", None)]
+    #[case::unknown("maybe", None)]
+    fn boolean_words_follow_pydantic_lax_bool_parsing(
+        #[case] input: &str,
+        #[case] expected: Option<bool>,
+    ) {
+        assert_eq!(parse_pydantic_bool(input), expected, "{input:?}");
+    }
 }

litellm-rust/crates/cost/Cargo.toml

@@ -5,14 +5,17 @@ edition.workspace = true
 license.workspace = true
 repository.workspace = true
 
+[features]
+serde = ["dep:serde"]
+schema = ["serde", "dep:schemars"]
+
 [dependencies]
+jiff = { version = "0.2.37", default-features = false, features = ["std"] }
+schemars = { workspace = true, optional = true }
+serde = { workspace = true, optional = true }
 thiserror.workspace = true
+strum.workspace = true
 
 [dev-dependencies]
-criterion.workspace = true
-proptest.workspace = true
 rstest.workspace = true
-
-[[bench]]
-name = "calculate"
-harness = false
+serde_json.workspace = true

litellm-rust/crates/cost/README.md

@@ -1,13 +1,3 @@
 # litellm-cost
 
-This crate calculates text token charges from rates and usage supplied by its caller. It is standalone and has no Python bridge or proxy integration
-
-Call `compile(&pricing)` once for an immutable plan, then `plan.calculate(&request)` for each supported request. `calculate(&pricing, &request)` compiles on each call. A successful result exposes pre-multiplier component costs, selected rates, the multiplier, and derived `input()`, `output()`, and `total()` values
-
-The caller states whether `prompt_tokens` includes cache tokens. Threshold selection uses total input tokens for either convention and selects one rate for the whole request. Thresholds are sorted when compiled, and duplicate thresholds or tier overrides fail deterministically. `Fast` selects priority rates; unknown tiers use standard rates
-
-`Rate::Missing`, `Rate::Null`, and `Rate::Value(0.0)` remain distinct. Missing cache rates fall back to the selected input rate, and an absent one-hour write rate falls back to the selected write rate. Missing input or output rates return typed errors, including for zero usage. Python's sparse-entry behavior remains outside this native contract
-
-The supported off-peak shape is one non-wrapping UTC daily window. The caller supplies the applicable regional multiplier after provider-specific selection. Negative or non-finite rates, ambiguous rules, inconsistent cache counts, incomplete write splits, invalid windows and overflow return errors. Callers must decline unsupported inputs before native execution if their public contract accepts those shapes
-
-This crate does not select models, read catalogs, fetch provider prices, normalize multimodal usage, process provider-reported costs, or calculate non-token charges. It does not change proxy behavior. The reference fixture was generated by `tests/generate_python_reference.py` against the Python implementation at the commit recorded in `tests/python_reference.tsv`, using synthetic rates and fixed usage
+This crate holds the price data model that `litellm-model-catalog` lowers each catalog entry into, plus the usage, call context and priced outcome types that `calculate` will compute over. There is no `calculate` until the token pricing PR lands. `UnitPrice` is USD per unit and `Usd` is an amount, and only the operations that make sense between them exist, so a price is never added to an amount. `Prices::band` and `Prices::rate` resolve a token rate the way Python's `_get_token_base_cost` does

litellm-rust/crates/cost/benches/calculate.rs

@@ -1,66 +0,0 @@
-use criterion::{Criterion, criterion_group, criterion_main};
-use litellm_cost::{
-    Pricing, PromptConvention, Rate, Rates, Request, ServiceTier, ThresholdPolicy, ThresholdRates,
-    Usage, calculate, compile,
-};
-use std::hint::black_box;
-
-fn bench(c: &mut Criterion) {
-    let pricing = Pricing {
-        standard: Rates {
-            input: Rate::Value(0.000002),
-            output: Rate::Value(0.000008),
-            cache_read: Rate::Value(0.0000005),
-            cache_write: Rate::Missing,
-            cache_write_1h: Rate::Missing,
-        },
-        tiers: &[],
-        thresholds: &[],
-        off_peak: None,
-    };
-    let request = Request {
-        usage: Usage {
-            prompt_tokens: 1000,
-            completion_tokens: 200,
-            cache_read_tokens: 250,
-            cache_write_tokens: 0,
-            cache_write_5m_tokens: None,
-            cache_write_1h_tokens: None,
-            prompt_convention: PromptConvention::IncludesCache,
-        },
-        service_tier: ServiceTier::Standard,
-        threshold_policy: ThresholdPolicy::Exclusive,
-        region_multiplier: None,
-        billed_at_utc_minute: None,
-    };
-    let plan = compile(&pricing).unwrap();
-    c.bench_function("native_compiled_calculation", |b| {
-        b.iter(|| black_box(plan.calculate(black_box(&request)).unwrap()))
-    });
-    c.bench_function("native_full_wrapper", |b| {
-        b.iter(|| black_box(calculate(black_box(&pricing), black_box(&request)).unwrap()))
-    });
-    c.bench_function("native_rate_compilation", |b| {
-        b.iter(|| black_box(compile(black_box(&pricing)).unwrap()))
-    });
-    let threshold = ThresholdRates {
-        above_prompt_tokens: 1000,
-        standard: Rates {
-            input: Rate::Value(0.000004),
-            output: Rate::Value(0.000016),
-            ..Rates::EMPTY
-        },
-        tiers: &[],
-    };
-    let threshold_pricing = Pricing {
-        thresholds: &[threshold],
-        ..pricing
-    };
-    let threshold_plan = compile(&threshold_pricing).unwrap();
-    c.bench_function("native_threshold_boundary", |b| {
-        b.iter(|| black_box(threshold_plan.calculate(black_box(&request)).unwrap()))
-    });
-}
-
-criterion_group!(benches, bench);
-criterion_main!(benches);

litellm-rust/crates/cost/examples/charge.rs

@@ -1,40 +0,0 @@
-use litellm_cost::{
-    Pricing, PromptConvention, Rate, Rates, Request, ServiceTier, ThresholdPolicy, Usage, compile,
-};
-
-fn main() {
-    let pricing = Pricing {
-        standard: Rates {
-            input: Rate::Value(2.0),
-            output: Rate::Value(4.0),
-            cache_read: Rate::Value(0.5),
-            cache_write: Rate::Value(3.0),
-            cache_write_1h: Rate::Missing,
-        },
-        tiers: &[],
-        thresholds: &[],
-        off_peak: None,
-    };
-    let request = Request {
-        usage: Usage {
-            prompt_tokens: 100,
-            completion_tokens: 20,
-            cache_read_tokens: 25,
-            cache_write_tokens: 10,
-            cache_write_5m_tokens: None,
-            cache_write_1h_tokens: None,
-            prompt_convention: PromptConvention::IncludesCache,
-        },
-        service_tier: ServiceTier::Standard,
-        threshold_policy: ThresholdPolicy::Exclusive,
-        region_multiplier: None,
-        billed_at_utc_minute: None,
-    };
-    let cost = compile(&pricing).unwrap().calculate(&request).unwrap();
-    println!(
-        "input={} output={} total={}",
-        cost.input(),
-        cost.output(),
-        cost.total()
-    );
-}

litellm-rust/crates/cost/src/call.rs

@@ -0,0 +1,42 @@
+use crate::price::{Channel, Prices, Residency, ServiceTier};
+
+/// Facts about the call, never about the row or the provider
+#[derive(Clone, Debug, PartialEq)]
+pub struct CallContext {
+    /// Requested service tier, `"fast"` maps to `Priority` as on main
+    pub tier: ServiceTier,
+    /// Online, batch or decisions
+    pub channel: Channel,
+    /// OpenAI data residency, picks the residency uplift
+    pub residency: Option<Residency>,
+    /// Vertex call sent to a regional endpoint, picks the regional uplift
+    pub non_global_endpoint: bool,
+    /// Anthropic `speed: fast`, picks the speed uplift
+    pub speed: Option<Speed>,
+    /// Anthropic and Bedrock `inference_geo`, picks the geo uplift
+    pub inference_geo: Option<InferenceGeo>,
+    /// When the call was billed, for off-peak windows
+    pub at: jiff::Timestamp,
+    /// Per-call custom pricing, lowered like a row and applied field by field over it
+    pub overlay: Option<Prices>,
+}
+
+/// Anthropic `speed`
+#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
+pub enum Speed {
+    Fast,
+}
+
+/// Anthropic and Bedrock `inference_geo`, lowercased as on main
+#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
+pub struct InferenceGeo(Box<str>);
+
+impl InferenceGeo {
+    pub fn new(geo: &str) -> Self {
+        Self(geo.to_lowercase().into())
+    }
+
+    pub fn as_str(&self) -> &str {
+        &self.0
+    }
+}

litellm-rust/crates/cost/src/error.rs

@@ -1,27 +1,24 @@
-#[derive(Clone, Copy, Debug, Eq, PartialEq, thiserror::Error)]
+use thiserror::Error;
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Error, strum::VariantArray)]
 pub enum Error {
-    #[error("model has no input rate")]
-    MissingInputRate,
-    #[error("model has no output rate")]
-    MissingOutputRate,
-    #[error("rate must be finite and nonnegative")]
-    InvalidRate,
-    #[error("region multiplier must be finite and positive")]
-    InvalidRegionMultiplier,
-    #[error("off-peak pricing needs a billing minute within the day")]
-    InvalidBillingTime,
-    #[error("off-peak window must start before it ends within the day")]
-    InvalidOffPeakWindow,
-    #[error("cached tokens exceed prompt tokens")]
-    CacheExceedsPrompt,
-    #[error("cache write details do not add up to the cache write tokens")]
-    InvalidCacheWriteDetails,
-    #[error("token count or cost overflows")]
-    TokenCountOverflow,
-    #[error("service tier is repeated or cannot carry its own rates")]
-    DuplicateTier,
-    #[error("prompt token threshold is repeated")]
-    DuplicateThreshold,
-    #[error("service tier is repeated or cannot carry its own rates within a threshold")]
-    DuplicateThresholdTier,
+    #[error("expected a UTC window HH:MM-HH:MM")]
+    InvalidUtcWindow,
+    #[error("expected an ISO weekday number 1-7 or an English day name")]
+    InvalidWeekday,
+}
+
+#[cfg(test)]
+mod tests {
+    use rstest::rstest;
+
+    use super::Error;
+    use crate::schedule::{UtcWindow, Weekday};
+
+    #[rstest]
+    #[case::window("not-a-window".parse::<UtcWindow>().unwrap_err(), Error::InvalidUtcWindow)]
+    #[case::weekday("not-a-weekday".parse::<Weekday>().unwrap_err(), Error::InvalidWeekday)]
+    fn schedule_parsers_share_the_crate_error(#[case] error: Error, #[case] expected: Error) {
+        assert_eq!(error, expected);
+    }
 }

litellm-rust/crates/cost/src/lib.rs

@@ -1,383 +1,9 @@
+pub mod call;
 mod error;
+pub mod money;
+pub mod price;
+pub mod priced;
+pub mod schedule;
+pub mod usage;
 
-pub use error::Error;
-
-#[derive(Clone, Copy, Debug, PartialEq)]
-pub enum Rate {
-    Missing,
-    Null,
-    Value(f64),
-}
-
-impl Rate {
-    fn value(self) -> Option<f64> {
-        match self {
-            Self::Value(value) => Some(value),
-            Self::Missing | Self::Null => None,
-        }
-    }
-
-    fn or(self, fallback: Self) -> Self {
-        if self.value().is_some() {
-            self
-        } else {
-            fallback
-        }
-    }
-}
-
-#[derive(Clone, Copy, Debug, PartialEq)]
-pub struct Rates {
-    pub input: Rate,
-    pub output: Rate,
-    pub cache_read: Rate,
-    pub cache_write: Rate,
-    pub cache_write_1h: Rate,
-}
-
-impl Rates {
-    pub const EMPTY: Self = Self {
-        input: Rate::Missing,
-        output: Rate::Missing,
-        cache_read: Rate::Missing,
-        cache_write: Rate::Missing,
-        cache_write_1h: Rate::Missing,
-    };
-
-    fn overlay(self, base: Self) -> Self {
-        Self {
-            input: self.input.or(base.input),
-            output: self.output.or(base.output),
-            cache_read: self.cache_read.or(base.cache_read),
-            cache_write: self.cache_write.or(base.cache_write),
-            cache_write_1h: self.cache_write_1h.or(base.cache_write_1h),
-        }
-    }
-}
-
-#[derive(Clone, Copy, Debug, Eq, PartialEq)]
-pub enum ServiceTier {
-    Standard,
-    Flex,
-    Priority,
-    Fast,
-    Ultrafast,
-    Unknown,
-}
-
-#[derive(Clone, Copy, Debug, Eq, PartialEq)]
-pub enum ThresholdPolicy {
-    Exclusive,
-    Inclusive,
-}
-
-#[derive(Clone, Copy, Debug, Eq, PartialEq)]
-pub enum PromptConvention {
-    IncludesCache,
-    ExcludesCache,
-}
-
-#[derive(Clone, Copy, Debug, PartialEq)]
-pub struct Usage {
-    pub prompt_tokens: u64,
-    pub completion_tokens: u64,
-    pub cache_read_tokens: u64,
-    pub cache_write_tokens: u64,
-    pub cache_write_5m_tokens: Option<u64>,
-    pub cache_write_1h_tokens: Option<u64>,
-    pub prompt_convention: PromptConvention,
-}
-
-#[derive(Clone, Copy, Debug, PartialEq)]
-pub struct TierRates {
-    pub tier: ServiceTier,
-    pub rates: Rates,
-}
-
-#[derive(Clone, Copy, Debug, PartialEq)]
-pub struct ThresholdRates<'a> {
-    pub above_prompt_tokens: u64,
-    pub standard: Rates,
-    pub tiers: &'a [TierRates],
-}
-
-#[derive(Clone, Copy, Debug, PartialEq)]
-pub struct OffPeakRates {
-    pub start_utc_minute: u16,
-    pub end_utc_minute: u16,
-    pub rates: Rates,
-}
-
-#[derive(Clone, Copy, Debug, PartialEq)]
-pub struct Pricing<'a> {
-    pub standard: Rates,
-    pub tiers: &'a [TierRates],
-    pub thresholds: &'a [ThresholdRates<'a>],
-    pub off_peak: Option<OffPeakRates>,
-}
-
-#[derive(Clone, Copy, Debug, PartialEq)]
-pub struct Request {
-    pub usage: Usage,
-    pub service_tier: ServiceTier,
-    pub threshold_policy: ThresholdPolicy,
-    pub region_multiplier: Option<f64>,
-    pub billed_at_utc_minute: Option<u16>,
-}
-
-#[derive(Clone, Copy, Debug, PartialEq)]
-pub struct Cost {
-    pub uncached_input: f64,
-    pub cache_read: f64,
-    pub cache_write_5m: f64,
-    pub cache_write_1h: f64,
-    pub output: f64,
-    pub multiplier: f64,
-    pub rates: EffectiveRates,
-}
-
-impl Cost {
-    pub fn input(self) -> f64 {
-        (self.uncached_input + self.cache_read + self.cache_write_5m + self.cache_write_1h)
-            * self.multiplier
-    }
-
-    pub fn output(self) -> f64 {
-        self.output * self.multiplier
-    }
-
-    pub fn total(self) -> f64 {
-        self.input() + self.output()
-    }
-}
-
-#[derive(Clone, Copy, Debug, PartialEq)]
-pub struct EffectiveRates {
-    pub input: f64,
-    pub output: f64,
-    pub cache_read: f64,
-    pub cache_write_5m: f64,
-    pub cache_write_1h: f64,
-}
-
-fn selected_tier(tier: ServiceTier) -> ServiceTier {
-    if tier == ServiceTier::Fast {
-        ServiceTier::Priority
-    } else {
-        tier
-    }
-}
-
-#[derive(Clone, Debug)]
-struct CompiledThreshold {
-    above_prompt_tokens: u64,
-    standard: Rates,
-    tiers: Vec<TierRates>,
-}
-
-#[derive(Clone, Debug)]
-pub struct PricingPlan {
-    standard: Rates,
-    tiers: Vec<TierRates>,
-    thresholds: Vec<CompiledThreshold>,
-    off_peak: Option<OffPeakRates>,
-}
-
-fn valid_rates(rates: Rates) -> bool {
-    [
-        rates.input,
-        rates.output,
-        rates.cache_read,
-        rates.cache_write,
-        rates.cache_write_1h,
-    ]
-    .into_iter()
-    .all(|rate| {
-        rate.value()
-            .is_none_or(|value| value.is_finite() && value >= 0.0)
-    })
-}
-
-fn validate_tiers(tiers: &[TierRates], duplicate: Error) -> Result<(), Error> {
-    if tiers.iter().any(|entry| !valid_rates(entry.rates)) {
-        return Err(Error::InvalidRate);
-    }
-    if tiers.iter().enumerate().any(|(index, entry)| {
-        matches!(
-            entry.tier,
-            ServiceTier::Standard | ServiceTier::Unknown | ServiceTier::Fast
-        ) || tiers[..index]
-            .iter()
-            .any(|previous| previous.tier == entry.tier)
-    }) {
-        return Err(duplicate);
-    }
-    Ok(())
-}
-
-pub fn compile(pricing: &Pricing<'_>) -> Result<PricingPlan, Error> {
-    if !valid_rates(pricing.standard) {
-        return Err(Error::InvalidRate);
-    }
-    validate_tiers(pricing.tiers, Error::DuplicateTier)?;
-    if let Some(window) = pricing.off_peak {
-        if window.start_utc_minute >= 1440
-            || window.end_utc_minute > 1440
-            || window.start_utc_minute >= window.end_utc_minute
-        {
-            return Err(Error::InvalidOffPeakWindow);
-        }
-        if !valid_rates(window.rates) {
-            return Err(Error::InvalidRate);
-        }
-    }
-    let mut thresholds: Vec<_> = pricing
-        .thresholds
-        .iter()
-        .map(|entry| {
-            if !valid_rates(entry.standard) {
-                return Err(Error::InvalidRate);
-            }
-            validate_tiers(entry.tiers, Error::DuplicateThresholdTier)?;
-            Ok(CompiledThreshold {
-                above_prompt_tokens: entry.above_prompt_tokens,
-                standard: entry.standard,
-                tiers: entry.tiers.to_vec(),
-            })
-        })
-        .collect::<Result<_, _>>()?;
-    thresholds.sort_unstable_by_key(|entry| entry.above_prompt_tokens);
-    if thresholds
-        .windows(2)
-        .any(|pair| pair[0].above_prompt_tokens == pair[1].above_prompt_tokens)
-    {
-        return Err(Error::DuplicateThreshold);
-    }
-    Ok(PricingPlan {
-        standard: pricing.standard,
-        tiers: pricing.tiers.to_vec(),
-        thresholds,
-        off_peak: pricing.off_peak,
-    })
-}
-
-impl PricingPlan {
-    fn resolve_rates(&self, request: &Request, threshold_tokens: u64) -> Result<Rates, Error> {
-        let tier = selected_tier(request.service_tier);
-        let base = self
-            .tiers
-            .iter()
-            .find(|entry| tier != ServiceTier::Standard && entry.tier == tier)
-            .map_or(self.standard, |entry| entry.rates.overlay(self.standard));
-        let threshold = self.thresholds.iter().rev().find(|entry| {
-            threshold_tokens > entry.above_prompt_tokens
-                || (request.threshold_policy == ThresholdPolicy::Inclusive
-                    && threshold_tokens == entry.above_prompt_tokens)
-        });
-        let selected = threshold.map_or(base, |entry| {
-            let standard = entry.standard.overlay(base);
-            entry
-                .tiers
-                .iter()
-                .find(|specific| tier != ServiceTier::Standard && specific.tier == tier)
-                .map_or(standard, |specific| specific.rates.overlay(standard))
-        });
-        match self.off_peak {
-            None => Ok(selected),
-            Some(window) => {
-                if window.start_utc_minute >= 1440
-                    || window.end_utc_minute > 1440
-                    || window.start_utc_minute >= window.end_utc_minute
-                {
-                    return Err(Error::InvalidOffPeakWindow);
-                }
-                let minute = request
-                    .billed_at_utc_minute
-                    .ok_or(Error::InvalidBillingTime)?;
-                if minute >= 1440 {
-                    return Err(Error::InvalidBillingTime);
-                }
-                if (window.start_utc_minute..window.end_utc_minute).contains(&minute) {
-                    Ok(window.rates.overlay(selected))
-                } else {
-                    Ok(selected)
-                }
-            }
-        }
-    }
-
-    fn checked_rate(rate: Rate, missing: Error) -> Result<f64, Error> {
-        let value = rate.value().ok_or(missing)?;
-        if !value.is_finite() || value < 0.0 {
-            return Err(Error::InvalidRate);
-        }
-        Ok(value)
-    }
-
-    pub fn calculate(&self, request: &Request) -> Result<Cost, Error> {
-        let usage = request.usage;
-        let cached = usage
-            .cache_read_tokens
-            .checked_add(usage.cache_write_tokens)
-            .ok_or(Error::TokenCountOverflow)?;
-        let (regular, threshold_tokens) = match usage.prompt_convention {
-            PromptConvention::IncludesCache => (
-                usage
-                    .prompt_tokens
-                    .checked_sub(cached)
-                    .ok_or(Error::CacheExceedsPrompt)?,
-                usage.prompt_tokens,
-            ),
-            PromptConvention::ExcludesCache => (
-                usage.prompt_tokens,
-                usage
-                    .prompt_tokens
-                    .checked_add(cached)
-                    .ok_or(Error::TokenCountOverflow)?,
-            ),
-        };
-        let writes = match (usage.cache_write_5m_tokens, usage.cache_write_1h_tokens) {
-            (None, None) => (usage.cache_write_tokens, 0),
-            (Some(five), Some(one)) if five.checked_add(one) == Some(usage.cache_write_tokens) => {
-                (five, one)
-            }
-            _ => return Err(Error::InvalidCacheWriteDetails),
-        };
-        let rates = self.resolve_rates(request, threshold_tokens)?;
-        let input = Self::checked_rate(rates.input, Error::MissingInputRate)?;
-        let output = Self::checked_rate(rates.output, Error::MissingOutputRate)?;
-        let read = Self::checked_rate(rates.cache_read.or(rates.input), Error::MissingInputRate)?;
-        let write = Self::checked_rate(rates.cache_write.or(rates.input), Error::MissingInputRate)?;
-        let write_1h = Self::checked_rate(
-            rates.cache_write_1h.or(rates.cache_write).or(rates.input),
-            Error::MissingInputRate,
-        )?;
-        let multiplier = request.region_multiplier.unwrap_or(1.0);
-        if !multiplier.is_finite() || multiplier <= 0.0 {
-            return Err(Error::InvalidRegionMultiplier);
-        }
-        let cost = Cost {
-            uncached_input: regular as f64 * input,
-            cache_read: usage.cache_read_tokens as f64 * read,
-            cache_write_5m: writes.0 as f64 * write,
-            cache_write_1h: writes.1 as f64 * write_1h,
-            output: usage.completion_tokens as f64 * output,
-            multiplier,
-            rates: EffectiveRates {
-                input,
-                output,
-                cache_read: read,
-                cache_write_5m: write,
-                cache_write_1h: write_1h,
-            },
-        };
-        if !cost.total().is_finite() {
-            return Err(Error::TokenCountOverflow);
-        }
-        Ok(cost)
-    }
-}
-
-pub fn calculate(pricing: &Pricing<'_>, request: &Request) -> Result<Cost, Error> {
-    compile(pricing)?.calculate(request)
-}
+pub use error::*;

litellm-rust/crates/cost/src/money.rs

@@ -0,0 +1,257 @@
+use std::iter::Sum;
+use std::ops::{Add, Mul};
+
+use crate::priced::Quantity;
+
+#[derive(Clone, Copy, Debug, PartialEq, PartialOrd)]
+pub struct UnitPrice(f64);
+
+#[derive(Clone, Copy, Debug, Default, PartialEq, PartialOrd)]
+pub struct Usd(f64);
+
+#[derive(Clone, Copy, Debug, PartialEq, PartialOrd)]
+pub struct Multiplier(f64);
+
+impl UnitPrice {
+    pub fn new(usd_per_unit: f64) -> Option<Self> {
+        (usd_per_unit.is_finite() && usd_per_unit >= 0.0).then_some(Self(usd_per_unit))
+    }
+    pub fn get(self) -> f64 {
+        self.0
+    }
+}
+
+impl Usd {
+    pub const ZERO: Self = Self(0.0);
+
+    pub fn new(amount: f64) -> Option<Self> {
+        (amount.is_finite() && amount >= 0.0).then_some(Self(amount))
+    }
+    pub fn get(self) -> f64 {
+        self.0
+    }
+}
+
+impl Multiplier {
+    pub fn new(factor: f64) -> Option<Self> {
+        (factor.is_finite() && factor >= 1.0).then_some(Self(factor))
+    }
+    pub fn get(self) -> f64 {
+        self.0
+    }
+}
+
+impl Mul<Quantity> for UnitPrice {
+    type Output = Usd;
+
+    fn mul(self, quantity: Quantity) -> Usd {
+        match quantity {
+            Quantity::Count(units) => Usd(self.0 * units as f64),
+            Quantity::Duration(duration) => Usd(self.0 * duration.as_secs_f64()),
+        }
+    }
+}
+
+impl Mul<Multiplier> for UnitPrice {
+    type Output = UnitPrice;
+
+    fn mul(self, factor: Multiplier) -> UnitPrice {
+        UnitPrice(self.0 * factor.0)
+    }
+}
+
+impl Mul<Multiplier> for Usd {
+    type Output = Usd;
+
+    fn mul(self, factor: Multiplier) -> Usd {
+        Usd(self.0 * factor.0)
+    }
+}
+
+impl Add for Usd {
+    type Output = Usd;
+
+    fn add(self, other: Usd) -> Usd {
+        Usd(self.0 + other.0)
+    }
+}
+
+impl Sum for Usd {
+    fn sum<I: Iterator<Item = Usd>>(amounts: I) -> Usd {
+        amounts.fold(Usd::ZERO, Add::add)
+    }
+}
+
+#[cfg(feature = "serde")]
+mod serde_impls {
+    use std::fmt;
+
+    use serde::de::{self, Deserialize, Deserializer, Visitor};
+
+    use super::{Multiplier, UnitPrice};
+
+    #[derive(Clone, Copy)]
+    struct Number(&'static str);
+
+    impl Visitor<'_> for Number {
+        type Value = f64;
+
+        fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
+            formatter.write_str(self.0)
+        }
+
+        fn visit_i64<E: de::Error>(self, value: i64) -> Result<f64, E> {
+            Ok(value as f64)
+        }
+
+        fn visit_u64<E: de::Error>(self, value: u64) -> Result<f64, E> {
+            Ok(value as f64)
+        }
+
+        fn visit_f64<E: de::Error>(self, value: f64) -> Result<f64, E> {
+            Ok(value)
+        }
+
+        fn visit_str<E: de::Error>(self, value: &str) -> Result<f64, E> {
+            value
+                .trim()
+                .parse()
+                .map_err(|_| E::invalid_value(de::Unexpected::Str(value), &self))
+        }
+    }
+
+    fn checked<'de, D: Deserializer<'de>, T>(
+        deserializer: D,
+        expected: Number,
+        new: fn(f64) -> Option<T>,
+    ) -> Result<T, D::Error> {
+        let number = deserializer.deserialize_any(expected)?;
+        new(number)
+            .ok_or_else(|| de::Error::invalid_value(de::Unexpected::Float(number), &expected))
+    }
+
+    impl<'de> Deserialize<'de> for UnitPrice {
+        fn deserialize<D: Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
+            checked(deserializer, Number("a non-negative USD amount"), Self::new)
+        }
+    }
+
+    impl<'de> Deserialize<'de> for Multiplier {
+        fn deserialize<D: Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
+            checked(
+                deserializer,
+                Number("a multiplier of at least 1"),
+                Self::new,
+            )
+        }
+    }
+}
+
+#[cfg(feature = "schema")]
+mod schema {
+    use std::borrow::Cow;
+
+    use schemars::{JsonSchema, Schema, SchemaGenerator, json_schema};
+
+    use super::{Multiplier, UnitPrice};
+
+    impl JsonSchema for UnitPrice {
+        fn inline_schema() -> bool {
+            true
+        }
+        fn schema_name() -> Cow<'static, str> {
+            "UnitPrice".into()
+        }
+        fn json_schema(_: &mut SchemaGenerator) -> Schema {
+            json_schema!({"type": "number", "minimum": 0})
+        }
+    }
+
+    impl JsonSchema for Multiplier {
+        fn inline_schema() -> bool {
+            true
+        }
+        fn schema_name() -> Cow<'static, str> {
+            "Multiplier".into()
+        }
+        fn json_schema(_: &mut SchemaGenerator) -> Schema {
+            json_schema!({"type": "number", "minimum": 1})
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use rstest::rstest;
+
+    use super::*;
+
+    #[rstest]
+    #[case::price(0.5)]
+    #[case::free(0.0)]
+    fn unit_price_keeps_finite_non_negative_values(#[case] value: f64) {
+        assert_eq!(UnitPrice::new(value).map(UnitPrice::get), Some(value));
+    }
+
+    #[rstest]
+    #[case::negative(-1e-9)]
+    #[case::nan(f64::NAN)]
+    #[case::infinite(f64::INFINITY)]
+    fn unit_price_rejects_what_no_price_can_be(#[case] value: f64) {
+        assert_eq!(UnitPrice::new(value), None);
+    }
+
+    #[rstest]
+    #[case::uplift(1.1, true)]
+    #[case::neutral(1.0, true)]
+    #[case::discount(0.5, false)]
+    #[case::nan(f64::NAN, false)]
+    fn multiplier_is_never_a_discount(#[case] value: f64, #[case] accepted: bool) {
+        assert_eq!(Multiplier::new(value).is_some(), accepted);
+    }
+
+    #[rstest]
+    fn amounts_follow_units_times_price_then_uplift() {
+        let price = UnitPrice::new(0.25).unwrap();
+        let uplift = Multiplier::new(2.0).unwrap();
+
+        let total: Usd = [
+            price * Quantity::Count(4),
+            (price * uplift) * Quantity::Count(2),
+        ]
+        .into_iter()
+        .sum();
+
+        assert_eq!(total.get(), 2.0);
+        assert_eq!((total * uplift).get(), 4.0);
+    }
+
+    #[rstest]
+    fn a_duration_is_billed_in_fractional_seconds() {
+        let per_second = UnitPrice::new(0.01).unwrap();
+
+        let amount = per_second * Quantity::Duration(std::time::Duration::from_millis(2_500));
+
+        assert!((amount.get() - 0.025).abs() < 1e-12);
+    }
+
+    #[cfg(feature = "serde")]
+    #[rstest]
+    #[case::number(serde_json::json!(1.5e-6), Some(1.5e-6))]
+    #[case::integer(serde_json::json!(2), Some(2.0))]
+    #[case::numeric_string(serde_json::json!(" 0.000002 "), Some(2e-6))]
+    #[case::negative(serde_json::json!(-1), None)]
+    #[case::word(serde_json::json!("free"), None)]
+    #[case::boolean(serde_json::json!(true), None)]
+    fn unit_price_reads_numbers_and_numeric_strings(
+        #[case] written: serde_json::Value,
+        #[case] expected: Option<f64>,
+    ) {
+        use serde::Deserialize;
+
+        assert_eq!(
+            UnitPrice::deserialize(&written).ok().map(UnitPrice::get),
+            expected
+        );
+    }
+}

litellm-rust/crates/cost/src/price.rs

@@ -0,0 +1,505 @@
+mod meter;
+
+use std::collections::BTreeMap;
+use std::num::NonZeroU64;
+
+pub use meter::*;
+
+use crate::call::{InferenceGeo, Speed};
+use crate::money::{Multiplier, UnitPrice};
+use crate::schedule::Schedule;
+
+#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, PartialOrd, Ord, Hash)]
+pub enum ServiceTier {
+    #[default]
+    Standard,
+    Priority,
+    Flex,
+    Ultrafast,
+    Balanced,
+}
+
+#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, PartialOrd, Ord, Hash)]
+pub enum Band {
+    #[default]
+    Base,
+    OverPromptTokens(NonZeroU64),
+}
+
+#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, PartialOrd, Ord, Hash)]
+pub struct When {
+    pub tier: ServiceTier,
+    pub band: Band,
+}
+
+#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, PartialOrd, Ord, Hash)]
+pub enum Channel {
+    #[default]
+    Online,
+    Batch,
+    Decisions,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
+pub struct Price {
+    pub meter: Meter,
+    pub channel: Channel,
+    pub when: When,
+}
+
+#[derive(Clone, Debug, Default, PartialEq)]
+pub struct Prices {
+    rates: BTreeMap<Price, UnitPrice>,
+    off_peak: Option<OffPeak>,
+    billing: Billing,
+}
+
+/// Row attributes pricing reads, lowered by model-catalog from the row and its provider
+#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
+pub struct Billing {
+    /// Unit the row bills text in. `Character` only when the row prices characters and not tokens
+    pub text_unit: TextUnit,
+    /// Whether a band's floor is in the band. xAI rows bill the band from the floor itself
+    pub band_edge: BandEdge,
+    /// Rate a cache read borrows when the row has no cache-read price. Fireworks rows borrow half the input rate
+    pub cache_read_fallback: CacheReadFallback,
+    /// Whether a row with no token price is priced by wall clock, from its `mode`
+    pub wall_clock: WallClock,
+    /// From `web_search_billing_unit`, whether web search and Maps grounding bill per query or once per prompt
+    pub web_search: WebSearchUnit,
+    /// Whether `tiered_pricing` bills each meter in its own tier or the whole call at the tier the input picks.
+    /// Dashscope rows bill the whole call (inferred, 10 checks no other row has that shape)
+    pub tier_scope: TierScope,
+}
+
+#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
+pub enum TextUnit {
+    #[default]
+    Token,
+    Character,
+}
+
+#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
+pub enum BandEdge {
+    #[default]
+    Exclusive,
+    Inclusive,
+}
+
+#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
+pub enum CacheReadFallback {
+    #[default]
+    InputRate,
+    HalfInputRate,
+}
+
+/// `ResponseTime` for `chat`, `completion`, `embedding`, `responses` or no mode, `AudioFile` for
+/// `audio_transcription`, which prices the input audio when it has one, `Never` for every other mode
+#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
+pub enum WallClock {
+    Never,
+    #[default]
+    ResponseTime,
+    AudioFile,
+}
+
+#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
+pub enum WebSearchUnit {
+    PerQuery,
+    #[default]
+    PerPrompt,
+}
+
+#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
+pub enum TierScope {
+    #[default]
+    PerMeter,
+    WholeCall,
+}
+
+#[derive(Clone, Debug, PartialEq)]
+pub struct OffPeak {
+    schedule: Schedule,
+    rates: OffPeakRates,
+}
+
+#[derive(Clone, Copy, Debug, Default, PartialEq)]
+pub struct OffPeakRates {
+    pub input: Option<UnitPrice>,
+    pub output: Option<UnitPrice>,
+    pub cache_read: Option<UnitPrice>,
+    pub explicit_cache_read: Option<UnitPrice>,
+    pub cache_write: Option<UnitPrice>,
+    pub reasoning: Option<UnitPrice>,
+}
+
+#[derive(Clone, Debug, Default, PartialEq)]
+pub struct Adjustments(BTreeMap<Uplift, Multiplier>);
+
+/// Which call fact picks which multiplier in `Adjustments`, lowered by model-catalog from the row
+#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
+pub enum Uplift {
+    /// OpenAI data residency
+    Residency(Residency),
+    /// Vertex non-global endpoint, from `regional_endpoint_uplift_multiplier`
+    NonGlobalEndpoint,
+    /// Anthropic `speed: fast`, from `provider_specific_entry.fast`
+    Speed(Speed),
+    /// Anthropic and Bedrock geo, from `provider_specific_entry.<geo>`, lowercased as on main
+    InferenceGeo(InferenceGeo),
+}
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
+pub enum Residency {
+    Eu,
+    Us,
+}
+
+impl Band {
+    pub fn over_prompt_tokens(tokens: u64) -> Self {
+        NonZeroU64::new(tokens).map_or(Self::Base, Self::OverPromptTokens)
+    }
+}
+
+impl Price {
+    pub fn online(meter: Meter, when: When) -> Self {
+        Self {
+            meter,
+            channel: Channel::Online,
+            when,
+        }
+    }
+}
+
+impl Prices {
+    pub fn new(
+        rates: BTreeMap<Price, UnitPrice>,
+        off_peak: Option<OffPeak>,
+        billing: Billing,
+    ) -> Self {
+        Self {
+            rates,
+            off_peak,
+            billing,
+        }
+    }
+
+    pub fn get(&self, price: &Price) -> Option<UnitPrice> {
+        self.rates.get(price).copied()
+    }
+
+    pub fn iter(&self) -> impl Iterator<Item = (&Price, UnitPrice)> {
+        self.rates.iter().map(|(price, rate)| (price, *rate))
+    }
+
+    pub fn off_peak(&self) -> Option<&OffPeak> {
+        self.off_peak.as_ref()
+    }
+
+    pub fn billing(&self) -> Billing {
+        self.billing
+    }
+
+    /// The online prompt band a request falls in. Every token meter's `_above_` prices count,
+    /// and the highest band the prompt passes wins. `billing.band_edge` says whether the floor is in the band.
+    pub fn band(&self, tier: ServiceTier, prompt_tokens: u64) -> Band {
+        let inclusive = self.billing.band_edge == BandEdge::Inclusive;
+        self.rates
+            .keys()
+            .filter(|price| {
+                price.channel == Channel::Online
+                    && price.meter.sets_prompt_band()
+                    && (price.when.tier == ServiceTier::Standard || price.when.tier == tier)
+            })
+            .filter_map(|price| match price.when.band {
+                Band::OverPromptTokens(floor) => Some(floor.get()),
+                Band::Base => None,
+            })
+            .filter(|floor| prompt_tokens > *floor || (inclusive && prompt_tokens == *floor))
+            .max()
+            .map_or(Band::Base, Band::over_prompt_tokens)
+    }
+
+    /// The online rate for `meter`, tried at `(tier, band)`, `(standard, band)`, `(tier, base)`
+    /// and `(standard, base)` in that order. A missing cache price falls back to the input price
+    /// resolved the same way.
+    pub fn rate(&self, meter: &Meter, tier: ServiceTier, band: Band) -> Option<UnitPrice> {
+        [
+            When { tier, band },
+            When {
+                tier: ServiceTier::Standard,
+                band,
+            },
+            When {
+                tier,
+                band: Band::Base,
+            },
+            When::default(),
+        ]
+        .into_iter()
+        .find_map(|when| self.get(&Price::online(meter.clone(), when)))
+        .or_else(|| self.rate(&meter.fallback()?, tier, band))
+    }
+}
+
+impl OffPeak {
+    pub fn new(schedule: Schedule, rates: OffPeakRates) -> Option<Self> {
+        (rates != OffPeakRates::default()).then_some(Self { schedule, rates })
+    }
+
+    pub fn schedule(&self) -> &Schedule {
+        &self.schedule
+    }
+
+    pub fn rates(&self) -> &OffPeakRates {
+        &self.rates
+    }
+}
+
+impl Adjustments {
+    pub fn new(uplifts: impl IntoIterator<Item = (Uplift, Multiplier)>) -> Self {
+        Self(uplifts.into_iter().collect())
+    }
+
+    pub fn get(&self, uplift: &Uplift) -> Option<Multiplier> {
+        self.0.get(uplift).copied()
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use rstest::{fixture, rstest};
+
+    use super::*;
+
+    const INPUT: Meter = Meter::TEXT_INPUT;
+    const OUTPUT: Meter = Meter::Output(OutputUnit::Token(TokenKind::Text));
+    const CACHE_READ: Meter = Meter::CacheRead {
+        kind: TokenKind::Text,
+        mode: CacheReadMode::Implicit,
+    };
+    const CACHE_WRITE_HOUR: Meter = Meter::CacheWrite {
+        kind: TokenKind::Text,
+        ttl: CacheTtl::OneHour,
+    };
+
+    fn over(tokens: u64) -> Band {
+        Band::over_prompt_tokens(tokens)
+    }
+
+    fn prices(rates: &[(Meter, Channel, ServiceTier, Band, f64)]) -> Prices {
+        priced_with(rates, Billing::default())
+    }
+
+    fn priced_with(rates: &[(Meter, Channel, ServiceTier, Band, f64)], billing: Billing) -> Prices {
+        Prices::new(
+            rates
+                .iter()
+                .map(|(meter, channel, tier, band, rate)| {
+                    (
+                        Price {
+                            meter: meter.clone(),
+                            channel: *channel,
+                            when: When {
+                                tier: *tier,
+                                band: *band,
+                            },
+                        },
+                        UnitPrice::new(*rate).unwrap(),
+                    )
+                })
+                .collect(),
+            None,
+            billing,
+        )
+    }
+
+    #[fixture]
+    fn priority_with_a_standard_band() -> Prices {
+        use Channel::Online;
+        use ServiceTier::{Priority, Standard};
+        prices(&[
+            (INPUT, Online, Standard, Band::Base, 1.0),
+            (INPUT, Online, Priority, Band::Base, 2.0),
+            (INPUT, Online, Standard, over(200_000), 10.0),
+            (OUTPUT, Online, Standard, Band::Base, 3.0),
+            (OUTPUT, Online, Priority, over(200_000), 30.0),
+        ])
+    }
+
+    #[rstest]
+    #[case::below(1_000, false, Band::Base)]
+    #[case::at_floor_exclusive(200_000, false, Band::Base)]
+    #[case::at_floor_inclusive(200_000, true, over(200_000))]
+    #[case::above(200_001, false, over(200_000))]
+    fn the_band_starts_strictly_above_its_floor_unless_inclusive(
+        #[case] prompt_tokens: u64,
+        #[case] inclusive: bool,
+        #[case] expected: Band,
+    ) {
+        let band_edge = if inclusive {
+            BandEdge::Inclusive
+        } else {
+            BandEdge::Exclusive
+        };
+        let prices = priced_with(
+            &[(
+                INPUT,
+                Channel::Online,
+                ServiceTier::Standard,
+                over(200_000),
+                10.0,
+            )],
+            Billing {
+                band_edge,
+                ..Billing::default()
+            },
+        );
+        assert_eq!(prices.band(ServiceTier::Standard, prompt_tokens), expected);
+    }
+
+    #[rstest]
+    fn a_band_only_another_tier_prices_does_not_apply() {
+        let prices = prices(&[(
+            OUTPUT,
+            Channel::Online,
+            ServiceTier::Flex,
+            over(100_000),
+            1.0,
+        )]);
+        assert_eq!(prices.band(ServiceTier::Standard, 300_000), Band::Base);
+        assert_eq!(prices.band(ServiceTier::Flex, 300_000), over(100_000));
+    }
+
+    #[rstest]
+    fn batch_bands_never_pick_the_online_band() {
+        let prices = prices(&[(
+            INPUT,
+            Channel::Batch,
+            ServiceTier::Standard,
+            over(100_000),
+            1.0,
+        )]);
+        assert_eq!(prices.band(ServiceTier::Standard, 300_000), Band::Base);
+    }
+
+    #[rstest]
+    fn the_highest_band_passed_wins() {
+        let prices = prices(&[
+            (
+                INPUT,
+                Channel::Online,
+                ServiceTier::Standard,
+                over(100_000),
+                1.0,
+            ),
+            (
+                OUTPUT,
+                Channel::Online,
+                ServiceTier::Standard,
+                over(200_000),
+                1.0,
+            ),
+        ]);
+        assert_eq!(prices.band(ServiceTier::Standard, 300_000), over(200_000));
+    }
+
+    #[rstest]
+    #[case::tier_base(INPUT, ServiceTier::Priority, Band::Base, Some(2.0))]
+    #[case::standard_band_beats_tier_base(INPUT, ServiceTier::Priority, over(200_000), Some(10.0))]
+    #[case::tier_band(OUTPUT, ServiceTier::Priority, over(200_000), Some(30.0))]
+    #[case::standard_base_when_nothing_else(OUTPUT, ServiceTier::Flex, over(200_000), Some(3.0))]
+    #[case::cache_read_falls_back_to_the_resolved_input(
+        CACHE_READ,
+        ServiceTier::Priority,
+        over(200_000),
+        Some(10.0)
+    )]
+    #[case::one_hour_write_falls_back_through_write_to_input(
+        CACHE_WRITE_HOUR,
+        ServiceTier::Priority,
+        Band::Base,
+        Some(2.0)
+    )]
+    #[case::twelve_hour_write_falls_back_through_write_to_input(
+        Meter::CacheWrite { kind: TokenKind::Text, ttl: CacheTtl::TwelveHours },
+        ServiceTier::Standard,
+        Band::Base,
+        Some(1.0)
+    )]
+    #[case::no_fallback_for_other_meters(
+        Meter::Input(InputUnit::Token(TokenKind::Audio)),
+        ServiceTier::Standard,
+        Band::Base,
+        None
+    )]
+    fn rate_walks_tier_then_band(
+        priority_with_a_standard_band: Prices,
+        #[case] meter: Meter,
+        #[case] tier: ServiceTier,
+        #[case] band: Band,
+        #[case] expected: Option<f64>,
+    ) {
+        assert_eq!(
+            priority_with_a_standard_band
+                .rate(&meter, tier, band)
+                .map(UnitPrice::get),
+            expected
+        );
+    }
+
+    #[rstest]
+    fn an_explicit_cache_price_wins_over_the_input_fallback() {
+        let prices = prices(&[
+            (
+                INPUT,
+                Channel::Online,
+                ServiceTier::Standard,
+                Band::Base,
+                1.0,
+            ),
+            (
+                CACHE_READ,
+                Channel::Online,
+                ServiceTier::Standard,
+                Band::Base,
+                0.0,
+            ),
+        ]);
+        assert_eq!(
+            prices
+                .rate(&CACHE_READ, ServiceTier::Standard, Band::Base)
+                .map(UnitPrice::get),
+            Some(0.0)
+        );
+    }
+
+    #[rstest]
+    fn off_peak_needs_a_price() {
+        let schedule = Schedule {
+            windows: Vec::new(),
+            weekday_timezone: None,
+        };
+        assert_eq!(
+            OffPeak::new(schedule.clone(), OffPeakRates::default()),
+            None
+        );
+        assert!(
+            OffPeak::new(
+                schedule,
+                OffPeakRates {
+                    output: UnitPrice::new(1.0),
+                    ..OffPeakRates::default()
+                }
+            )
+            .is_some()
+        );
+    }
+
+    #[rstest]
+    #[case::zero(0, Band::Base)]
+    #[case::positive(272_000, Band::OverPromptTokens(NonZeroU64::new(272_000).unwrap()))]
+    fn a_zero_floor_is_the_base_band(#[case] tokens: u64, #[case] expected: Band) {
+        assert_eq!(Band::over_prompt_tokens(tokens), expected);
+    }
+}

litellm-rust/crates/cost/src/price/meter.rs

@@ -0,0 +1,197 @@
+use std::num::NonZeroU64;
+
+#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
+pub enum Meter {
+    Input(InputUnit),
+    Output(OutputUnit),
+    CacheRead {
+        kind: TokenKind,
+        mode: CacheReadMode,
+    },
+    CacheWrite {
+        kind: TokenKind,
+        ttl: CacheTtl,
+    },
+    Feature(Feature),
+    WallClockSecond,
+    WebSearch(SearchContextSize),
+    Guardrail(GuardrailUnit),
+}
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
+pub enum TokenKind {
+    Text,
+    Audio,
+    Image,
+    Video,
+}
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
+pub enum InputUnit {
+    Token(TokenKind),
+    Character,
+    Image,
+    Pixel,
+    Query { over_results: Option<NonZeroU64> },
+    Request,
+    Second,
+    TextMessage,
+    AudioSecond,
+    VideoSecond { over_seconds: Option<NonZeroU64> },
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
+pub enum OutputUnit {
+    Token(TokenKind),
+    ReasoningToken,
+    Character,
+    Image(Option<ImageSize>),
+    Pixel,
+    Second(Option<Resolution>),
+    VideoSecond,
+}
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
+pub enum CacheReadMode {
+    Implicit,
+    Explicit,
+}
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
+pub enum CacheTtl {
+    Default,
+    OneHour,
+    TwelveHours,
+}
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
+pub enum Feature {
+    AnnotationPage,
+    CitationToken,
+    CodeInterpreterSession,
+    GoogleMapsGroundingQuery,
+    OcrCredit,
+    OcrPage,
+    /// `browser_open_cost_per_request`, Groq `browser.open`
+    BrowserOpen,
+    /// `cost_per_additional_result`, Parallel AI results past the ones the query price includes
+    AdditionalResult,
+}
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
+pub enum SearchContextSize {
+    Low,
+    Medium,
+    High,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
+pub struct GuardrailUnit(Box<str>);
+
+#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
+pub struct ImageSize(Box<str>);
+
+#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
+pub struct Resolution(Box<str>);
+
+impl Meter {
+    pub(crate) const TEXT_INPUT: Self = Self::Input(InputUnit::Token(TokenKind::Text));
+
+    pub(crate) fn sets_prompt_band(&self) -> bool {
+        matches!(
+            self,
+            Self::Input(InputUnit::Token(TokenKind::Text))
+                | Self::Output(OutputUnit::Token(TokenKind::Text))
+                | Self::CacheRead {
+                    kind: TokenKind::Text,
+                    mode: CacheReadMode::Implicit,
+                }
+                | Self::CacheWrite {
+                    kind: TokenKind::Text,
+                    ..
+                }
+        )
+    }
+
+    pub(crate) fn fallback(&self) -> Option<Self> {
+        match self {
+            Self::CacheRead {
+                kind: TokenKind::Text,
+                mode: CacheReadMode::Implicit,
+            }
+            | Self::CacheWrite {
+                kind: TokenKind::Text,
+                ttl: CacheTtl::Default,
+            } => Some(Self::TEXT_INPUT),
+            Self::CacheWrite {
+                kind: TokenKind::Text,
+                ttl: CacheTtl::OneHour | CacheTtl::TwelveHours,
+            } => Some(Self::CacheWrite {
+                kind: TokenKind::Text,
+                ttl: CacheTtl::Default,
+            }),
+            _ => None,
+        }
+    }
+}
+
+impl GuardrailUnit {
+    pub fn new(counter: &str) -> Self {
+        Self(counter.into())
+    }
+    pub fn as_str(&self) -> &str {
+        &self.0
+    }
+}
+
+impl ImageSize {
+    pub fn new(label: &str) -> Option<Self> {
+        let number = label.strip_suffix('K').unwrap_or(label);
+        let (whole, fraction) = number.split_once('.').unwrap_or((number, "0"));
+        (is_digits(whole) && is_digits(fraction)).then(|| Self(label.into()))
+    }
+    pub fn as_str(&self) -> &str {
+        &self.0
+    }
+}
+
+impl Resolution {
+    pub fn new(label: &str) -> Option<Self> {
+        let digits = label.strip_suffix(['p', 'k'])?;
+        is_digits(digits).then(|| Self(label.into()))
+    }
+    pub fn as_str(&self) -> &str {
+        &self.0
+    }
+}
+
+fn is_digits(value: &str) -> bool {
+    !value.is_empty() && value.bytes().all(|byte| byte.is_ascii_digit())
+}
+
+#[cfg(test)]
+mod tests {
+    use rstest::rstest;
+
+    use super::*;
+
+    #[rstest]
+    #[case::kilo("1K", true)]
+    #[case::half_kilo("0.5K", true)]
+    #[case::pixels("1024", true)]
+    #[case::word("big", false)]
+    #[case::bare_suffix("K", false)]
+    #[case::trailing_dot("1.K", false)]
+    fn image_size_labels_are_numbers_with_an_optional_k(#[case] label: &str, #[case] valid: bool) {
+        assert_eq!(ImageSize::new(label).is_some(), valid);
+    }
+
+    #[rstest]
+    #[case::progressive("720p", true)]
+    #[case::kilo("4k", true)]
+    #[case::bare("720", false)]
+    #[case::word("hd", false)]
+    fn resolution_labels_are_numbers_with_p_or_k(#[case] label: &str, #[case] valid: bool) {
+        assert_eq!(Resolution::new(label).is_some(), valid);
+    }
+}

litellm-rust/crates/cost/src/priced.rs

@@ -0,0 +1,53 @@
+use std::time::Duration;
+
+use crate::money::{UnitPrice, Usd};
+use crate::price::Meter;
+
+#[derive(Clone, Debug, PartialEq)]
+pub enum Outcome {
+    Priced(Priced),
+    /// No meter with usage had a price
+    Unpriced {
+        missing: Vec<Meter>,
+    },
+}
+
+#[derive(Clone, Debug, PartialEq)]
+pub struct Priced {
+    /// One line per billed meter, fallbacks included
+    pub lines: Vec<Line>,
+    /// Meters with usage and no price and no fallback
+    pub missing: Vec<Meter>,
+    /// Sum of `lines`
+    pub total: Usd,
+}
+
+#[derive(Clone, Debug, PartialEq)]
+pub struct Line {
+    /// The meter this line bills
+    pub meter: Meter,
+    /// How much of the meter the call used
+    pub quantity: Quantity,
+    /// Rate after off-peak and before uplifts
+    pub unit_price: UnitPrice,
+    /// `unit_price * quantity` times every uplift that applies
+    pub amount: Usd,
+    /// The meter whose rate was borrowed, when this meter had none
+    pub fallback_from: Option<Meter>,
+    /// Which half of main's `(prompt, completion)` pair the line belongs to
+    pub side: Side,
+}
+
+/// A billed amount of one meter
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub enum Quantity {
+    Count(u64),
+    Duration(Duration),
+}
+
+/// Meters main bills as one total (video, Replicate, Amazon Transcribe) take a fixed side, chosen in 09
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub enum Side {
+    Prompt,
+    Completion,
+}

litellm-rust/crates/cost/src/schedule.rs

@@ -0,0 +1,241 @@
+use std::str::FromStr;
+
+use crate::Error;
+
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct Schedule {
+    pub windows: Vec<Window>,
+    pub weekday_timezone: Option<String>,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct Window {
+    pub hours: Vec<UtcWindow>,
+    pub weekdays: Option<Vec<Weekday>>,
+}
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
+pub struct UtcWindow {
+    pub start_minute: u16,
+    pub end_minute: u16,
+}
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
+pub enum Weekday {
+    Monday,
+    Tuesday,
+    Wednesday,
+    Thursday,
+    Friday,
+    Saturday,
+    Sunday,
+}
+
+impl FromStr for UtcWindow {
+    type Err = Error;
+
+    fn from_str(window: &str) -> Result<Self, Self::Err> {
+        let (start, end) = window.split_once('-').ok_or(Error::InvalidUtcWindow)?;
+        Ok(Self {
+            start_minute: minute_of_day(start).ok_or(Error::InvalidUtcWindow)?,
+            end_minute: minute_of_day(end).ok_or(Error::InvalidUtcWindow)?,
+        })
+    }
+}
+
+fn minute_of_day(clock: &str) -> Option<u16> {
+    let (hour, minute) = clock.split_once(':')?;
+    if hour.len() != 2 || minute.len() != 2 {
+        return None;
+    }
+    let hour: u16 = hour.parse().ok().filter(|hour| *hour < 24)?;
+    let minute: u16 = minute.parse().ok().filter(|minute| *minute < 60)?;
+    Some(hour * 60 + minute)
+}
+
+impl Weekday {
+    pub fn from_iso(number: u8) -> Option<Self> {
+        match number {
+            1 => Some(Self::Monday),
+            2 => Some(Self::Tuesday),
+            3 => Some(Self::Wednesday),
+            4 => Some(Self::Thursday),
+            5 => Some(Self::Friday),
+            6 => Some(Self::Saturday),
+            7 => Some(Self::Sunday),
+            _ => None,
+        }
+    }
+}
+
+impl FromStr for Weekday {
+    type Err = Error;
+
+    fn from_str(name: &str) -> Result<Self, Self::Err> {
+        match name.trim().to_ascii_lowercase().as_str() {
+            "mon" | "monday" => Ok(Self::Monday),
+            "tue" | "tues" | "tuesday" => Ok(Self::Tuesday),
+            "wed" | "wednesday" => Ok(Self::Wednesday),
+            "thu" | "thur" | "thurs" | "thursday" => Ok(Self::Thursday),
+            "fri" | "friday" => Ok(Self::Friday),
+            "sat" | "saturday" => Ok(Self::Saturday),
+            "sun" | "sunday" => Ok(Self::Sunday),
+            _ => Err(Error::InvalidWeekday),
+        }
+    }
+}
+
+#[cfg(feature = "serde")]
+mod serde_impls {
+    use std::fmt;
+
+    use serde::de::{self, Deserialize, Deserializer, Visitor};
+
+    use super::{UtcWindow, Weekday};
+
+    impl<'de> Deserialize<'de> for UtcWindow {
+        fn deserialize<D: Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
+            let window = String::deserialize(deserializer)?;
+            window.parse().map_err(de::Error::custom)
+        }
+    }
+
+    struct WrittenWeekday;
+
+    impl Visitor<'_> for WrittenWeekday {
+        type Value = Weekday;
+
+        fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
+            formatter.write_str("an ISO weekday number 1-7 or an English day name")
+        }
+
+        fn visit_u64<E: de::Error>(self, number: u64) -> Result<Weekday, E> {
+            u8::try_from(number)
+                .ok()
+                .and_then(Weekday::from_iso)
+                .ok_or_else(|| E::invalid_value(de::Unexpected::Unsigned(number), &self))
+        }
+
+        fn visit_i64<E: de::Error>(self, number: i64) -> Result<Weekday, E> {
+            u64::try_from(number)
+                .map_err(|_| E::invalid_value(de::Unexpected::Signed(number), &self))
+                .and_then(|number| self.visit_u64(number))
+        }
+
+        fn visit_str<E: de::Error>(self, name: &str) -> Result<Weekday, E> {
+            name.parse()
+                .map_err(|_| E::invalid_value(de::Unexpected::Str(name), &self))
+        }
+    }
+
+    impl<'de> Deserialize<'de> for Weekday {
+        fn deserialize<D: Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
+            deserializer.deserialize_any(WrittenWeekday)
+        }
+    }
+}
+
+#[cfg(feature = "schema")]
+mod schema {
+    use std::borrow::Cow;
+
+    use schemars::{JsonSchema, Schema, SchemaGenerator, json_schema};
+
+    use super::{UtcWindow, Weekday};
+
+    impl JsonSchema for UtcWindow {
+        fn inline_schema() -> bool {
+            true
+        }
+        fn schema_name() -> Cow<'static, str> {
+            "UtcWindow".into()
+        }
+        fn json_schema(_: &mut SchemaGenerator) -> Schema {
+            json_schema!({
+                "type": "string",
+                "pattern": r"^([01]\d|2[0-3]):[0-5]\d-([01]\d|2[0-3]):[0-5]\d$"
+            })
+        }
+    }
+
+    impl JsonSchema for Weekday {
+        fn inline_schema() -> bool {
+            true
+        }
+        fn schema_name() -> Cow<'static, str> {
+            "Weekday".into()
+        }
+        fn json_schema(_: &mut SchemaGenerator) -> Schema {
+            json_schema!({
+                "anyOf": [
+                    {"type": "integer", "minimum": 1, "maximum": 7},
+                    {
+                        "type": "string",
+                        "pattern": r"(?i)^\s*(mon|monday|tue|tues|tuesday|wed|wednesday|thu|thur|thurs|thursday|fri|friday|sat|saturday|sun|sunday)\s*$"
+                    }
+                ]
+            })
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use rstest::rstest;
+
+    use super::*;
+
+    #[rstest]
+    #[case::same_day("10:00-18:30", Some((600, 1110)))]
+    #[case::wraps_midnight("22:00-02:00", Some((1320, 120)))]
+    #[case::whole_day("00:00-00:00", Some((0, 0)))]
+    #[case::hour_out_of_range("25:00-01:00", None)]
+    #[case::minute_out_of_range("10:60-11:00", None)]
+    #[case::one_digit_hour("9:00-10:00", None)]
+    #[case::no_separator("10:00", None)]
+    fn utc_window_reads_hh_mm_ranges(#[case] text: &str, #[case] expected: Option<(u16, u16)>) {
+        assert_eq!(
+            text.parse::<UtcWindow>()
+                .ok()
+                .map(|window| (window.start_minute, window.end_minute)),
+            expected
+        );
+    }
+
+    #[rstest]
+    #[case::short("thurs", Some(Weekday::Thursday))]
+    #[case::upper("SATURDAY", Some(Weekday::Saturday))]
+    #[case::padded(" mon ", Some(Weekday::Monday))]
+    #[case::unknown("someday", None)]
+    #[case::empty("", None)]
+    fn weekday_names_match_python_after_trimming(
+        #[case] name: &str,
+        #[case] expected: Option<Weekday>,
+    ) {
+        assert_eq!(name.parse().ok(), expected);
+    }
+
+    #[rstest]
+    #[case::monday(1, Some(Weekday::Monday))]
+    #[case::sunday(7, Some(Weekday::Sunday))]
+    #[case::zero(0, None)]
+    #[case::eight(8, None)]
+    fn weekday_numbers_are_iso(#[case] number: u8, #[case] expected: Option<Weekday>) {
+        assert_eq!(Weekday::from_iso(number), expected);
+    }
+
+    #[cfg(feature = "serde")]
+    #[rstest]
+    #[case::number(serde_json::json!(6), Some(Weekday::Saturday))]
+    #[case::name(serde_json::json!("Sun"), Some(Weekday::Sunday))]
+    #[case::negative(serde_json::json!(-1), None)]
+    #[case::boolean(serde_json::json!(true), None)]
+    fn weekday_reads_numbers_and_names(
+        #[case] written: serde_json::Value,
+        #[case] expected: Option<Weekday>,
+    ) {
+        use serde::Deserialize;
+
+        assert_eq!(Weekday::deserialize(&written).ok(), expected);
+    }
+}

litellm-rust/crates/cost/src/usage.rs

@@ -0,0 +1,104 @@
+use std::collections::BTreeMap;
+use std::time::Duration;
+
+use crate::price::{CacheReadMode, CacheTtl, ImageSize, Resolution, SearchContextSize, TokenKind};
+
+/// What one call consumed. Python fills it from the provider response, `calculate` never reads the catalog for it
+#[derive(Clone, Debug, Default, PartialEq, Eq)]
+pub struct Usage {
+    /// Prompt size as the provider reported it. Only picks `_above_Nk_tokens` bands, never billed.
+    /// Filled even when the call bills characters, `0` when the provider reports none
+    pub prompt_tokens: u64,
+    /// Billed token quantities by meter
+    pub tokens: Tokens,
+    /// Billed character quantities. Used only when the row's `billing.text_unit` is `Character`
+    pub characters: Characters,
+    /// Billed time per second meter, never a bare number of seconds or milliseconds
+    pub durations: BTreeMap<DurationMeter, Duration>,
+    /// Images sent and returned
+    pub images: Images,
+    /// Search queries, web searches and grounding requests
+    pub search: Search,
+    /// Realtime text messages, xAI bills them per message
+    pub text_messages: u64,
+}
+
+#[derive(Clone, Debug, Default, PartialEq, Eq)]
+pub struct Tokens {
+    /// Uncached prompt tokens by modality
+    pub input: BTreeMap<TokenKind, u64>,
+    /// Prompt tokens served from cache, by modality and implicit or explicit cache
+    pub cache_read: BTreeMap<(TokenKind, CacheReadMode), u64>,
+    /// Prompt tokens written to cache, by modality and cache lifetime
+    pub cache_write: BTreeMap<(TokenKind, CacheTtl), u64>,
+    /// Completion tokens by modality, reasoning excluded
+    pub output: BTreeMap<TokenKind, u64>,
+    /// Reasoning tokens, billed at the reasoning rate or the text output rate
+    pub reasoning: u64,
+    /// Perplexity citation tokens, prompt side
+    pub citation: u64,
+}
+
+#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
+pub struct Characters {
+    /// Characters of the formatted prompt
+    pub prompt: u64,
+    /// Characters of the response text
+    pub completion: u64,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
+pub enum DurationMeter {
+    /// Response time of the call, for rows priced by wall clock
+    WallClock,
+    /// Length of the input audio, a transcription file or a realtime transcription
+    InputAudio,
+    /// Realtime voice session time
+    LiveVoice,
+    /// Length of generated video, already multiplied by the video count. `resolution` picks `output_cost_per_second_<resolution>`
+    OutputVideo { resolution: Option<Resolution> },
+}
+
+#[derive(Clone, Debug, Default, PartialEq, Eq)]
+pub struct Images {
+    /// Images sent in the request
+    pub input: u64,
+    /// Pixels sent in the request, for rows priced per input pixel
+    pub input_pixels: u64,
+    /// One entry per returned image, since price can depend on each image's size
+    pub output: Vec<ImageOut>,
+}
+
+#[derive(Clone, Debug, Default, PartialEq, Eq)]
+pub struct ImageOut {
+    /// Size the request asked for, picks a size keyed price like fal's `output_cost_per_image_<size>`
+    pub size: Option<ImageSize>,
+    /// Pixel width the response reported, for per-pixel and nearest-size pricing
+    pub width: Option<u32>,
+    /// Pixel height the response reported, for per-pixel and nearest-size pricing
+    pub height: Option<u32>,
+}
+
+#[derive(Clone, Debug, Default, PartialEq, Eq)]
+pub struct Search {
+    /// Search API queries, for `input_cost_per_query` rows
+    pub queries: u64,
+    /// Results the query asked for, picks the `max_results_range` tier
+    pub max_results: Option<u64>,
+    /// Web searches a model ran as a tool
+    pub web_search: Option<WebSearch>,
+    /// Google Maps grounding requests
+    pub maps_grounding: u64,
+    /// Groq `browser.open` calls
+    pub browser_opens: u64,
+    /// Results past the ones the query price includes. Python counts them, since the included 10 is a request default
+    pub additional_results: u64,
+}
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub struct WebSearch {
+    /// Web search requests the model made
+    pub requests: u64,
+    /// Context size the request asked for, picks `search_context_size_<size>`
+    pub context: SearchContextSize,
+}

litellm-rust/crates/cost/tests/calculation.rs

@@ -1,535 +0,0 @@
-use litellm_cost::{
-    Error, OffPeakRates, Pricing, PromptConvention, Rate, Rates, Request, ServiceTier,
-    ThresholdPolicy, ThresholdRates, TierRates, Usage, calculate, compile,
-};
-use rstest::rstest;
-
-fn rates(input: Rate, output: Rate) -> Rates {
-    Rates {
-        input,
-        output,
-        ..Rates::EMPTY
-    }
-}
-
-fn request() -> Request {
-    Request {
-        usage: Usage {
-            prompt_tokens: 100,
-            completion_tokens: 20,
-            cache_read_tokens: 25,
-            cache_write_tokens: 10,
-            cache_write_5m_tokens: None,
-            cache_write_1h_tokens: None,
-            prompt_convention: PromptConvention::IncludesCache,
-        },
-        service_tier: ServiceTier::Standard,
-        threshold_policy: ThresholdPolicy::Exclusive,
-        region_multiplier: None,
-        billed_at_utc_minute: None,
-    }
-}
-
-fn pricing(standard: Rates) -> Pricing<'static> {
-    Pricing {
-        standard,
-        tiers: &[],
-        thresholds: &[],
-        off_peak: None,
-    }
-}
-
-#[test]
-fn breakdown_and_total_agree() {
-    let standard = Rates {
-        cache_read: Rate::Value(0.5),
-        cache_write: Rate::Value(3.0),
-        ..rates(Rate::Value(2.0), Rate::Value(4.0))
-    };
-    let result = calculate(&pricing(standard), &request()).unwrap();
-    assert_eq!(result.uncached_input, 65.0 * 2.0);
-    assert_eq!(result.cache_read, 25.0 * 0.5);
-    assert_eq!(result.cache_write_5m, 10.0 * 3.0);
-    assert_eq!(result.output(), 20.0 * 4.0);
-    assert_eq!(result.total(), result.input() + result.output());
-    assert_eq!(result.rates.cache_read, 0.5);
-}
-
-#[rstest]
-#[case::missing(Rate::Missing, Rate::Missing, 200.0)]
-#[case::null(Rate::Null, Rate::Missing, 200.0)]
-#[case::zero(Rate::Value(0.0), Rate::Value(0.0), 130.0)]
-fn absent_null_and_zero_cache_rates_are_distinct(
-    #[case] cache_read: Rate,
-    #[case] cache_write: Rate,
-    #[case] expected_input: f64,
-) {
-    let base = rates(Rate::Value(2.0), Rate::Value(4.0));
-    let standard = Rates {
-        cache_read,
-        cache_write,
-        ..base
-    };
-    assert_eq!(
-        calculate(&pricing(standard), &request()).unwrap().input(),
-        expected_input
-    );
-}
-
-#[test]
-fn equivalent_prompt_conventions_select_the_same_threshold() {
-    let threshold = ThresholdRates {
-        above_prompt_tokens: 90,
-        standard: rates(Rate::Value(5.0), Rate::Value(8.0)),
-        tiers: &[],
-    };
-    let specification = Pricing {
-        standard: rates(Rate::Value(2.0), Rate::Value(4.0)),
-        tiers: &[],
-        thresholds: &[threshold],
-        off_peak: None,
-    };
-    let included = request();
-    let excluded = Request {
-        usage: Usage {
-            prompt_tokens: 65,
-            prompt_convention: PromptConvention::ExcludesCache,
-            ..included.usage
-        },
-        ..included
-    };
-    let plan = compile(&specification).unwrap();
-    assert_eq!(plan.calculate(&included), plan.calculate(&excluded));
-    assert_eq!(plan.calculate(&included).unwrap().rates.input, 5.0);
-}
-
-#[test]
-fn split_writes_and_invalid_accounting() {
-    let standard = Rates {
-        cache_read: Rate::Value(0.5),
-        cache_write: Rate::Value(3.0),
-        cache_write_1h: Rate::Value(5.0),
-        ..rates(Rate::Value(2.0), Rate::Value(4.0))
-    };
-    let base = request();
-    let split = Request {
-        usage: Usage {
-            cache_write_5m_tokens: Some(4),
-            cache_write_1h_tokens: Some(6),
-            ..base.usage
-        },
-        ..base
-    };
-    let result = calculate(&pricing(standard), &split).unwrap();
-    assert_eq!(result.cache_write_5m, 12.0);
-    assert_eq!(result.cache_write_1h, 30.0);
-    let overlapping = Request {
-        usage: Usage {
-            prompt_tokens: 30,
-            ..split.usage
-        },
-        ..split
-    };
-    assert_eq!(
-        calculate(&pricing(standard), &overlapping),
-        Err(Error::CacheExceedsPrompt)
-    );
-    let incomplete = Request {
-        usage: Usage {
-            cache_write_1h_tokens: None,
-            ..split.usage
-        },
-        ..split
-    };
-    assert_eq!(
-        calculate(&pricing(standard), &incomplete),
-        Err(Error::InvalidCacheWriteDetails)
-    );
-}
-
-#[test]
-fn threshold_tiers_and_boundaries() {
-    let priority = TierRates {
-        tier: ServiceTier::Priority,
-        rates: rates(Rate::Value(3.0), Rate::Missing),
-    };
-    let threshold = ThresholdRates {
-        above_prompt_tokens: 100,
-        standard: rates(Rate::Value(5.0), Rate::Value(8.0)),
-        tiers: &[
-            TierRates {
-                tier: ServiceTier::Priority,
-                rates: rates(Rate::Value(7.0), Rate::Missing),
-            },
-            TierRates {
-                tier: ServiceTier::Flex,
-                rates: rates(Rate::Value(6.0), Rate::Missing),
-            },
-        ],
-    };
-    let specification = Pricing {
-        standard: rates(Rate::Value(2.0), Rate::Value(4.0)),
-        tiers: &[priority],
-        thresholds: &[threshold],
-        off_peak: None,
-    };
-    let base = request();
-    let no_cache = Request {
-        usage: Usage {
-            cache_read_tokens: 0,
-            cache_write_tokens: 0,
-            ..base.usage
-        },
-        ..base
-    };
-    let fast = Request {
-        service_tier: ServiceTier::Fast,
-        ..no_cache
-    };
-    let inclusive = Request {
-        threshold_policy: ThresholdPolicy::Inclusive,
-        ..fast
-    };
-    let flex = Request {
-        service_tier: ServiceTier::Flex,
-        ..inclusive
-    };
-    assert_eq!(calculate(&specification, &no_cache).unwrap().input(), 200.0);
-    assert_eq!(calculate(&specification, &fast).unwrap().input(), 300.0);
-    assert_eq!(
-        calculate(&specification, &inclusive).unwrap().input(),
-        700.0
-    );
-    assert_eq!(calculate(&specification, &flex).unwrap().input(), 600.0);
-}
-
-#[rstest]
-#[case::ultrafast_above_threshold(ServiceTier::Ultrafast, 300_000, 9_301_000.0, 37_000.0)]
-#[case::ultrafast_at_threshold(ServiceTier::Ultrafast, 272_000, 544_500.0, 5_000.0)]
-#[case::standard_above_threshold(ServiceTier::Standard, 300_000, 3_300_600.0, 13_000.0)]
-#[case::priority_above_threshold(ServiceTier::Priority, 300_000, 5_701_000.0, 23_000.0)]
-fn tiered_long_context_rates_are_selected_by_service_tier(
-    #[case] service_tier: ServiceTier,
-    #[case] prompt_tokens: u64,
-    #[case] expected_input: f64,
-    #[case] expected_output: f64,
-) {
-    let standard = Rates {
-        cache_read: Rate::Value(3.0),
-        ..rates(Rate::Value(1.0), Rate::Value(2.0))
-    };
-    let tiers = [
-        TierRates {
-            tier: ServiceTier::Priority,
-            rates: Rates {
-                cache_read: Rate::Value(5.0),
-                ..rates(Rate::Value(3.0), Rate::Value(4.0))
-            },
-        },
-        TierRates {
-            tier: ServiceTier::Ultrafast,
-            rates: Rates {
-                cache_read: Rate::Value(7.0),
-                ..rates(Rate::Value(2.0), Rate::Value(5.0))
-            },
-        },
-    ];
-    let threshold_tiers = [
-        TierRates {
-            tier: ServiceTier::Priority,
-            rates: Rates {
-                cache_read: Rate::Value(29.0),
-                ..rates(Rate::Value(19.0), Rate::Value(23.0))
-            },
-        },
-        TierRates {
-            tier: ServiceTier::Ultrafast,
-            rates: Rates {
-                cache_read: Rate::Value(41.0),
-                ..rates(Rate::Value(31.0), Rate::Value(37.0))
-            },
-        },
-    ];
-    let thresholds = [ThresholdRates {
-        above_prompt_tokens: 272_000,
-        standard: Rates {
-            cache_read: Rate::Value(17.0),
-            ..rates(Rate::Value(11.0), Rate::Value(13.0))
-        },
-        tiers: &threshold_tiers,
-    }];
-    let pricing = Pricing {
-        standard,
-        tiers: &tiers,
-        thresholds: &thresholds,
-        off_peak: None,
-    };
-    let base = request();
-    let long_context_request = Request {
-        usage: Usage {
-            prompt_tokens,
-            completion_tokens: 1_000,
-            cache_read_tokens: 100,
-            cache_write_tokens: 0,
-            ..base.usage
-        },
-        service_tier,
-        ..base
-    };
-    let cost = calculate(&pricing, &long_context_request).unwrap();
-
-    assert_eq!(cost.input(), expected_input);
-    assert_eq!(cost.output(), expected_output);
-}
-
-#[test]
-fn compile_rejects_ambiguous_rates() {
-    let duplicate = ThresholdRates {
-        above_prompt_tokens: 100,
-        standard: Rates::EMPTY,
-        tiers: &[],
-    };
-    let specification = Pricing {
-        standard: rates(Rate::Value(1.0), Rate::Value(1.0)),
-        tiers: &[],
-        thresholds: &[duplicate, duplicate],
-        off_peak: None,
-    };
-    assert_eq!(
-        compile(&specification).err(),
-        Some(Error::DuplicateThreshold)
-    );
-    let invalid = pricing(rates(Rate::Value(f64::NAN), Rate::Value(1.0)));
-    assert_eq!(compile(&invalid).err(), Some(Error::InvalidRate));
-}
-
-#[test]
-fn off_peak_is_one_non_wrapping_utc_window() {
-    let specification = Pricing {
-        standard: rates(Rate::Value(2.0), Rate::Value(4.0)),
-        tiers: &[],
-        thresholds: &[],
-        off_peak: Some(OffPeakRates {
-            start_utc_minute: 60,
-            end_utc_minute: 120,
-            rates: rates(Rate::Value(1.0), Rate::Value(2.0)),
-        }),
-    };
-    let base = request();
-    let start = Request {
-        billed_at_utc_minute: Some(60),
-        ..base
-    };
-    let end = Request {
-        billed_at_utc_minute: Some(120),
-        ..base
-    };
-    assert_eq!(
-        calculate(&specification, &base),
-        Err(Error::InvalidBillingTime)
-    );
-    assert_eq!(calculate(&specification, &start).unwrap().input(), 100.0);
-    assert_eq!(calculate(&specification, &end).unwrap().input(), 200.0);
-}
-
-#[test]
-fn missing_rates_and_free_rates_remain_distinct() {
-    let base = request();
-    let empty = Request {
-        usage: Usage {
-            prompt_tokens: 0,
-            completion_tokens: 0,
-            cache_read_tokens: 0,
-            cache_write_tokens: 0,
-            ..base.usage
-        },
-        ..base
-    };
-    assert_eq!(
-        calculate(&pricing(Rates::EMPTY), &empty),
-        Err(Error::MissingInputRate)
-    );
-    assert_eq!(
-        calculate(&pricing(rates(Rate::Value(0.0), Rate::Missing)), &empty),
-        Err(Error::MissingOutputRate)
-    );
-    assert_eq!(
-        calculate(&pricing(rates(Rate::Value(0.0), Rate::Value(0.0))), &empty)
-            .unwrap()
-            .total(),
-        0.0
-    );
-}
-
-#[test]
-fn matches_executed_python_reference_cases() {
-    for row in include_str!("python_reference.tsv")
-        .lines()
-        .filter(|line| !line.starts_with('#'))
-    {
-        let fields: Vec<_> = row.split('\t').collect();
-        let count = |index: usize| fields[index].parse::<u64>().unwrap();
-        let number = |index: usize| fields[index].parse::<f64>().unwrap();
-        let optional_rate = |index: usize| {
-            if fields[index].is_empty() {
-                Rate::Missing
-            } else {
-                Rate::Value(number(index))
-            }
-        };
-        let threshold = ThresholdRates {
-            above_prompt_tokens: if fields[9].is_empty() { 0 } else { count(9) },
-            standard: rates(optional_rate(10), optional_rate(11)),
-            tiers: &[],
-        };
-        let thresholds = if fields[9].is_empty() {
-            &[][..]
-        } else {
-            std::slice::from_ref(&threshold)
-        };
-        let specification = Pricing {
-            standard: Rates {
-                cache_read: optional_rate(7),
-                cache_write: optional_rate(8),
-                ..rates(Rate::Value(number(5)), Rate::Value(number(6)))
-            },
-            tiers: &[],
-            thresholds,
-            off_peak: None,
-        };
-        let base = request();
-        let input = Request {
-            usage: Usage {
-                prompt_tokens: count(1),
-                completion_tokens: count(2),
-                cache_read_tokens: count(3),
-                cache_write_tokens: count(4),
-                ..base.usage
-            },
-            ..base
-        };
-        let actual = calculate(&specification, &input).unwrap();
-        assert_eq!(actual.input(), number(12), "{}", fields[0]);
-        assert_eq!(actual.output(), number(13), "{}", fields[0]);
-    }
-}
-
-proptest::proptest! {
-    #[test]
-    fn equivalent_usage_conventions_and_breakdown_agree(
-        regular in 0_u64..1000,
-        read in 0_u64..1000,
-        write in 0_u64..1000,
-        output in 0_u64..1000,
-    ) {
-        let threshold = ThresholdRates {
-            above_prompt_tokens: 1000,
-            standard: rates(Rate::Value(5.0), Rate::Value(8.0)),
-            tiers: &[],
-        };
-        let specification = Pricing {
-            standard: rates(Rate::Value(2.0), Rate::Value(4.0)),
-            tiers: &[],
-            thresholds: &[threshold],
-            off_peak: None,
-        };
-        let base = request();
-        let included = Request {
-            usage: Usage {
-                prompt_tokens: regular + read + write,
-                completion_tokens: output,
-                cache_read_tokens: read,
-                cache_write_tokens: write,
-                ..base.usage
-            },
-            ..base
-        };
-        let excluded = Request {
-            usage: Usage {
-                prompt_tokens: regular,
-                prompt_convention: PromptConvention::ExcludesCache,
-                ..included.usage
-            },
-            ..included
-        };
-        let plan = compile(&specification).unwrap();
-        let left = plan.calculate(&included).unwrap();
-        let right = plan.calculate(&excluded).unwrap();
-        proptest::prop_assert_eq!(left, right);
-        proptest::prop_assert_eq!(left.total(), left.input() + left.output());
-    }
-}
-
-#[test]
-fn regional_multiplier_applies_after_input_components_are_summed() {
-    let standard = Rates {
-        cache_read: Rate::Value(0.5),
-        cache_write: Rate::Value(3.0),
-        ..rates(Rate::Value(2.0), Rate::Value(4.0))
-    };
-    let base = request();
-    let regional = Request {
-        region_multiplier: Some(1.1),
-        ..base
-    };
-    let result = calculate(&pricing(standard), &regional).unwrap();
-    assert_eq!(result.input(), (65.0 * 2.0 + 25.0 * 0.5 + 10.0 * 3.0) * 1.1);
-    assert_eq!(result.output(), 20.0 * 4.0 * 1.1);
-    let invalid = Request {
-        region_multiplier: Some(f64::NAN),
-        ..base
-    };
-    assert_eq!(
-        calculate(&pricing(standard), &invalid),
-        Err(Error::InvalidRegionMultiplier)
-    );
-}
-
-#[test]
-fn compilation_sorts_thresholds_and_rejects_duplicate_tiers() {
-    let high = ThresholdRates {
-        above_prompt_tokens: 200,
-        standard: rates(Rate::Value(7.0), Rate::Missing),
-        tiers: &[],
-    };
-    let low = ThresholdRates {
-        above_prompt_tokens: 100,
-        standard: rates(Rate::Value(5.0), Rate::Missing),
-        tiers: &[],
-    };
-    let specification = Pricing {
-        standard: rates(Rate::Value(2.0), Rate::Value(4.0)),
-        tiers: &[],
-        thresholds: &[high, low],
-        off_peak: None,
-    };
-    let base = request();
-    let above_both = Request {
-        usage: Usage {
-            prompt_tokens: 201,
-            cache_read_tokens: 0,
-            cache_write_tokens: 0,
-            ..base.usage
-        },
-        ..base
-    };
-    assert_eq!(
-        compile(&specification)
-            .unwrap()
-            .calculate(&above_both)
-            .unwrap()
-            .rates
-            .input,
-        7.0
-    );
-    let duplicate = TierRates {
-        tier: ServiceTier::Flex,
-        rates: Rates::EMPTY,
-    };
-    let invalid = Pricing {
-        tiers: &[duplicate, duplicate],
-        thresholds: &[],
-        ..specification
-    };
-    assert_eq!(compile(&invalid).err(), Some(Error::DuplicateTier));
-}

litellm-rust/crates/cost/tests/generate_python_reference.py

@@ -1,96 +0,0 @@
-import subprocess
-from dataclasses import dataclass
-from pathlib import Path
-
-from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token
-from litellm.types.utils import Usage
-
-
-@dataclass(frozen=True, slots=True)
-class Case:
-    name: str
-    prompt: int
-    completion: int
-    cache_read: int
-    cache_write: int
-    input_rate: float
-    output_rate: float
-    cache_read_rate: float | None = None
-    cache_write_rate: float | None = None
-    threshold: int | None = None
-    threshold_input_rate: float | None = None
-    threshold_output_rate: float | None = None
-
-
-CASES = (
-    Case("ordinary", 100, 20, 0, 0, 2.0, 4.0),
-    Case("cache_fallback", 100, 20, 25, 10, 2.0, 4.0),
-    Case("cache_specific", 100, 20, 25, 10, 2.0, 4.0, 0.5, 3.0),
-    Case("free_cache", 100, 20, 25, 10, 2.0, 4.0, 0.0, 0.0),
-    Case("threshold_below", 99, 20, 0, 0, 2.0, 4.0, threshold=100, threshold_input_rate=5.0, threshold_output_rate=8.0),
-    Case("threshold_at", 100, 20, 0, 0, 2.0, 4.0, threshold=100, threshold_input_rate=5.0, threshold_output_rate=8.0),
-    Case(
-        "threshold_above", 101, 20, 0, 0, 2.0, 4.0, threshold=100, threshold_input_rate=5.0, threshold_output_rate=8.0
-    ),
-    Case(
-        "cache_threshold_above",
-        101,
-        20,
-        25,
-        10,
-        2.0,
-        4.0,
-        threshold=100,
-        threshold_input_rate=5.0,
-        threshold_output_rate=8.0,
-    ),
-)
-
-
-def reference(case: Case) -> tuple[float, float]:
-    info = {"input_cost_per_token": case.input_rate, "output_cost_per_token": case.output_rate}
-    if case.cache_read_rate is not None:
-        info["cache_read_input_token_cost"] = case.cache_read_rate
-    if case.cache_write_rate is not None:
-        info["cache_creation_input_token_cost"] = case.cache_write_rate
-    if case.threshold is not None:
-        info[f"input_cost_per_token_above_{case.threshold}_tokens"] = case.threshold_input_rate
-        info[f"output_cost_per_token_above_{case.threshold}_tokens"] = case.threshold_output_rate
-    details = {"cached_tokens": case.cache_read, "cache_write_tokens": case.cache_write}
-    usage = Usage(prompt_tokens=case.prompt, completion_tokens=case.completion, prompt_tokens_details=details)
-    return generic_cost_per_token(
-        model="synthetic",
-        usage=usage,
-        custom_llm_provider="openai",
-        model_info=info,
-    )
-
-
-def main() -> None:
-    revision = subprocess.check_output(("git", "rev-parse", "HEAD"), text=True).strip()
-    rows = ("# Python reference commit: " + revision,) + tuple(
-        "\t".join(
-            str(value) if value is not None else ""
-            for value in (
-                case.name,
-                case.prompt,
-                case.completion,
-                case.cache_read,
-                case.cache_write,
-                case.input_rate,
-                case.output_rate,
-                case.cache_read_rate,
-                case.cache_write_rate,
-                case.threshold,
-                case.threshold_input_rate,
-                case.threshold_output_rate,
-                *reference(case),
-            )
-        )
-        for case in CASES
-    )
-    Path(__file__).with_name("python_reference.tsv").write_text("\n".join(rows) + "\n")
-
-
-if __name__ == "__main__":
-    main()

litellm-rust/crates/cost/tests/python_reference.tsv

@@ -1,9 +0,0 @@
-# Python reference commit: dc4be2fd987c993aefcf16444e34f960c12c8627
-ordinary	100	20	0	0	2.0	4.0						200.0	80.0
-cache_fallback	100	20	25	10	2.0	4.0						200.0	80.0
-cache_specific	100	20	25	10	2.0	4.0	0.5	3.0				172.5	80.0
-free_cache	100	20	25	10	2.0	4.0	0.0	0.0				130.0	80.0
-threshold_below	99	20	0	0	2.0	4.0			100	5.0	8.0	198.0	80.0
-threshold_at	100	20	0	0	2.0	4.0			100	5.0	8.0	200.0	80.0
-threshold_above	101	20	0	0	2.0	4.0			100	5.0	8.0	505.0	160.0
-cache_threshold_above	101	20	25	10	2.0	4.0			100	5.0	8.0	505.0	160.0

litellm-rust/crates/gateway/src/lib.rs

@@ -37,6 +37,7 @@ pub fn build_inference(config: &Config) -> Result<Arc<Gateway>, Error> {
             .map(|value| value.expose().to_owned())
             .or_else(|| std::env::var(name).ok())
     };
+    let router = ModelRouter::from_config(config, &lookup);
     let settings = &config.litellm_settings;
     let http_settings = HttpSettings::from_layers([
         HttpSettingsLayer::from_environment(&lookup),
@@ -64,12 +65,7 @@ pub fn build_inference(config: &Config) -> Result<Arc<Gateway>, Error> {
         Arc::new(EnvironmentSecrets::python_compatible(client)),
     ));
     let resources = CoreResources::new(pool);
-    Ok(Arc::new(Gateway::new(
-        resources,
-        http,
-        secrets,
-        ModelRouter::from_model_list(&config.model_list),
-    )?))
+    Ok(Arc::new(Gateway::new(resources, http, secrets, router)?))
 }
 
 pub async fn build_redis(config: &Config) -> Result<RedisClients, Error> {

litellm-rust/crates/inference-messages/src/handler.rs

@@ -141,9 +141,13 @@ async fn provider_call(
         &env_lookup,
     )?;
 
-    let body = config.transform_anthropic_messages_request(
-        trimmed,
-        &MessagesTransformContext::new(shaping.capabilities, shaping.settings.drop_params),
+    let body = config.request_body(
+        config.transform_anthropic_messages_request(
+            trimmed,
+            &MessagesTransformContext::new(shaping.capabilities, shaping.settings.drop_params),
+        )?,
+        &validated.headers,
+        shaping.settings.drop_params,
     )?;
     let environment = ValidatedEnvironment {
         headers: config.request_headers(
@@ -514,6 +518,84 @@ mod tests {
         );
     }
 
+    fn vertex_call_requesting(
+        shaping: MessagesShaping,
+        requested: &str,
+        drop_params: bool,
+    ) -> MessagesCall {
+        MessagesCall {
+            body: body(
+                json!({"model": "claude-sonnet-4-5", "messages": [{"role": "user", "content": "hi"}], "max_tokens": 16}),
+            ),
+            api_key: None,
+            api_base: Some("https://vertex.test".into()),
+            custom_llm_provider: Some("vertex_ai".into()),
+            litellm_params: Default::default(),
+            extra_headers: Some(Map::from_iter([
+                ("authorization".to_string(), json!("Bearer vertex-token")),
+                ("anthropic-version".to_string(), json!(requested)),
+            ])),
+            provider_specific_header: None,
+            timeout: None,
+            shaping: MessagesShaping {
+                settings: MessagesSettings {
+                    drop_params,
+                    ..shaping.settings
+                },
+                ..shaping
+            },
+        }
+    }
+
+    fn vertex_project(name: &str) -> Option<String> {
+        (name == "VERTEXAI_PROJECT").then(|| "version-project".to_string())
+    }
+
+    #[rstest]
+    #[case::sdk_2("2026-10-21", false, "2026-10-21")]
+    #[case::sdk_1("2023-06-01", false, "vertex-2023-10-16")]
+    #[case::unknown_under_drop_params("latest", true, "vertex-2023-10-16")]
+    #[tokio::test]
+    async fn vertex_gets_the_requested_version_in_the_body_alone(
+        shaping: MessagesShaping,
+        #[case] requested: &str,
+        #[case] drop_params: bool,
+        #[case] body_version: &str,
+    ) {
+        let call = provider_call_with_secrets(
+            vertex_call_requesting(shaping, requested, drop_params),
+            &vertex_project,
+        )
+        .await
+        .unwrap();
+        let version_headers: Vec<&str> = call
+            .wire
+            .headers
+            .iter()
+            .filter(|(name, _)| name.eq_ignore_ascii_case("anthropic-version"))
+            .map(|(_, value)| value.as_str())
+            .collect();
+        assert_eq!(
+            (version_headers, &call.wire.body["anthropic_version"]),
+            (Vec::<&str>::new(), &json!(body_version))
+        );
+    }
+
+    #[rstest]
+    #[tokio::test]
+    async fn vertex_refuses_an_unsupported_version_without_drop_params(shaping: MessagesShaping) {
+        assert!(matches!(
+            provider_call_with_secrets(
+                vertex_call_requesting(shaping, "latest", false),
+                &vertex_project
+            )
+            .await,
+            Err(Error::Llm(litellm_llms::Error::InvalidRequest(
+                litellm_llms::ErrorDetail::UnsupportedParameter { .. }
+            )))
+        ));
+    }
+
     #[rstest]
     #[tokio::test]
     async fn missing_credentials_are_reported_before_the_request_is_transformed(

litellm-rust/crates/inference-ocr/tests/ocr/azure_document_intelligence.rs

@@ -12,6 +12,7 @@ use rstest::rstest;
 use super::*;
 
 const MODEL: &str = "azure_ai/doc-intelligence/prebuilt-read";
+const LAYOUT_MODEL: &str = "azure_ai/doc-intelligence/prebuilt-layout";
 
 fn read_request(base: &str, options: Value) -> LiteLLMOcrRequest {
     ocr_request(MODEL, base, options)
@@ -70,6 +71,7 @@ async fn pages_features_and_extra_options_map_to_the_analyze_call() {
 #[case(json!({"pages": "1&&features=bad"}), Error::Pages(PagesFault::Range))]
 #[case(json!({"features": "languages&pages=1"}), Error::Features)]
 #[case(json!({"req_format": "azure"}), Error::RequestFormat)]
+#[case(json!({"outputContentFormat": "html"}), Error::OutputContentFormat)]
 #[tokio::test]
 async fn invalid_pages_features_and_format_are_rejected_before_sending(
     #[case] options: Value,
@@ -130,6 +132,137 @@ async fn an_inline_document_is_sent_as_base64_and_only_page_text_is_kept(#[case]
     assert_eq!(sent.json(), json!({"base64Source": "YWJj"}));
 }
 
+#[rstest]
+#[case::lower_case("markdown")]
+#[case::capitalized("Markdown")]
+#[tokio::test]
+async fn markdown_output_is_requested_on_the_analyze_call_and_split_into_pages(
+    #[case] output_content_format: &str,
+) {
+    let upstream = upstream([json_response(json!({
+        "status": "succeeded",
+        "analyzeResult": {
+            "content": "# Tïtle\n\n<!-- PageBreak -->\n\n| a |",
+            "contentFormat": "markdown",
+            "stringIndexType": "unicodeCodePoint",
+            "pages": [
+                {"pageNumber": 1, "spans": [{"offset": 0, "length": 29}], "lines": [{"content": "Tïtle"}]},
+                {"pageNumber": 2, "spans": [{"offset": 29, "length": 5}], "lines": [{"content": "a"}]}
+            ]
+        }
+    }))])
+    .await;
+
+    let response = perform(ocr_request(
+        LAYOUT_MODEL,
+        &upstream.uri(),
+        json!({"outputContentFormat": output_content_format, "pages": [0, 1]}),
+    ))
+    .await
+    .unwrap();
+
+    let sent = only_request(&upstream).await;
+    assert_eq!(
+        sent.query("outputContentFormat").as_deref(),
+        Some("markdown")
+    );
+    assert_eq!(
+        sent.query("stringIndexType").as_deref(),
+        Some("unicodeCodePoint")
+    );
+    assert_eq!(sent.query("pages").as_deref(), Some("1,2"));
+    assert_eq!(sent.json(), json!({"base64Source": "YWJj"}));
+    assert_eq!(
+        response.content.as_deref(),
+        Some("# Tïtle\n\n<!-- PageBreak -->\n\n| a |")
+    );
+    let pages = response
+        .pages
+        .iter()
+        .map(|page| page.markdown.as_str())
+        .collect::<Vec<_>>();
+    assert_eq!(pages, ["# Tïtle\n\n<!-- PageBreak -->\n\n", "| a |"]);
+    assert_eq!(
+        response.extra_fields.get("stringIndexType"),
+        Some(&json!("unicodeCodePoint"))
+    );
+    assert_eq!(
+        response.usage_info.as_ref().unwrap().pages_processed,
+        Some(2)
+    );
+}
+
+#[rstest]
+#[case::native_markdown(LAYOUT_MODEL, json!({"outputContentFormat": "markdown", "req_format": "native"}), Some("markdown"))]
+#[case::markdown_on_read(MODEL, json!({"outputContentFormat": "markdown"}), Some("markdown"))]
+#[case::text(LAYOUT_MODEL, json!({"outputContentFormat": "text"}), Some("text"))]
+#[case::omitted(LAYOUT_MODEL, json!({}), None)]
+#[tokio::test]
+async fn only_layout_markdown_in_litellm_format_asks_for_code_point_offsets(
+    #[case] model: &str,
+    #[case] options: Value,
+    #[case] output_content_format: Option<&str>,
+) {
+    let upstream = upstream([json_response(json!({
+        "status": "succeeded",
+        "analyzeResult": {
+            "contentFormat": "text",
+            "stringIndexType": "textElements",
+            "pages": [{"pageNumber": 1, "lines": [{"content": "hello"}]}]
+        }
+    }))])
+    .await;
+
+    let response = perform(ocr_request(model, &upstream.uri(), options))
+        .await
+        .unwrap();
+
+    let sent = only_request(&upstream).await;
+    assert_eq!(
+        sent.query("outputContentFormat").as_deref(),
+        output_content_format
+    );
+    assert_eq!(sent.query("stringIndexType"), None);
+    assert_eq!(sent.json(), json!({"base64Source": "YWJj"}));
+    assert_eq!(response.pages[0].markdown, "hello");
+    assert_eq!(response.extra_fields.get("stringIndexType"), None);
+}
+
+#[rstest]
+#[case::snake_case(json!({"output_content_format": "markdown"}), Some("markdown"))]
+#[case::padded(json!({"outputContentFormat": " markdown "}), Some("markdown"))]
+#[case::empty(json!({"outputContentFormat": ""}), None)]
+#[case::empty_camel_case_falls_back_to_snake_case(
+    json!({"outputContentFormat": "", "output_content_format": "markdown"}),
+    Some("markdown")
+)]
+#[case::dropped(json!({"outputContentFormat": "html", "drop_params": true}), None)]
+#[tokio::test]
+async fn the_requested_output_content_format_reaches_azure_in_its_query_spelling(
+    #[case] options: Value,
+    #[case] output_content_format: Option<&str>,
+) {
+    let upstream = upstream([json_response(json!({
+        "status": "succeeded",
+        "analyzeResult": {"pages": [{"pageNumber": 1, "lines": [{"content": "hello"}]}]}
+    }))])
+    .await;
+
+    let response = perform(ocr_request(MODEL, &upstream.uri(), options))
+        .await
+        .unwrap();
+
+    let sent = only_request(&upstream).await;
+    assert_eq!(
+        sent.query("outputContentFormat").as_deref(),
+        output_content_format
+    );
+    assert_eq!(sent.query("output_content_format"), None);
+    assert_eq!(sent.query("drop_params"), None);
+    assert_eq!(sent.json(), json!({"base64Source": "YWJj"}));
+    assert_eq!(response.pages[0].markdown, "hello");
+}
+
 #[tokio::test]
 async fn native_format_normalizes_pages_and_keeps_the_provider_response() {
     let operation = json!({

litellm-rust/crates/llms-types/src/formats/audio_transcription/response.rs

@@ -1,14 +1,31 @@
 use serde_json::Value;
 
+#[macro_rules_attribute::apply(crate::wire_type)]
+pub struct AudioTranscriptionUsage {
+    pub input_tokens: u64,
+    pub output_tokens: u64,
+    pub total_tokens: u64,
+}
+
 #[macro_rules_attribute::apply(crate::wire_type)]
 pub struct AudioTranscriptionResponseData {
     pub text: String,
+    pub usage: Option<AudioTranscriptionUsage>,
 }
 
 impl AudioTranscriptionResponseData {
     pub fn into_json(self) -> Value {
-        serde_json::json!({
-            "text": self.text,
-        })
+        match self.usage {
+            None => serde_json::json!({"text": self.text}),
+            Some(usage) => serde_json::json!({
+                "text": self.text,
+                "usage": {
+                    "type": "tokens",
+                    "input_tokens": usage.input_tokens,
+                    "output_tokens": usage.output_tokens,
+                    "total_tokens": usage.total_tokens,
+                },
+            }),
+        }
     }
 }

litellm-rust/crates/llms/src/azure_ai/ocr/document_intelligence/transformation.rs

@@ -1,4 +1,7 @@
-use std::{collections::BTreeSet, time::Duration};
+use std::{
+    collections::{BTreeMap, BTreeSet},
+    time::Duration,
+};
 
 use base64::{Engine, engine::general_purpose::STANDARD};
 use litellm_auth::{InputSource, Sourced};
@@ -8,7 +11,7 @@ use litellm_llms_types::serde_compat::{FiniteF64, LaxI64};
 use reqwest::Url;
 use serde::{Deserialize, Serialize};
 use serde_json::{Map, Value};
-use serde_with::serde_as;
+use serde_with::{DefaultOnError, serde_as};
 use tokio::time::Instant;
 
 use crate::base_llm::ocr::{
@@ -17,9 +20,10 @@ use crate::base_llm::ocr::{
     handler::{CallHooks, OcrClient, read_json_response},
     settings::OcrSettings,
     transformation::{
-        BaseOcrConfig, DecodedOcrResponse, OCR_INLINE_MAX_BYTES, OCR_POLL_RETRY_SECS,
-        OcrConnection, OcrCredentialInputs, OcrResponseContext, PreparedOcrRequest,
-        ResolvedOcrCredentials, decode_and_normalize_response, decode_response,
+        BaseOcrConfig, DROP_PARAMS_PARAM, DecodedOcrResponse, OCR_INLINE_MAX_BYTES,
+        OCR_POLL_RETRY_SECS, OcrConnection, OcrCredentialInputs, OcrResponseContext,
+        PreparedOcrRequest, ResolvedOcrCredentials, decode_and_normalize_response, decode_response,
+        drops_unsupported_params, response_format,
     },
 };
 use litellm_llms_types::formats::ocr::{
@@ -32,6 +36,30 @@ const AZURE_DI_DEFAULT_HEIGHT: f64 = 11.0;
 
 const AZURE_DI_API_KEY_ENV: &str = "AZURE_DOCUMENT_INTELLIGENCE_API_KEY";
 const AZURE_DI_ENDPOINT_ENV: &str = "AZURE_DOCUMENT_INTELLIGENCE_ENDPOINT";
+const OUTPUT_CONTENT_FORMAT_PARAM: &str = "outputContentFormat";
+const OUTPUT_CONTENT_FORMAT_ALIAS: &str = "output_content_format";
+const CODE_POINT_STRING_INDEX: &str = "unicodeCodePoint";
+const STRING_INDEX_TYPE_FIELD: &str = "stringIndexType";
+const MARKDOWN_MODEL_ID: &str = "prebuilt-layout";
+
+#[derive(
+    Clone,
+    Copy,
+    Debug,
+    PartialEq,
+    strum::EnumString,
+    strum::IntoStaticStr,
+    strum::Display,
+    strum::VariantArray,
+    serde_with::SerializeDisplay,
+)]
+#[strum(ascii_case_insensitive)]
+pub enum OutputContentFormat {
+    #[strum(serialize = "text")]
+    Text,
+    #[strum(serialize = "markdown")]
+    Markdown,
+}
 
 #[derive(Clone, Debug, PartialEq, Serialize)]
 pub struct DocumentIntelligenceParams {
@@ -39,6 +67,13 @@ pub struct DocumentIntelligenceParams {
     pub pages: Option<String>,
     #[serde(skip_serializing_if = "Option::is_none")]
     pub features: Option<String>,
+    #[serde(
+        rename = "outputContentFormat",
+        skip_serializing_if = "Option::is_none"
+    )]
+    pub output_content_format: Option<OutputContentFormat>,
+    #[serde(rename = "stringIndexType", skip_serializing_if = "Option::is_none")]
+    pub string_index_type: Option<&'static str>,
 }
 
 #[derive(Clone, Debug, Serialize, Deserialize)]
@@ -80,6 +115,10 @@ pub struct AzureDocumentIntelligenceOperation {
 #[derive(Clone, Debug, Default, Deserialize)]
 struct AzureDocumentIntelligenceAnalyzeResult {
     pub content: Option<String>,
+    #[serde(rename = "contentFormat")]
+    pub content_format: Option<String>,
+    #[serde(rename = "stringIndexType")]
+    pub string_index_type: Option<String>,
     #[serde(default)]
     pub pages: Vec<AzureDocumentIntelligencePage>,
     pub tables: Option<Vec<Map<String, Value>>>,
@@ -100,6 +139,9 @@ struct AzureDocumentIntelligencePage {
     pub unit: Option<String>,
     #[serde(default)]
     pub lines: Vec<AzureDocumentIntelligenceLine>,
+    #[serde(default)]
+    #[serde_as(deserialize_as = "DefaultOnError")]
+    pub spans: Option<Vec<AzureDocumentIntelligenceSpan>>,
 }
 
 #[derive(Clone, Debug, Deserialize)]
@@ -107,6 +149,18 @@ struct AzureDocumentIntelligenceLine {
     pub content: Option<String>,
 }
 
+#[derive(Clone, Debug, Deserialize)]
+struct AzureDocumentIntelligenceSpan {
+    pub offset: usize,
+    pub length: usize,
+}
+
+impl AzureDocumentIntelligenceSpan {
+    fn end(&self) -> usize {
+        self.offset.saturating_add(self.length)
+    }
+}
+
 #[derive(Clone, Debug)]
 pub struct AzureDocumentIntelligenceOcrConfig;
 
@@ -116,7 +170,14 @@ impl BaseOcrConfig for AzureDocumentIntelligenceOcrConfig {
     type Environment = Vec<(String, String)>;
 
     fn get_supported_ocr_params(&self, _model: &str) -> &'static [&'static str] {
-        &["pages", "features", "req_format"]
+        &[
+            "pages",
+            "features",
+            OUTPUT_CONTENT_FORMAT_PARAM,
+            OUTPUT_CONTENT_FORMAT_ALIAS,
+            "req_format",
+            DROP_PARAMS_PARAM,
+        ]
     }
 
     fn get_api_key_env_var(&self) -> Option<&'static str> {
@@ -154,11 +215,17 @@ impl BaseOcrConfig for AzureDocumentIntelligenceOcrConfig {
     fn map_ocr_params(
         &self,
         non_default_params: &CallArguments,
-        _model: &str,
+        model: &str,
     ) -> Result<DocumentIntelligenceParams, Error> {
+        let output_content_format = requested_output_content_format(non_default_params)?;
+        let slices_markdown_pages = output_content_format == Some(OutputContentFormat::Markdown)
+            && response_format(non_default_params)? != OcrResponseFormat::Native
+            && model_id(model).is_ok_and(|id| id == MARKDOWN_MODEL_ID);
         Ok(DocumentIntelligenceParams {
             pages: normalize_pages_param(non_default_params.get("pages"))?,
             features: normalize_features_param(non_default_params.get("features"))?,
+            output_content_format,
+            string_index_type: slices_markdown_pages.then_some(CODE_POINT_STRING_INDEX),
         })
     }
 
@@ -337,6 +404,36 @@ fn normalize_features_param(features: Option<&Value>) -> Result<Option<String>,
     Ok(Some(normalized.join(",")))
 }
 
+fn requested_output_content_format(
+    params: &CallArguments,
+) -> Result<Option<OutputContentFormat>, Error> {
+    let requested = [OUTPUT_CONTENT_FORMAT_PARAM, OUTPUT_CONTENT_FORMAT_ALIAS]
+        .into_iter()
+        .filter_map(|name| params.get(name))
+        .find(|format| !is_unset(format));
+    match requested.map(parse_output_content_format).transpose() {
+        Err(Error::OutputContentFormat) if drops_unsupported_params(params) => Ok(None),
+        parsed => parsed,
+    }
+}
+
+fn is_unset(format: &Value) -> bool {
+    match format {
+        Value::Null => true,
+        Value::String(format) => format.trim().is_empty(),
+        _ => false,
+    }
+}
+
+fn parse_output_content_format(format: &Value) -> Result<OutputContentFormat, Error> {
+    format
+        .as_str()
+        .ok_or(Error::OutputContentFormat)?
+        .trim()
+        .parse()
+        .map_err(|_| Error::OutputContentFormat)
+}
+
 fn build_request(document: OcrDocument) -> Result<DocumentIntelligenceRequest, Error> {
     let source = document.source();
     if source.is_empty() {
@@ -367,10 +464,27 @@ fn transform_completed_response(
         ));
     }
     let result = response.analyze_result.unwrap_or_default();
+    let markdown_content = result
+        .content
+        .as_deref()
+        .filter(|_| {
+            result.content_format.as_deref() == Some(<&str>::from(OutputContentFormat::Markdown))
+                && result.string_index_type.as_deref() == Some(CODE_POINT_STRING_INDEX)
+                && result.pages.iter().all(|page| page.spans.is_some())
+        })
+        .map(|content| {
+            CodePointText::new(
+                content,
+                result
+                    .pages
+                    .iter()
+                    .flat_map(|page| page.spans.iter().flatten()),
+            )
+        });
     let pages = result
         .pages
         .into_iter()
-        .map(|page| transform_azure_page(page, dpi))
+        .map(|page| transform_azure_page(page, markdown_content.as_ref(), dpi))
         .collect::<Result<Vec<_>, _>>()?;
     let pages_processed = i64::try_from(pages.len()).map_err(|_| Error::NumericRange("pages"))?;
     Ok(LiteLLMOcrResponse {
@@ -381,11 +495,20 @@ fn transform_completed_response(
             pages_processed: Some(pages_processed),
             ..Default::default()
         }),
+        extra_fields: result
+            .string_index_type
+            .filter(|unit| unit == CODE_POINT_STRING_INDEX)
+            .map(|unit| Map::from_iter([(STRING_INDEX_TYPE_FIELD.to_owned(), Value::String(unit))]))
+            .unwrap_or_default(),
         ..LiteLLMOcrResponse::new(model, pages)
     })
 }
 
-fn transform_azure_page(page: AzureDocumentIntelligencePage, dpi: i64) -> Result<OcrPage, Error> {
+fn transform_azure_page(
+    page: AzureDocumentIntelligencePage,
+    markdown_content: Option<&CodePointText<'_>>,
+    dpi: i64,
+) -> Result<OcrPage, Error> {
     let index = page
         .page_number
         .unwrap_or(1)
@@ -397,12 +520,20 @@ fn transform_azure_page(page: AzureDocumentIntelligencePage, dpi: i64) -> Result
         page.unit.as_deref().unwrap_or("inch"),
         dpi,
     )?;
-    let markdown = page
-        .lines
-        .iter()
-        .map(|line| line.content.as_deref().unwrap_or_default())
-        .collect::<Vec<_>>()
-        .join("\n");
+    let markdown = match markdown_content {
+        Some(content) => page
+            .spans
+            .iter()
+            .flatten()
+            .map(|span| content.slice(span))
+            .collect::<String>(),
+        None => page
+            .lines
+            .iter()
+            .map(|line| line.content.as_deref().unwrap_or_default())
+            .collect::<Vec<_>>()
+            .join("\n"),
+    };
     Ok(OcrPage {
         index,
         markdown,
@@ -411,6 +542,43 @@ fn transform_azure_page(page: AzureDocumentIntelligencePage, dpi: i64) -> Result
     })
 }
 
+struct CodePointText<'a> {
+    text: &'a str,
+    byte_offsets: BTreeMap<usize, usize>,
+}
+
+impl<'a> CodePointText<'a> {
+    fn new<'s>(
+        text: &'a str,
+        spans: impl Iterator<Item = &'s AzureDocumentIntelligenceSpan>,
+    ) -> Self {
+        let span_bounds = spans
+            .flat_map(|span| [span.offset, span.end()])
+            .collect::<BTreeSet<_>>();
+        Self {
+            text,
+            byte_offsets: text
+                .char_indices()
+                .map(|(byte_offset, _)| byte_offset)
+                .enumerate()
+                .filter(|(code_point, _)| span_bounds.contains(code_point))
+                .collect(),
+        }
+    }
+
+    fn slice(&self, span: &AzureDocumentIntelligenceSpan) -> &'a str {
+        let byte_offset = |code_point: usize| {
+            self.byte_offsets
+                .get(&code_point)
+                .copied()
+                .unwrap_or(self.text.len())
+        };
+        self.text
+            .get(byte_offset(span.offset)..byte_offset(span.end()))
+            .unwrap_or_default()
+    }
+}
+
 fn convert_dimensions(
     width: f64,
     height: f64,
@@ -578,6 +746,16 @@ impl AzureDocumentIntelligenceOcrConfig {
                                 .features
                                 .iter()
                                 .map(|features| ("features", features.as_str())),
+                        )
+                        .chain(
+                            params
+                                .output_content_format
+                                .map(|format| (OUTPUT_CONTENT_FORMAT_PARAM, format.into())),
+                        )
+                        .chain(
+                            params
+                                .string_index_type
+                                .map(|index_type| ("stringIndexType", index_type)),
                         ),
                 )
                 .into_string()
@@ -697,8 +875,12 @@ mod tests {
     }
 
     fn map(value: Value) -> Result<DocumentIntelligenceParams, Error> {
+        map_for_model(value, "azure_ai/doc-intelligence/prebuilt-layout")
+    }
+
+    fn map_for_model(value: Value, model: &str) -> Result<DocumentIntelligenceParams, Error> {
         let arguments = serde_json::from_value(value).unwrap();
-        AzureDocumentIntelligenceOcrConfig.map_ocr_params(&arguments, "model")
+        AzureDocumentIntelligenceOcrConfig.map_ocr_params(&arguments, model)
     }
 
     #[test]
@@ -828,6 +1010,218 @@ mod tests {
         assert_eq!(map(json!({"features": []})).unwrap().features, None);
     }
 
+    #[rstest]
+    #[case::markdown(
+        json!({"outputContentFormat": "markdown"}),
+        Some(OutputContentFormat::Markdown),
+        Some("unicodeCodePoint")
+    )]
+    #[case::markdown_litellm_format(
+        json!({"outputContentFormat": "markdown", "req_format": "litellm"}),
+        Some(OutputContentFormat::Markdown),
+        Some("unicodeCodePoint")
+    )]
+    #[case::markdown_native_format(
+        json!({"outputContentFormat": "markdown", "req_format": "native"}),
+        Some(OutputContentFormat::Markdown),
+        None
+    )]
+    #[case::capitalized_markdown(
+        json!({"outputContentFormat": "Markdown"}),
+        Some(OutputContentFormat::Markdown),
+        Some("unicodeCodePoint")
+    )]
+    #[case::text(json!({"outputContentFormat": "text"}), Some(OutputContentFormat::Text), None)]
+    #[case::upper_case_text(json!({"outputContentFormat": "TEXT"}), Some(OutputContentFormat::Text), None)]
+    #[case::padded_markdown(
+        json!({"outputContentFormat": " markdown\n"}),
+        Some(OutputContentFormat::Markdown),
+        Some("unicodeCodePoint")
+    )]
+    #[case::snake_case_markdown(
+        json!({"output_content_format": "markdown"}),
+        Some(OutputContentFormat::Markdown),
+        Some("unicodeCodePoint")
+    )]
+    #[case::camel_case_wins_over_snake_case(
+        json!({"outputContentFormat": "text", "output_content_format": "markdown"}),
+        Some(OutputContentFormat::Text),
+        None
+    )]
+    #[case::empty_camel_case_falls_back_to_snake_case(
+        json!({"outputContentFormat": "", "output_content_format": "markdown"}),
+        Some(OutputContentFormat::Markdown),
+        Some("unicodeCodePoint")
+    )]
+    #[case::null_camel_case_falls_back_to_snake_case(
+        json!({"outputContentFormat": null, "output_content_format": "markdown"}),
+        Some(OutputContentFormat::Markdown),
+        Some("unicodeCodePoint")
+    )]
+    #[case::empty(json!({"outputContentFormat": ""}), None, None)]
+    #[case::blank(json!({"outputContentFormat": "  "}), None, None)]
+    #[case::null(json!({"outputContentFormat": null}), None, None)]
+    #[case::omitted(json!({}), None, None)]
+    fn output_content_format_maps_to_query_fields(
+        #[case] options: Value,
+        #[case] format: Option<OutputContentFormat>,
+        #[case] string_index_type: Option<&str>,
+    ) {
+        let mapped = map(options).unwrap();
+
+        assert_eq!(mapped.output_content_format, format);
+        assert_eq!(mapped.string_index_type, string_index_type);
+    }
+
+    #[rstest]
+    #[case::layout("azure_ai/doc-intelligence/prebuilt-layout", Some("unicodeCodePoint"))]
+    #[case::bare_layout("prebuilt-layout", Some("unicodeCodePoint"))]
+    #[case::read("azure_ai/doc-intelligence/prebuilt-read", None)]
+    #[case::invoice("azure_ai/doc-intelligence/prebuilt-invoice", None)]
+    #[case::custom("azure_ai/doc-intelligence/my-extraction-model", None)]
+    fn only_the_layout_model_asks_for_code_point_offsets(
+        #[case] model: &str,
+        #[case] string_index_type: Option<&str>,
+    ) {
+        let mapped = map_for_model(json!({"outputContentFormat": "markdown"}), model).unwrap();
+
+        assert_eq!(
+            mapped.output_content_format,
+            Some(OutputContentFormat::Markdown)
+        );
+        assert_eq!(mapped.string_index_type, string_index_type);
+    }
+
+    #[rstest]
+    #[case::unknown(json!({"outputContentFormat": "html"}))]
+    #[case::number(json!({"outputContentFormat": 1}))]
+    #[case::boolean(json!({"outputContentFormat": true}))]
+    #[case::list(json!({"outputContentFormat": ["markdown"]}))]
+    #[case::snake_case_unknown(json!({"output_content_format": "html"}))]
+    #[case::drop_params_off(json!({"outputContentFormat": "html", "drop_params": false}))]
+    fn invalid_output_content_format_is_a_request_error(#[case] options: Value) {
+        let error = map(options).unwrap_err();
+
+        assert!(matches!(error, Error::OutputContentFormat), "{error:?}");
+        assert_eq!(error.http_status_code(), Some(400));
+    }
+
+    #[rstest]
+    #[case::unknown(json!({"outputContentFormat": "html", "drop_params": true, "pages": [0]}))]
+    #[case::number(json!({"outputContentFormat": 1, "drop_params": true, "pages": [0]}))]
+    #[case::snake_case_unknown(
+        json!({"output_content_format": "html", "drop_params": true, "pages": [0]})
+    )]
+    fn drop_params_drops_an_invalid_output_content_format(#[case] options: Value) {
+        let mapped = map(options).unwrap();
+
+        assert_eq!(serde_json::to_value(mapped).unwrap(), json!({"pages": "1"}));
+    }
+
+    #[rstest]
+    #[case::camel_case(json!({"outputContentFormat": "markdown", "drop_params": true}))]
+    #[case::snake_case(json!({"output_content_format": "markdown", "drop_params": true}))]
+    fn drop_params_keeps_a_valid_output_content_format(#[case] options: Value) {
+        let mapped = map(options).unwrap();
+
+        assert_eq!(
+            mapped.output_content_format,
+            Some(OutputContentFormat::Markdown)
+        );
+    }
+
+    fn markdown_pages(analyze_result: Value) -> Vec<String> {
+        let raw =
+            serde_json::to_vec(&json!({"status": "succeeded", "analyzeResult": analyze_result}))
+                .unwrap();
+        AzureDocumentIntelligenceOcrConfig
+            .transform_ocr_response("model", &raw, OcrResponseFormat::Litellm)
+            .unwrap()
+            .pages
+            .into_iter()
+            .map(|page| page.markdown)
+            .collect()
+    }
+
+    #[rstest]
+    fn markdown_pages_are_code_point_slices_of_the_content() {
+        let content = "# Ünïcode क्ष\n\n<!-- PageBreak -->\n\n## Två 🙂\n\n|a|\n\n> tail";
+        let second_page_start = "# Ünïcode क्ष\n\n<!-- PageBreak -->\n\n".chars().count();
+        let pages = markdown_pages(json!({
+            "content": content,
+            "contentFormat": "markdown",
+            "stringIndexType": "unicodeCodePoint",
+            "pages": [
+                {"pageNumber": 1, "spans": [{"offset": 0, "length": second_page_start}],
+                 "lines": [{"content": "Ünïcode क्ष"}]},
+                {"pageNumber": 2, "spans": [
+                    {"offset": second_page_start, "length": 8},
+                    {"offset": second_page_start + 10, "length": 3}
+                ], "lines": [{"content": "Två"}]}
+            ]
+        }));
+
+        assert_eq!(
+            pages,
+            ["# Ünïcode क्ष\n\n<!-- PageBreak -->\n\n", "## Två 🙂|a|"]
+        );
+    }
+
+    #[rstest]
+    fn spans_past_the_content_end_are_clamped() {
+        let pages = markdown_pages(json!({
+            "content": "ab",
+            "contentFormat": "markdown",
+            "stringIndexType": "unicodeCodePoint",
+            "pages": [{"spans": [{"offset": 1, "length": 9}, {"offset": 7, "length": 1}]}]
+        }));
+
+        assert_eq!(pages, ["b"]);
+    }
+
+    #[rstest]
+    #[case::text_content(json!("text"), json!("unicodeCodePoint"))]
+    #[case::grapheme_offsets(json!("markdown"), json!("textElements"))]
+    #[case::no_index_type(json!("markdown"), Value::Null)]
+    #[case::no_content_format(Value::Null, json!("unicodeCodePoint"))]
+    fn other_responses_keep_line_joined_pages(
+        #[case] content_format: Value,
+        #[case] string_index_type: Value,
+    ) {
+        let pages = markdown_pages(json!({
+            "content": "# A\n\nB",
+            "contentFormat": content_format,
+            "stringIndexType": string_index_type,
+            "pages": [{"spans": [{"offset": 0, "length": 6}],
+                       "lines": [{"content": "A"}, {"content": "B"}]}]
+        }));
+
+        assert_eq!(pages, ["A\nB"]);
+    }
+
+    #[rstest]
+    #[case::null_spans(json!("markdown"), json!(null))]
+    #[case::fractional_offset(json!("markdown"), json!([{"offset": 0.5, "length": 1}]))]
+    #[case::missing_length(json!("markdown"), json!([{"offset": 0}]))]
+    #[case::negative_offset(json!("markdown"), json!([{"offset": -1, "length": 1}]))]
+    #[case::text_content(json!("text"), json!([{"offset": "0", "length": 1}]))]
+    fn unusable_page_spans_keep_line_joined_pages(
+        #[case] content_format: Value,
+        #[case] broken_spans: Value,
+    ) {
+        let pages = markdown_pages(json!({
+            "content": "# A\n\nB",
+            "contentFormat": content_format,
+            "stringIndexType": "unicodeCodePoint",
+            "pages": [
+                {"spans": [{"offset": 0, "length": 5}], "lines": [{"content": "A"}]},
+                {"spans": broken_spans, "lines": [{"content": "B"}]}
+            ]
+        }));
+
+        assert_eq!(pages, ["A", "B"]);
+    }
+
     #[tokio::test]
     async fn request_endpoint_cannot_receive_environment_key() {
         let connection = OcrConnection {

litellm-rust/crates/llms/src/base_llm/messages/transformation.rs

@@ -77,6 +77,15 @@ pub trait BaseMessagesConfig: Sync {
         headers
     }
 
+    fn request_body(
+        &self,
+        request: MessagesRequest,
+        _headers: &Headers,
+        _drop_params: bool,
+    ) -> Result<MessagesRequest, Error> {
+        Ok(request)
+    }
+
     /// The JSON that goes on the wire, for a host whose body differs from the typed request:
     /// Python's configs `pop("model")` when the model is addressed by the URL.
     fn wire_body(&self, body: Value) -> Value {

litellm-rust/crates/llms/src/base_llm/ocr/error.rs

@@ -59,6 +59,10 @@ pub enum Error {
     Pages(PagesFault),
     #[error("invalid OCR features")]
     Features,
+    #[error(
+        "invalid OCR outputContentFormat, expected text or markdown. To drop unsupported params, set `litellm.drop_params = True`."
+    )]
+    OutputContentFormat,
     #[error("OCR model cannot be a dot segment")]
     DotModel,
     #[error("OCR response exceeds the size limit of {limit} bytes")]
@@ -200,6 +204,7 @@ impl Error {
             | Self::InlineDocumentTooLarge
             | Self::Pages(_)
             | Self::Features
+            | Self::OutputContentFormat
             | Self::DotModel
             | Self::TooLarge { .. }
             | Self::ResponseField { .. }
@@ -242,6 +247,7 @@ impl Error {
             | Self::InlineDocumentTooLarge
             | Self::Pages(_)
             | Self::Features
+            | Self::OutputContentFormat
             | Self::DotModel
             | Self::TooLarge { .. }
             | Self::ResponseField { .. }
@@ -291,6 +297,7 @@ impl Error {
             | Self::InlineDocumentTooLarge
             | Self::Pages(_)
             | Self::Features
+            | Self::OutputContentFormat
             | Self::DotModel
             | Self::InvalidProvider(_)
             | Self::InvalidModel { .. }
@@ -345,6 +352,7 @@ impl Error {
             | Self::InlineDocumentTooLarge
             | Self::Pages(_)
             | Self::Features
+            | Self::OutputContentFormat
             | Self::DotModel
             | Self::InvalidProvider(_)
             | Self::InvalidModel { .. }

litellm-rust/crates/llms/src/base_llm/ocr/transformation.rs

@@ -21,6 +21,7 @@ pub const OCR_RESPONSE_MAX_BYTES: usize = 64 * 1024 * 1024;
 pub const OCR_INLINE_MAX_BYTES: usize = 50 * 1024 * 1024;
 pub const OCR_MAX_FETCH_REDIRECTS: usize = 10;
 pub const OCR_POLL_RETRY_SECS: u64 = 2;
+pub const DROP_PARAMS_PARAM: &str = "drop_params";
 
 #[derive(Clone, Default)]
 pub struct OcrCredentialInputs {
@@ -185,6 +186,10 @@ pub fn response_format(optional_params: &CallArguments) -> Result<OcrResponseFor
         .map(|format| format.unwrap_or_default())
 }
 
+pub fn drops_unsupported_params(optional_params: &CallArguments) -> bool {
+    optional_params.get(DROP_PARAMS_PARAM) == Some(&Value::Bool(true))
+}
+
 #[derive(Debug)]
 pub struct DecodedOcrResponse<T> {
     pub data: T,

litellm-rust/crates/llms/src/bedrock/audio_transcription/mod.rs

@@ -6,8 +6,8 @@ use litellm_auth_aws::{
 };
 use litellm_core_utils::core_helpers::json_type_name;
 use litellm_llms_types::{
-    formats::audio_transcription::AudioTranscriptionResponseData,
-    providers::bedrock::ConverseResponse,
+    formats::audio_transcription::{AudioTranscriptionResponseData, AudioTranscriptionUsage},
+    providers::bedrock::{ConverseResponse, ConverseUsage},
 };
 use serde::Deserialize;
 use serde_json::{Map, Value, json};
@@ -101,10 +101,10 @@ impl BaseAudioTranscriptionConfig for BedrockAudioTranscriptionConfig {
         if let Some(prompt) = optional_string(&optional_params, "prompt") {
             instruction.push_str(&format!(" Additional context: {prompt}"));
         }
-        let mut inference_config = Map::from_iter([("maxTokens".to_string(), json!(4096))]);
-        if let Some(temperature) = optional_params.get("temperature") {
-            inference_config.insert("temperature".to_string(), temperature.clone());
-        }
+        let temperature = optional_params
+            .get("temperature")
+            .cloned()
+            .unwrap_or(json!(0));
         Ok(AudioTranscriptionRequestData {
             body: json!({
                 "messages": [{
@@ -114,8 +114,7 @@ impl BaseAudioTranscriptionConfig for BedrockAudioTranscriptionConfig {
                         {"text": instruction}
                     ]
                 }],
-                "system": [{"text": "You are a transcription assistant."}],
-                "inferenceConfig": inference_config,
+                "inferenceConfig": {"maxTokens": 4096, "temperature": temperature},
             }),
         })
     }
@@ -125,6 +124,7 @@ impl BaseAudioTranscriptionConfig for BedrockAudioTranscriptionConfig {
         _model: &str,
         response_json: Value,
     ) -> Result<AudioTranscriptionResponseData, Error> {
+        let reports_usage = response_json.get("usage").is_some();
         let response: ConverseResponse =
             serde_json::from_value(response_json).map_err(|error| {
                 Error::InvalidResponse(ErrorDetail::invalid("Converse response", error))
@@ -137,6 +137,7 @@ impl BaseAudioTranscriptionConfig for BedrockAudioTranscriptionConfig {
         }
         Ok(AudioTranscriptionResponseData {
             text: response.content_text(),
+            usage: reports_usage.then(|| transcription_usage(&response.usage)),
         })
     }
 
@@ -191,22 +192,41 @@ impl BaseAudioTranscriptionConfig for BedrockAudioTranscriptionConfig {
     }
 }
 
+fn transcription_usage(usage: &ConverseUsage) -> AudioTranscriptionUsage {
+    AudioTranscriptionUsage {
+        input_tokens: usage.input_tokens,
+        output_tokens: usage.output_tokens,
+        total_tokens: usage
+            .total_tokens
+            .unwrap_or(usage.input_tokens + usage.output_tokens),
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
+    use rstest::rstest;
 
     fn no_env(_: &str) -> Option<String> {
         None
     }
 
-    #[test]
-    fn request_matches_python_shape() {
-        let params = Map::from_iter([
-            ("language".to_string(), json!("en")),
-            ("prompt".to_string(), json!("Speaker names")),
-            ("temperature".to_string(), json!(0)),
-            ("timestamp_granularities".to_string(), json!(["word"])),
-        ]);
+    #[rstest]
+    #[case::caller_temperature(Some(json!(0.4)), json!(0.4))]
+    #[case::default_temperature(None, json!(0))]
+    fn request_sends_audio_and_instruction_without_system_block(
+        #[case] temperature: Option<Value>,
+        #[case] expected_temperature: Value,
+    ) {
+        let params = Map::from_iter(
+            [
+                ("language".to_string(), json!("en")),
+                ("prompt".to_string(), json!("Speaker names")),
+                ("timestamp_granularities".to_string(), json!(["word"])),
+            ]
+            .into_iter()
+            .chain(temperature.map(|value| ("temperature".to_string(), value))),
+        );
         let params = BEDROCK_AUDIO_TRANSCRIPTION_CONFIG.map_transcription_params(&params);
         let result = BEDROCK_AUDIO_TRANSCRIPTION_CONFIG
             .transform_audio_transcription_request(
@@ -225,22 +245,39 @@ mod tests {
                         {"text": "Transcribe the audio. Respond with only the transcript. The audio language is en. Additional context: Speaker names"}
                     ]
                 }],
-                "system": [{"text": "You are a transcription assistant."}],
-                "inferenceConfig": {"maxTokens": 4096, "temperature": 0}
+                "inferenceConfig": {"maxTokens": 4096, "temperature": expected_temperature}
             })
         );
     }
 
-    #[test]
-    fn response_concatenates_content_blocks() {
+    #[rstest]
+    #[case::reported_total(
+        json!({"inputTokens": 390, "outputTokens": 60, "totalTokens": 450}),
+        json!({"text": "hello world", "usage": {"type": "tokens", "input_tokens": 390, "output_tokens": 60, "total_tokens": 450}})
+    )]
+    #[case::derived_total(
+        json!({"inputTokens": 1, "outputTokens": 2}),
+        json!({"text": "hello world", "usage": {"type": "tokens", "input_tokens": 1, "output_tokens": 2, "total_tokens": 3}})
+    )]
+    fn response_carries_text_and_converse_usage(#[case] usage: Value, #[case] expected: Value) {
         let result = BEDROCK_AUDIO_TRANSCRIPTION_CONFIG
             .transform_audio_transcription_response(
                 "model",
-                json!({"output": {"message": {"content": [{"text": "hello "}, {"text": "world"}]}}, "usage": {"inputTokens": 1, "outputTokens": 2}}),
+                json!({"output": {"message": {"content": [{"text": "hello "}, {"text": "world"}]}}, "usage": usage}),
             )
             .expect("response");
-        assert_eq!(result.text, "hello world");
-        assert_eq!(result.into_json(), json!({"text": "hello world"}));
+        assert_eq!(result.into_json(), expected);
+    }
+
+    #[rstest]
+    fn response_without_usage_reports_none() {
+        let result = BEDROCK_AUDIO_TRANSCRIPTION_CONFIG
+            .transform_audio_transcription_response(
+                "model",
+                json!({"output": {"message": {"content": [{"text": "hello"}]}}}),
+            )
+            .expect("response");
+        assert_eq!(result.into_json(), json!({"text": "hello"}));
     }
 
     #[test]

litellm-rust/crates/llms/src/vertex_ai/messages/AGENTS.md

@@ -1,6 +1,7 @@
 # rules
 
-- Claude on Vertex AI: reuses `anthropic/messages` shaping, moves the model into the publisher URL, puts the Vertex `anthropic_version` in the body, and authenticates with a Google access token
+- Claude on Vertex AI: reuses `anthropic/messages` shaping, moves the model into the publisher URL, and authenticates with a Google access token
+- Vertex reads the API version only from the body's `anthropic_version` and answers 400 when an `anthropic-version` header names a different one, so the caller's header never reaches Vertex. A version Vertex accepts in the body is carried there, `2023-06-01`, `2023-01-01`, or no header get `vertex-2023-10-16`, and any other version drops under `drop_params` and is refused without it
 - Applies Vertex's beta policy and strips `output_config.effort` for models that do not accept it
 
 # references

litellm-rust/crates/llms/src/vertex_ai/messages/transformation.rs

@@ -3,10 +3,10 @@ use litellm_auth_gcp::{
     VertexConfig, get_vertex_ai_location, get_vertex_ai_project,
     get_vertex_ai_project_from_credentials, secret_names,
 };
-use litellm_http::request::has_header;
+use litellm_http::request::{has_header, header_value, without_headers};
 use litellm_llms_types::{
     formats::messages::{MessagesOptionalParams, MessagesRequest, MessagesTool, OutputConfig},
-    providers::anthropic::{AnthropicBeta, BetaProvider, BetaSet},
+    providers::anthropic::{AnthropicBeta, BetaProvider, BetaSet, VERSION_HEADER},
     recognized::Recognized,
 };
 use litellm_router_types::LitellmParams;
@@ -36,6 +36,8 @@ use litellm_llms_types::providers::vertex_ai::{
 };
 
 const WEB_SEARCH_TOOL_PREFIX: &str = "web_search";
+const VERSIONS_SERVED_AS_VERTEX_DEFAULT: [&str; 3] = ["", "2023-01-01", "2023-06-01"];
+const VERSIONS_VERTEX_ACCEPTS_IN_BODY: [&str; 2] = [ANTHROPIC_VERSION, "2026-10-21"];
 
 /// Claude on Vertex AI, Python's `VertexAIPartnerModelsAnthropicMessagesConfig`: the Anthropic
 /// payload addressed to a project and location, authenticated with a Google access token.
@@ -90,12 +92,6 @@ impl BaseMessagesConfig for VertexAiPartnerModelsAnthropicMessagesConfig {
                     .params
                     .output_config
                     .and_then(|config| sanitize_output_config(config, accepts_effort)),
-                extra: request
-                    .params
-                    .extra
-                    .into_iter()
-                    .chain([("anthropic_version".to_string(), json!(ANTHROPIC_VERSION))])
-                    .collect(),
                 ..request.params
             },
             ..request
@@ -133,7 +129,35 @@ impl BaseMessagesConfig for VertexAiPartnerModelsAnthropicMessagesConfig {
     }
 
     fn request_headers(&self, headers: Headers, request: &MessagesRequest) -> Headers {
-        merge_beta_headers(headers, vertex_feature_betas(request))
+        merge_beta_headers(
+            without_headers(headers, &[VERSION_HEADER]),
+            vertex_feature_betas(request),
+        )
+    }
+
+    fn request_body(
+        &self,
+        request: MessagesRequest,
+        headers: &Headers,
+        drop_params: bool,
+    ) -> Result<MessagesRequest, Error> {
+        let version = vertex_version(
+            &request.model,
+            header_value(headers, VERSION_HEADER),
+            drop_params,
+        )?;
+        Ok(MessagesRequest {
+            params: MessagesOptionalParams {
+                extra: request
+                    .params
+                    .extra
+                    .into_iter()
+                    .chain([("anthropic_version".to_string(), json!(version))])
+                    .collect(),
+                ..request.params
+            },
+            ..request
+        })
     }
 
     /// Vertex addresses the model in the URL and rejects it in the body.
@@ -225,6 +249,26 @@ fn vertex_feature_betas(request: &MessagesRequest) -> BetaSet {
     )
 }
 
+fn vertex_version<'a>(
+    model: &str,
+    requested: Option<&'a str>,
+    drop_params: bool,
+) -> Result<&'a str, Error> {
+    match requested.map(str::trim).unwrap_or_default() {
+        version if VERSIONS_SERVED_AS_VERTEX_DEFAULT.contains(&version) => Ok(ANTHROPIC_VERSION),
+        version if VERSIONS_VERTEX_ACCEPTS_IN_BODY.contains(&version) => Ok(version),
+        _ if drop_params => Ok(ANTHROPIC_VERSION),
+        version => Err(Error::InvalidRequest(
+            crate::ErrorDetail::UnsupportedParameter {
+                model: model.into(),
+                param: VERSION_HEADER.into(),
+                value: version.into(),
+                hint: String::new(),
+            },
+        )),
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use rstest::rstest;
@@ -471,6 +515,28 @@ mod tests {
         );
     }
 
+    #[rstest]
+    #[case::sdk_spelling("anthropic-version")]
+    #[case::any_case("Anthropic-Version")]
+    fn a_forwarded_anthropic_version_never_reaches_vertex(#[case] name: &str) {
+        let request: MessagesRequest = serde_json::from_value(json!({
+            "model": "claude-sonnet-4-5",
+            "max_tokens": 16,
+            "messages": [{"role": "user", "content": "hi"}],
+        }))
+        .unwrap();
+        assert_eq!(
+            VERTEX_ANTHROPIC_MESSAGES_CONFIG.request_headers(
+                vec![
+                    (name.to_string(), "2026-10-21".to_string()),
+                    ("x-request-id".to_string(), "req-1".to_string()),
+                ],
+                &request,
+            ),
+            vec![("x-request-id".to_string(), "req-1".to_string())]
+        );
+    }
+
     fn transformed(value: Value, context: &MessagesTransformContext) -> Value {
         let request: MessagesRequest = serde_json::from_value(value).unwrap();
         serde_json::to_value(
@@ -482,16 +548,87 @@ mod tests {
     }
 
     #[rstest]
-    fn the_body_carries_the_vertex_version_and_the_wire_drops_the_model() {
-        let body = transformed(
-            json!({
-                "model": "claude-sonnet-4-5@20250929",
-                "max_tokens": 16,
-                "messages": [{"role": "user", "content": "hi"}]
-            }),
-            &MessagesTransformContext::default(),
+    #[case::sdk_2(&[("anthropic-version", "2026-10-21")], false, "2026-10-21")]
+    #[case::any_case(&[("Anthropic-Version", "2026-10-21")], false, "2026-10-21")]
+    #[case::vertex_name(&[("anthropic-version", "vertex-2023-10-16")], false, "vertex-2023-10-16")]
+    #[case::sdk_1(&[("anthropic-version", "2023-06-01")], false, "vertex-2023-10-16")]
+    #[case::first_version(&[("anthropic-version", "2023-01-01")], false, "vertex-2023-10-16")]
+    #[case::blank(&[("anthropic-version", " ")], false, "vertex-2023-10-16")]
+    #[case::no_header(&[], false, "vertex-2023-10-16")]
+    #[case::unknown_under_drop_params(&[("anthropic-version", "latest")], true, "vertex-2023-10-16")]
+    fn the_requested_version_goes_in_the_body(
+        #[case] forwarded: &[(&str, &str)],
+        #[case] drop_params: bool,
+        #[case] body_version: &str,
+    ) {
+        let request: MessagesRequest = serde_json::from_value(json!({
+            "model": "claude-sonnet-4-5",
+            "max_tokens": 16,
+            "messages": [{"role": "user", "content": "hi"}],
+        }))
+        .unwrap();
+        let headers: Headers = forwarded
+            .iter()
+            .map(|(name, value)| (name.to_string(), value.to_string()))
+            .collect();
+        let body = serde_json::to_value(
+            VERTEX_ANTHROPIC_MESSAGES_CONFIG
+                .request_body(request, &headers, drop_params)
+                .unwrap(),
+        )
+        .unwrap();
+        assert_eq!(body["anthropic_version"], json!(body_version));
+    }
+
+    #[rstest]
+    #[case::unreleased("2025-01-01")]
+    #[case::alias("latest")]
+    fn an_unsupported_version_is_refused_without_drop_params(#[case] requested: &str) {
+        let request: MessagesRequest = serde_json::from_value(json!({
+            "model": "claude-sonnet-4-5",
+            "max_tokens": 16,
+            "messages": [{"role": "user", "content": "hi"}],
+        }))
+        .unwrap();
+        let headers: Headers = vec![("anthropic-version".to_string(), requested.to_string())];
+        let Error::InvalidRequest(detail) = VERTEX_ANTHROPIC_MESSAGES_CONFIG
+            .request_body(request, &headers, false)
+            .unwrap_err()
+        else {
+            panic!("expected an invalid request");
+        };
+        assert_eq!(
+            detail.to_string(),
+            format!(
+                "claude-sonnet-4-5 does not support anthropic-version={requested}. \
+                 To drop unsupported params, set `litellm.drop_params = True`."
+            )
         );
-        assert_eq!(body["anthropic_version"], json!("vertex-2023-10-16"));
+    }
+
+    #[rstest]
+    fn the_body_carries_the_vertex_version_and_the_wire_drops_the_model() {
+        let request: MessagesRequest = serde_json::from_value(json!({
+            "model": "claude-sonnet-4-5@20250929",
+            "max_tokens": 16,
+            "messages": [{"role": "user", "content": "hi"}]
+        }))
+        .unwrap();
+        let body = serde_json::to_value(
+            VERTEX_ANTHROPIC_MESSAGES_CONFIG
+                .request_body(
+                    VERTEX_ANTHROPIC_MESSAGES_CONFIG
+                        .transform_anthropic_messages_request(
+                            request,
+                            &MessagesTransformContext::default(),
+                        )
+                        .unwrap(),
+                    &Vec::new(),
+                    false,
+                )
+                .unwrap(),
+        )
+        .unwrap();
         assert_eq!(body["model"], json!("claude-sonnet-4-5@20250929"));
         let wire = VERTEX_ANTHROPIC_MESSAGES_CONFIG.wire_body(body);
         assert_eq!(wire.get("model"), None);

litellm-rust/crates/model-catalog-refresh/Cargo.toml

@@ -0,0 +1,19 @@
+[package]
+name = "litellm-model-catalog-refresh"
+version = "0.1.0"
+edition.workspace = true
+license.workspace = true
+repository.workspace = true
+
+[dependencies]
+litellm-http.workspace = true
+litellm-model-catalog.workspace = true
+thiserror.workspace = true
+url.workspace = true
+
+[dev-dependencies]
+litellm-error.workspace = true
+litellm-http = { workspace = true, features = ["test-support"] }
+reqwest.workspace = true
+rstest.workspace = true
+tokio.workspace = true

litellm-rust/crates/model-catalog-refresh/src/error.rs

@@ -0,0 +1,105 @@
+use std::io;
+
+use thiserror::Error;
+
+#[derive(Debug, Error)]
+#[error(transparent)]
+pub struct Error(Box<Repr>);
+
+#[derive(Debug, Error)]
+enum Repr {
+    #[error(transparent)]
+    Http(#[from] litellm_http::Error),
+    #[error("could not read the catalog")]
+    Read(#[source] io::Error),
+    #[error("invalid catalog URL")]
+    InvalidUrl(#[source] url::ParseError),
+    #[error("unsupported catalog URL scheme {scheme}")]
+    UnsupportedScheme { scheme: String },
+}
+
+impl Error {
+    pub fn unsupported_scheme(scheme: String) -> Self {
+        Self(Box::new(Repr::UnsupportedScheme { scheme }))
+    }
+}
+
+impl From<litellm_http::Error> for Error {
+    fn from(cause: litellm_http::Error) -> Self {
+        Self(Box::new(Repr::Http(cause)))
+    }
+}
+
+impl From<io::Error> for Error {
+    fn from(cause: io::Error) -> Self {
+        Self(Box::new(Repr::Read(cause)))
+    }
+}
+
+impl From<url::ParseError> for Error {
+    fn from(cause: url::ParseError) -> Self {
+        Self(Box::new(Repr::InvalidUrl(cause)))
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use litellm_error::{Report, find};
+    use rstest::rstest;
+
+    use super::*;
+    use crate::{DoneReason, Reply, Step};
+
+    #[rstest]
+    fn exhausted_transport_keeps_its_typed_cause() {
+        let reply = Reply::Transport {
+            error: io::Error::from(io::ErrorKind::ConnectionReset).into(),
+        };
+        let Reply::Transport { error } = reply else {
+            panic!("transport failure")
+        };
+        let step = Step::Done {
+            reason: DoneReason::Exhausted(error),
+        };
+        let Step::Done {
+            reason: DoneReason::Exhausted(error),
+        } = step
+        else {
+            panic!("exhausted attempts")
+        };
+        let cause = find::<io::Error>(&error).expect("typed I/O cause");
+        assert_eq!(cause.kind(), io::ErrorKind::ConnectionReset);
+        assert_eq!(
+            Report::new(&error)
+                .to_string()
+                .matches(&cause.to_string())
+                .count(),
+            1
+        );
+        assert!(!error.to_string().contains(&cause.to_string()));
+    }
+
+    #[rstest]
+    fn invalid_source_keeps_its_typed_cause() {
+        let error = Error::from(url::Url::parse("http://[invalid").unwrap_err());
+        assert!(find::<url::ParseError>(&error).is_some());
+    }
+
+    #[rstest]
+    #[tokio::test]
+    async fn http_failures_keep_the_cause_without_the_url() {
+        let cause = litellm_http::Client::for_test()
+            .get("http://localhost:invalid/private?api_key=secret")
+            .send()
+            .await
+            .unwrap_err();
+        let error = Error::from(litellm_http::Error::from(cause));
+        assert!(find::<reqwest::Error>(&error).is_some());
+        let report = Report::new(&error).to_string();
+        assert!(!report.contains("secret"), "{report}");
+        assert!(!report.contains("private"), "{report}");
+        let debug = format!("{error:?}");
+        assert!(!debug.contains("secret"), "{debug}");
+        assert!(!debug.contains("private"), "{debug}");
+    }
+}

litellm-rust/crates/model-catalog-refresh/src/lib.rs

@@ -0,0 +1,9 @@
+mod error;
+mod policy;
+mod source;
+mod step;
+
+pub use error::Error;
+pub use policy::*;
+pub use source::*;
+pub use step::*;

litellm-rust/crates/model-catalog-refresh/src/policy.rs

@@ -0,0 +1,20 @@
+use std::time::Duration;
+
+use litellm_model_catalog::IntegrityLimits;
+
+/// Main's defaults, 3 attempts, 30 s max wait, 5 s timeout, backoff base 2.0, jitter 0.0 to 1.0 s
+#[derive(Clone, Copy, Debug, PartialEq)]
+pub struct RetryPolicy {
+    pub max_attempts: u32,
+    pub max_wait: Duration,
+    pub timeout: Duration,
+    pub backoff_base_seconds: f64,
+    pub jitter_min_seconds: f64,
+    pub jitter_max_seconds: f64,
+    /// Main's 50 model floor and 0.5 shrink ratio
+    pub limits: IntegrityLimits,
+}
+
+/// Main's default `LITELLM_MODEL_COST_MAP_URL`
+pub const DEFAULT_REGISTRY_URL: &str =
+    "https://raw.githubusercontent.com/BerriAI/litellm/main/model_prices_and_context_window.json";

litellm-rust/crates/model-catalog-refresh/src/source.rs

@@ -0,0 +1,17 @@
+use std::path::PathBuf;
+
+use url::Url;
+
+/// Where a fetched catalog comes from
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub enum Source {
+    Http(Url),
+    File(PathBuf),
+    S3(S3Object),
+}
+
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct S3Object {
+    pub bucket: String,
+    pub key: String,
+}

litellm-rust/crates/model-catalog-refresh/src/step.rs

@@ -0,0 +1,66 @@
+use std::time::Duration;
+
+use litellm_model_catalog::Catalog;
+
+use crate::Error;
+use crate::source::Source;
+
+/// One attempt. Where to fetch from is the `Source`, so the request carries no URL
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct FetchRequest {
+    pub if_none_match: Option<String>,
+    pub timeout: Duration,
+}
+
+#[derive(Debug)]
+pub enum Reply {
+    /// An HTTP, S3 or file answer. The body is read only on 200
+    Response {
+        status: u16,
+        etag: Option<String>,
+        retry_after: Option<Duration>,
+        body: Option<Vec<u8>>,
+    },
+    /// No answer reached us
+    Transport { error: Error },
+}
+
+/// What the fetch state machine asks its driver to do next
+#[derive(Debug)]
+pub enum Step {
+    Fetch(FetchRequest),
+    Sleep(Duration),
+    /// A parsed document to install, with its rejected row errors
+    Adopt {
+        catalog: Box<Catalog>,
+        installed: Installed,
+        errors: Vec<litellm_model_catalog::Error>,
+    },
+    /// 304, a final status, or retries exhausted
+    Done {
+        reason: DoneReason,
+    },
+}
+
+#[derive(Debug)]
+pub enum DoneReason {
+    NotModified,
+    Status(u16),
+    Exhausted(Error),
+}
+
+/// What is installed now, the input to the next revalidation
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct Installed {
+    pub origin: Origin,
+    pub etag: Option<String>,
+    /// `git_blob_id` of the installed bytes
+    pub revision: String,
+    pub model_count: usize,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub enum Origin {
+    Bundled,
+    Remote { source: Source },
+}

litellm-rust/crates/model-catalog/AGENTS.md

@@ -1,5 +1,29 @@
-## Validation
+## Layout
 
-For `model_prices_and_context_window.json` validation, we should eventually:
+`wire/` reads one entry as written. `EntryWire::read` makes one pass over the entry's keys and puts every non-null key in exactly one place, which is an attribute, a flat price parsed by `FlatKey::parse`, a price block, a `supports_*` capability or `unknown`. A known key with a bad value becomes an `Error` that names the key, and the rest of the entry is still read
 
-- Stop skipping this crate's tests
+`lower.rs` is the only code that knows both the wire types and `litellm-cost`. It lowers an `EntryWire` into `Prices`, `Adjustments` and `Model` and checks the rules that span keys, such as one price per meter and condition, contiguous tiers, no suffix inside a block, and off-peak needing hours and prices. Lowering only goes one way. Nothing turns `Prices` back into JSON, since the raw fields are kept
+
+`catalog.rs` reads the document, indexes aliases and runs the checks across entries. A bad entry keeps its raw fields and its errors are listed in `Loaded.errors`. Deciding what to do with them is the caller's job. `Catalog::written` gives back every top-level key and value as the document wrote them, in document order. `Catalog::served` gives back every model and alias as Python serves it, without `sample_spec` and `fallback_generalizations`
+
+`Billing` on `Prices` holds the row attributes pricing reads (text unit, band edge, cache-read fallback, wall clock, web search unit, tier scope). `lower.rs` derives it from the row and its `litellm_provider`, so `litellm-cost` holds no provider list
+
+`store/` is the catalog's in-memory store and the only place that indexes model names. Its rules are in [src/store/AGENTS.md](src/store/AGENTS.md)
+
+`layers.rs`, `resolve/`, `model_info.rs` and `register.rs` hold types only. The functions over them land with their first caller
+
+Catalog errors use one opaque `Error` with typed causes and JSON Pointer accessors (`path`, `key`, `field`). `Error::rejects` alone decides whether an error fails the document, rejects its row or neither, and `ErrorKind` is the stable name hosts report it by
+
+## Prices
+
+The price vocabulary lives in `litellm-cost`, and only this crate knows JSON key names. Every price is keyed by `Meter`, `Channel` and `When`. A variation only one meter has stays in the meter (cache TTL, image size, resolution, `_above_Ns_interval`, `max_results_range`). Service tier and prompt band make up `When`. Batch and decisions are a `Channel` because Python bills them with their own rules. Off-peak is an overlay on `Prices`, and regional uplifts are `Adjustments`, never prices
+
+A new `N`, image size or resolution in an existing key shape parses with no Rust change. A new unit, token kind, feature or tier needs a variant in `litellm-cost` plus its spelling in `wire/flat_key.rs` and `FLAT_KEY_PATTERN` in `schema.rs`
+
+`Prices::band` and `Prices::rate` are pinned to Python's `_get_token_base_cost` by `tests/price_lookup_reference.json`. After a billing change in Python, regenerate it from the repo root with `PYTHONPATH=. python litellm-rust/crates/model-catalog/tests/generate_price_lookup_reference.py`
+
+## Tests
+
+`BUNDLED_REGISTRY` embeds the root `model_prices_and_context_window.json`, so the tests over it fail a JSON change with a bad value or a key no type reads
+
+`Catalog::problems` is the one definition of a mergeable cost map. The bundled test asserts it is empty, and the cost map guard workflow runs it on a pull request's JSON through `cargo run --bin check-registry <path>`

litellm-rust/crates/model-catalog/Cargo.toml

@@ -6,19 +6,23 @@ license.workspace = true
 repository.workspace = true
 
 [features]
-schema = ["dep:schemars", "litellm-llms-types/schema"]
+schema = ["dep:schemars", "litellm-cost/schema"]
 
 [dependencies]
-litellm-llms-types.workspace = true
+litellm-cost = { workspace = true, features = ["serde"] }
+litellm-error.workspace = true
 
 indexmap = { version = "2.14.0", features = ["serde"] }
+jiff = { version = "0.2.37", default-features = false, features = ["std", "tzdb-bundle-always"] }
+macro_rules_attribute.workspace = true
 schemars = { workspace = true, optional = true }
 serde.workspace = true
-serde_json.workspace = true
+serde_json = { workspace = true, features = ["preserve_order"] }
+serde_with.workspace = true
+strum.workspace = true
 thiserror.workspace = true
 time.workspace = true
 
 [dev-dependencies]
 jsonschema = { version = "0.55.1", default-features = false }
-litellm-error.workspace = true
 rstest.workspace = true

litellm-rust/crates/model-catalog/src/bin/check-registry.rs

@@ -0,0 +1,26 @@
+use std::{env, fs, path::PathBuf, process::ExitCode};
+
+use litellm_model_catalog::Catalog;
+
+fn main() -> ExitCode {
+    let Some(path) = env::args_os().nth(1).map(PathBuf::from) else {
+        eprintln!("usage: check-registry <model_prices_and_context_window.json>");
+        return ExitCode::from(2);
+    };
+    let body = match fs::read(&path) {
+        Ok(body) => body,
+        Err(error) => {
+            eprintln!("{}: {error}", path.display());
+            return ExitCode::FAILURE;
+        }
+    };
+    let problems = Catalog::problems(&body);
+    for problem in &problems {
+        eprintln!("{}", litellm_error::Report::new(problem));
+    }
+    if problems.is_empty() {
+        ExitCode::SUCCESS
+    } else {
+        ExitCode::FAILURE
+    }
+}

litellm-rust/crates/model-catalog/src/bundled.rs

@@ -0,0 +1,3 @@
+/// The registry document this build serves when no other one is loaded.
+pub const BUNDLED_REGISTRY: &[u8] =
+    include_bytes!("../../../../model_prices_and_context_window.json");

litellm-rust/crates/model-catalog/src/capabilities.rs

@@ -1,67 +0,0 @@
-use serde::{Deserialize, Serialize};
-
-/// Primary API surface / task type of the model.
-#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)]
-#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
-#[serde(rename_all = "snake_case")]
-pub enum Mode {
-    AudioSpeech,
-    AudioTranscription,
-    Chat,
-    Completion,
-    Decisions,
-    Embedding,
-    Evaluation,
-    Guardrail,
-    ImageEdit,
-    ImageGeneration,
-    Moderation,
-    Ocr,
-    Realtime,
-    Rerank,
-    Responses,
-    Search,
-    VectorStore,
-    VideoGeneration,
-}
-
-/// Gemini audio generation API the model is served through.
-#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)]
-#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
-#[serde(rename_all = "snake_case")]
-pub enum VertexAiAudioApi {
-    LyriaPredict,
-    LyriaInteractions,
-}
-
-/// Audio container format the model can return.
-#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)]
-#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
-#[serde(rename_all = "snake_case")]
-pub enum AudioFormat {
-    Mp3,
-    Wav,
-}
-
-/// Input modality the model accepts.
-#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)]
-#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
-#[serde(rename_all = "snake_case")]
-pub enum InputModality {
-    Text,
-    Image,
-    Audio,
-    Video,
-}
-
-/// Output modality the model can produce.
-#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)]
-#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
-#[serde(rename_all = "snake_case")]
-pub enum OutputModality {
-    Text,
-    Image,
-    Audio,
-    Video,
-    Code,
-}

litellm-rust/crates/model-catalog/src/catalog.rs

@@ -1,54 +1,29 @@
-use crate::error::Error;
-use crate::fallback::{FallbackGeneralizations, FallbackRule};
-use crate::model_info::ModelInfo;
+use std::borrow::Cow;
+use std::collections::HashSet;
+
 use indexmap::IndexMap;
 use serde::Deserialize;
-use serde_json::{Map, Value};
-use std::collections::HashMap;
+use serde_json::Value;
 
-#[derive(Clone, Debug, Default, PartialEq, Eq)]
-pub struct Provenance {
-    pub source: Option<String>,
-    pub revision: Option<String>,
-    pub etag: Option<String>,
-}
+use crate::error::{Error, Loaded, Rejects, Repr, pointer_child, pointer_for_key};
+use crate::fallback::{FallbackGeneralizations, FallbackRule};
+use crate::model::Model;
+use crate::store::{Rows, Store};
+use crate::wire::{EntryWire, Fields};
+
+const SAMPLE_SPEC: &str = "sample_spec";
+const FALLBACK_GENERALIZATIONS: &str = "fallback_generalizations";
 
 #[derive(Clone, Copy, Debug, PartialEq)]
 pub struct IntegrityLimits {
-    pub reference_model_count: usize,
-    pub min_model_count: usize,
-    pub min_reference_ratio: f64,
-}
-
-#[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)]
-pub enum AliasIssue {
-    #[error("model {model:?} has aliases that are not an array")]
-    InvalidList { model: String },
-    #[error("model {model:?} has an alias that is not a string")]
-    InvalidName { model: String },
-    #[error("alias {alias:?} of model {model:?} is already a model name")]
-    CanonicalCollision { model: String, alias: String },
-    #[error("alias {alias:?} of model {model:?} is already an alias of another model")]
-    AliasCollision { model: String, alias: String },
+    min_model_count: usize,
+    min_reference_ratio: f64,
 }
 
 #[derive(Clone, Debug)]
 pub struct ModelEntry {
-    fields: Map<String, Value>,
-    info: ModelInfo,
-}
-
-impl ModelEntry {
-    pub fn field(&self, name: &str) -> Option<&Value> {
-        self.fields.get(name)
-    }
-    pub fn fields(&self) -> &Map<String, Value> {
-        &self.fields
-    }
-    /// The entry deserialized into the typed mirror of the catalog schema.
-    pub fn info(&self) -> &ModelInfo {
-        &self.info
-    }
+    fields: Fields,
+    model: Option<Model>,
 }
 
 #[derive(Clone, Copy, Debug)]
@@ -58,142 +33,342 @@ pub struct ModelMatch<'a> {
     pub entry: &'a ModelEntry,
 }
 
-#[derive(Debug)]
+#[derive(Clone, Debug)]
+struct Section {
+    index: usize,
+    key: String,
+    value: Value,
+}
+
+/// One fetched key kept off the served map because its row failed validation
+#[derive(Clone, Debug)]
+pub struct Rejection<'a> {
+    pub key: String,
+    pub errors: Vec<&'a Error>,
+    /// The bundled row for the key is served instead. `false` means the key is left out
+    pub replaced_by_bundled: bool,
+}
+
+#[derive(Clone, Copy, Debug, serde::Serialize)]
+#[serde(untagged)]
+pub enum Written<'a> {
+    Entry(&'a Fields),
+    Section(&'a Value),
+}
+
+#[derive(Clone, Debug)]
 pub struct Catalog {
+    store: Store<CatalogRows>,
+    sections: Vec<Section>,
+    fallback_generalizations: Option<FallbackGeneralizations>,
+}
+
+/// Models and aliases, an alias served as its owner's row
+#[derive(Clone, Debug)]
+struct CatalogRows {
     entries: IndexMap<String, ModelEntry>,
     aliases: IndexMap<String, String>,
-    lowercase_keys: HashMap<String, String>,
-    sample_spec: Option<Value>,
-    fallback_generalizations: Option<FallbackGeneralizations>,
-    provenance: Provenance,
-    alias_issues: Vec<AliasIssue>,
+}
+
+impl CatalogRows {
+    fn owner<'a>(&'a self, key: &'a str) -> &'a str {
+        self.aliases.get(key).map_or(key, String::as_str)
+    }
+
+    fn fields(&self, key: &str) -> Option<&Fields> {
+        Some(self.entries.get(self.owner(key))?.fields())
+    }
+}
+
+/// Model names match ignoring case, and keys are grouped by their raw `litellm_provider`
+impl Rows for CatalogRows {
+    type Row = Fields;
+    type Stamp = ();
+
+    fn fold(name: &str) -> String {
+        name.to_lowercase()
+    }
+
+    fn stamp(&self) -> Self::Stamp {}
+
+    fn contains(&self, key: &str) -> bool {
+        self.entries.contains_key(key) || self.aliases.contains_key(key)
+    }
+
+    fn row(&self, key: &str) -> Option<Cow<'_, Fields>> {
+        self.fields(key).map(Cow::Borrowed)
+    }
+
+    fn scan(&self, visit: &mut dyn FnMut(&str, Option<&str>)) {
+        let keys = self.entries.keys().chain(self.aliases.keys());
+        for key in keys {
+            let provider = self
+                .fields(key)
+                .and_then(|fields| fields.get("litellm_provider")?.as_str());
+            visit(key, provider);
+        }
+    }
+}
+
+impl IntegrityLimits {
+    pub fn new(min_model_count: usize, min_reference_ratio: f64) -> Option<Self> {
+        (0.0..=1.0).contains(&min_reference_ratio).then_some(Self {
+            min_model_count,
+            min_reference_ratio,
+        })
+    }
+}
+
+impl ModelEntry {
+    pub fn field(&self, name: &str) -> Option<&Value> {
+        self.fields.get(name)
+    }
+
+    pub fn fields(&self) -> &Fields {
+        &self.fields
+    }
+
+    pub fn model(&self) -> Option<&Model> {
+        self.model.as_ref()
+    }
 }
 
 impl Catalog {
-    pub fn parse(body: &[u8], provenance: Provenance) -> Result<Self, Error> {
-        let root: IndexMap<String, Value> = serde_json::from_slice(body)?;
-        if root.is_empty() {
-            return Err(Error::Empty);
-        }
+    pub fn parse(body: &[u8]) -> Result<Loaded<Self>, Error> {
+        let document = serde_json::from_slice::<Fields>(body)
+            .map_err(|error| Error::new("", Repr::Json(error)))?;
 
-        let mut entries = IndexMap::with_capacity(root.len());
+        let mut problems = Vec::new();
+        let mut sections = Vec::new();
+        let mut entries = IndexMap::with_capacity(document.len());
         let mut alias_lists = Vec::new();
-        let mut alias_issues = Vec::new();
-        let mut sample_spec = None;
-        let mut fallback_generalizations = None;
-        for (name, value) in root {
-            match name.as_str() {
-                "sample_spec" => {
-                    sample_spec = Some(value);
+        for (index, (name, value)) in document.into_iter().enumerate() {
+            let fields = match value {
+                Value::Object(fields) if !is_section(&name) => fields,
+                value => {
+                    if !is_section(&name) {
+                        problems.push(Error::from_key(&name, Repr::NotAnObject));
+                    }
+                    sections.push(Section {
+                        index,
+                        key: name,
+                        value,
+                    });
                     continue;
                 }
-                "fallback_generalizations" => {
-                    fallback_generalizations =
-                        Some(serde_json::from_value::<FallbackGeneralizations>(value)?);
-                    continue;
-                }
-                _ => {}
-            }
-            let Value::Object(mut fields) = value else {
-                return Err(Error::EntryNotObject { model: name });
             };
-            let info = ModelInfo::deserialize(&fields)?;
-            if let Some(aliases) = fields.remove("aliases") {
-                match aliases {
-                    Value::Array(names) => alias_lists.push((name.clone(), names)),
-                    _ => alias_issues.push(AliasIssue::InvalidList {
-                        model: name.clone(),
-                    }),
-                }
+            let (model, aliases, row_problems) = read_row(&fields);
+            problems.extend(
+                row_problems
+                    .into_iter()
+                    .map(|problem| problem.prepend(&name)),
+            );
+            if let Some(aliases) = aliases {
+                alias_lists.push((name.clone(), aliases));
             }
-            entries.insert(name, ModelEntry { fields, info });
+            entries.insert(name, ModelEntry { fields, model });
+        }
+        if entries.is_empty() {
+            return Err(Error::new("", Repr::Empty));
         }
 
-        let mut aliases = IndexMap::new();
+        let mut aliases: IndexMap<String, String> = IndexMap::new();
         for (model, names) in alias_lists {
-            for name in names {
-                let Value::String(alias) = name else {
-                    alias_issues.push(AliasIssue::InvalidName {
-                        model: model.clone(),
-                    });
-                    continue;
-                };
+            for alias in names {
                 if entries.contains_key(&alias) {
-                    alias_issues.push(AliasIssue::CanonicalCollision {
-                        model: model.clone(),
-                        alias,
-                    });
-                } else if aliases.contains_key(&alias) {
-                    alias_issues.push(AliasIssue::AliasCollision {
-                        model: model.clone(),
-                        alias,
-                    });
+                    problems.push(Error::from_key(&model, Repr::AliasShadowsModel { alias }));
+                } else if let Some(taken_by) = aliases.get(&alias) {
+                    problems.push(Error::from_key(
+                        &model,
+                        Repr::AliasTaken {
+                            alias,
+                            taken_by: taken_by.clone(),
+                        },
+                    ));
                 } else {
                     aliases.insert(alias, model.clone());
                 }
             }
         }
 
-        let lowercase_keys = entries
-            .keys()
-            .chain(aliases.keys())
-            .map(|key| (key.to_lowercase(), key.clone()))
-            .collect();
-        Ok(Self {
-            entries,
-            aliases,
-            lowercase_keys,
-            sample_spec,
-            fallback_generalizations,
-            provenance,
-            alias_issues,
+        let fallback_generalizations = sections
+            .iter()
+            .find(|section| section.key == FALLBACK_GENERALIZATIONS)
+            .and_then(|section| {
+                FallbackGeneralizations::deserialize(&section.value)
+                    .map_err(|error| {
+                        problems.push(Error::from_key(&section.key, Repr::Invalid(error)))
+                    })
+                    .ok()
+            });
+        Ok(Loaded {
+            value: Self {
+                store: Store::new(CatalogRows { entries, aliases }),
+                sections,
+                fallback_generalizations,
+            },
+            errors: problems,
         })
     }
 
-    pub fn validate(&self, limits: IntegrityLimits) -> Result<(), Error> {
-        if !limits.min_reference_ratio.is_finite()
-            || !(0.0..=1.0).contains(&limits.min_reference_ratio)
+    pub fn problems(body: &[u8]) -> Vec<Error> {
+        let Loaded {
+            value: catalog,
+            errors,
+        } = match Self::parse(body) {
+            Ok(loaded) => loaded,
+            Err(fatal) => return vec![fatal],
+        };
+        let unread = catalog.rows().entries.iter().flat_map(|(model, entry)| {
+            EntryWire::read(entry.fields())
+                .unknown
+                .into_iter()
+                .map(move |(key, _)| {
+                    Error::new(
+                        pointer_child(&pointer_for_key(model), &key),
+                        Repr::UnreadField,
+                    )
+                })
+        });
+        let no_fallback_rules = catalog
+            .fallback_rules()
+            .is_none_or(<[_]>::is_empty)
+            .then(|| Error::from_key(FALLBACK_GENERALIZATIONS, Repr::NoFallbackRules));
+        errors
+            .into_iter()
+            .chain(unread)
+            .chain(no_fallback_rules)
+            .collect()
+    }
+
+    pub fn row_errors(model: &str, fields: &Fields) -> Vec<Error> {
+        let (_, _, errors) = read_row(fields);
+        errors
+            .into_iter()
+            .map(|error| error.prepend(model))
+            .collect()
+    }
+
+    pub fn written(&self) -> impl Iterator<Item = (&str, Written<'_>)> {
+        let mut sections = self.sections.iter().peekable();
+        let mut entries = self.rows().entries.iter();
+        (0..self.rows().entries.len() + self.sections.len()).map_while(move |index| match sections
+            .next_if(|section| section.index == index)
         {
-            return Err(Error::InvalidRatio);
-        }
-        let actual = self.entries.len();
+            Some(section) => Some((section.key.as_str(), Written::Section(&section.value))),
+            None => entries
+                .next()
+                .map(|(name, entry)| (name.as_str(), Written::Entry(&entry.fields))),
+        })
+    }
+
+    pub fn served(&self) -> impl Iterator<Item = (&str, &Fields)> {
+        let rows = self.rows();
+        rows.entries
+            .iter()
+            .map(|(name, entry)| (name.as_str(), entry.fields()))
+            .chain(
+                rows.aliases
+                    .iter()
+                    .map(|(alias, owner)| (alias.as_str(), rows.entries[owner].fields())),
+            )
+    }
+
+    pub fn check_size(&self, reference: usize, limits: IntegrityLimits) -> Result<(), Error> {
+        let actual = self.model_count();
         if actual < limits.min_model_count {
-            return Err(Error::BelowMinimum {
-                actual,
-                minimum: limits.min_model_count,
-            });
+            return Err(Error::new(
+                "",
+                Repr::BelowMinimum {
+                    actual,
+                    minimum: limits.min_model_count,
+                },
+            ));
         }
-        if limits.reference_model_count > 0
-            && (actual as f64) < (limits.reference_model_count as f64) * limits.min_reference_ratio
-        {
-            return Err(Error::Shrunk {
-                actual,
-                reference: limits.reference_model_count,
-                ratio: limits.min_reference_ratio,
-            });
+        if (actual as f64) < (reference as f64) * limits.min_reference_ratio {
+            return Err(Error::new(
+                "",
+                Repr::Shrunk {
+                    actual,
+                    reference,
+                    ratio: limits.min_reference_ratio,
+                },
+            ));
         }
         Ok(())
     }
 
-    pub fn lookup(&self, key: &str) -> Option<ModelMatch<'_>> {
-        let matched_key = if self.entries.contains_key(key) || self.aliases.contains_key(key) {
-            key
-        } else {
-            self.lowercase_keys.get(&key.to_lowercase())?.as_str()
-        };
-        let canonical_key = self
-            .aliases
-            .get(matched_key)
-            .map(String::as_str)
-            .unwrap_or(matched_key);
-        let (canonical_key, entry) = self.entries.get_key_value(canonical_key)?;
-        let matched_key = self
+    pub fn without_rejected(self, errors: &[Error]) -> (Self, Vec<&Error>) {
+        let row_names: HashSet<String> = self
+            .rows()
             .entries
-            .get_key_value(matched_key)
+            .keys()
+            .cloned()
+            .chain(
+                self.sections
+                    .iter()
+                    .filter(|section| !is_section(&section.key))
+                    .map(|section| section.key.clone()),
+            )
+            .collect();
+        let rejected: HashSet<String> = errors
+            .iter()
+            .filter(|error| error.rejects() == Rejects::Row)
+            .filter_map(|error| crate::error::pointer_root(error.path()))
+            .filter(|root| row_names.contains(root))
+            .collect();
+        let removed_errors = errors
+            .iter()
+            .filter(|error| {
+                crate::error::pointer_root(error.path())
+                    .is_some_and(|root| rejected.contains(&root))
+            })
+            .collect();
+        let removed_positions: HashSet<usize> = self
+            .written()
+            .enumerate()
+            .filter(|(_, (key, _))| rejected.contains(*key))
+            .map(|(index, _)| index)
+            .collect();
+        let mut catalog = self;
+        catalog.store.edit(|rows| {
+            rows.entries.retain(|name, _| !rejected.contains(name));
+            rows.aliases.retain(|_, owner| !rejected.contains(owner));
+        });
+        catalog
+            .sections
+            .retain(|section| is_section(&section.key) || !rejected.contains(&section.key));
+        for section in &mut catalog.sections {
+            let removed_before = removed_positions
+                .iter()
+                .filter(|index| **index < section.index)
+                .count();
+            section.index -= removed_before;
+        }
+        (catalog, removed_errors)
+    }
+
+    pub fn unrecognized(&self) -> impl Iterator<Item = (&str, &str)> {
+        self.rows().entries.iter().flat_map(|(name, entry)| {
+            entry
+                .model
+                .iter()
+                .flat_map(|model| &model.unrecognized)
+                .map(move |key| (name.as_str(), key.as_str()))
+        })
+    }
+
+    pub fn lookup(&self, key: &str) -> Option<ModelMatch<'_>> {
+        let rows = self.rows();
+        let found = self.store.find_key(key)?;
+        let (canonical_key, entry) = rows.entries.get_key_value(rows.owner(&found))?;
+        let matched_key = rows
+            .entries
+            .get_key_value(found.as_str())
             .map(|(key, _)| key.as_str())
             .or_else(|| {
-                self.aliases
-                    .get_key_value(matched_key)
+                rows.aliases
+                    .get_key_value(found.as_str())
                     .map(|(key, _)| key.as_str())
             })?;
         Some(ModelMatch {
@@ -204,30 +379,140 @@ impl Catalog {
     }
 
     pub fn model_count(&self) -> usize {
-        self.entries.len()
+        self.rows().entries.len()
     }
+
     pub fn model_names(&self) -> impl Iterator<Item = &str> {
-        self.entries.keys().map(String::as_str)
-    }
-    pub fn alias_count(&self) -> usize {
-        self.aliases.len()
+        self.rows().entries.keys().map(String::as_str)
     }
+
     pub fn aliases(&self) -> &IndexMap<String, String> {
-        &self.aliases
+        &self.rows().aliases
     }
-    pub fn alias_issues(&self) -> &[AliasIssue] {
-        &self.alias_issues
+
+    fn rows(&self) -> &CatalogRows {
+        self.store.rows()
     }
+
     pub fn sample_spec(&self) -> Option<&Value> {
-        self.sample_spec.as_ref()
-    }
-    pub fn fallback_generalizations(&self) -> Option<&FallbackGeneralizations> {
-        self.fallback_generalizations.as_ref()
+        self.sections
+            .iter()
+            .find(|section| section.key == SAMPLE_SPEC)
+            .map(|section| &section.value)
     }
+
     pub fn fallback_rules(&self) -> Option<&[FallbackRule]> {
         Some(self.fallback_generalizations.as_ref()?.rules.as_slice())
     }
-    pub fn provenance(&self) -> &Provenance {
-        &self.provenance
+}
+
+fn read_row(fields: &Fields) -> (Option<Model>, Option<Vec<String>>, Vec<Error>) {
+    let wire = EntryWire::read(fields);
+    let aliases = wire.attributes.aliases.clone();
+    let timezone_error = unknown_timezone(&wire);
+    match Model::try_from(wire) {
+        Ok(model) => (Some(model), aliases, timezone_error.into_iter().collect()),
+        Err(mut errors) => {
+            errors.extend(timezone_error);
+            (None, aliases, errors)
+        }
+    }
+}
+
+fn is_section(key: &str) -> bool {
+    key == SAMPLE_SPEC || key == FALLBACK_GENERALIZATIONS
+}
+
+fn unknown_timezone(wire: &EntryWire) -> Option<Error> {
+    let zone = wire.off_peak.as_ref()?.weekday_timezone.as_deref()?.trim();
+    (!zone.is_empty() && jiff::tz::TimeZone::get(zone).is_err()).then(|| {
+        Error::from_path(
+            "off_peak_pricing.weekday_timezone",
+            Repr::UnknownTimezone {
+                zone: zone.to_owned(),
+            },
+        )
+    })
+}
+
+#[cfg(test)]
+mod tests {
+    use rstest::rstest;
+
+    use super::*;
+
+    const RULES: &str = r#""fallback_generalizations":{"rules":[{"name":"family","pattern":"^new-","model_info":{"mode":"chat"}}]}"#;
+
+    #[rstest]
+    #[case::clean(format!(r#"{{{RULES},"m":{{"litellm_provider":"test"}}}}"#), vec![])]
+    #[case::not_json("{".to_owned(), vec![("", "json")])]
+    #[case::row_error(format!(r#"{{{RULES},"m":{{}}}}"#), vec![("/m/litellm_provider", "missing_provider")])]
+    #[case::unread_key(
+        format!(r#"{{{RULES},"m":{{"litellm_provider":"test","typo_cost":1}}}}"#),
+        vec![("/m/typo_cost", "unread_field")]
+    )]
+    #[case::empty_model_key(
+        format!(r#"{{{RULES},"":{{"litellm_provider":"test","future_field":true}}}}"#),
+        vec![("//future_field", "unread_field")]
+    )]
+    #[case::unread_key_on_invalid_row(
+        format!(r#"{{{RULES},"m":{{"typo_cost":1}}}}"#),
+        vec![("/m/litellm_provider", "missing_provider"), ("/m/typo_cost", "unread_field")]
+    )]
+    #[case::escaped_unread_key_on_invalid_row(
+        format!(r#"{{{RULES},"open/router~v1":{{"typo/cost~":1}}}}"#),
+        vec![("/open~1router~0v1/litellm_provider", "missing_provider"), ("/open~1router~0v1/typo~1cost~0", "unread_field")]
+    )]
+    #[case::no_rules(
+        r#"{"m":{"litellm_provider":"test"}}"#.to_owned(),
+        vec![("/fallback_generalizations", "no_fallback_rules")]
+    )]
+    #[case::empty_rules(
+        r#"{"fallback_generalizations":{"rules":[]},"m":{"litellm_provider":"test"}}"#.to_owned(),
+        vec![("/fallback_generalizations", "no_fallback_rules")]
+    )]
+    fn problems_list_what_a_cost_map_change_must_fix(
+        #[case] body: String,
+        #[case] expected: Vec<(&str, &str)>,
+    ) {
+        let problems: Vec<_> = Catalog::problems(body.as_bytes())
+            .iter()
+            .map(|problem| (problem.path().to_owned(), <&str>::from(problem.kind())))
+            .collect();
+        let expected: Vec<_> = expected
+            .into_iter()
+            .map(|(path, kind)| (path.to_owned(), kind))
+            .collect();
+        assert_eq!(problems, expected);
+    }
+
+    #[rstest]
+    fn runtime_parsing_preserves_custom_fields_and_recovers_valid_rows() {
+        let body = br#"{
+            "good":{"litellm_provider":"test","custom_metadata":true},
+            "bad":{"litellm_provider":"test","input_cost_per_token":"wrong"}
+        }"#;
+        let loaded = Catalog::parse(body).unwrap();
+        assert_eq!(loaded.errors.len(), 1);
+        assert_eq!(loaded.errors[0].path(), "/bad/input_cost_per_token");
+        let (catalog, rejected) = loaded.value.without_rejected(&loaded.errors);
+        assert_eq!(rejected.len(), 1);
+        assert!(catalog.lookup("bad").is_none());
+        let good = catalog.lookup("good").unwrap();
+        assert!(good.entry.model().is_some());
+        assert_eq!(
+            good.entry.field("custom_metadata"),
+            Some(&Value::Bool(true))
+        );
+        assert!(
+            Catalog::problems(body)
+                .iter()
+                .any(|problem| problem.kind() == crate::ErrorKind::UnreadField)
+        );
+        assert!(
+            Catalog::problems(body)
+                .iter()
+                .any(|problem| problem.kind() == crate::ErrorKind::NoFallbackRules)
+        );
     }
 }

litellm-rust/crates/model-catalog/src/error.rs

@@ -1,21 +1,400 @@
+use std::num::ParseIntError;
+
+use strum::{EnumDiscriminants, IntoStaticStr, VariantArray};
 use thiserror::Error;
 
 #[derive(Debug, Error)]
-pub enum Error {
-    #[error("invalid catalog JSON")]
-    Json(#[from] serde_json::Error),
-    #[error("catalog is empty")]
+#[error(transparent)]
+pub struct Error(Box<Context>);
+
+#[derive(Debug, Error)]
+enum Context {
+    #[error(transparent)]
+    Bare(Repr),
+    #[error("{}", display_path(path))]
+    At {
+        path: String,
+        #[source]
+        cause: Repr,
+    },
+}
+
+#[derive(Debug, Error)]
+pub(crate) enum DateCause {
+    #[error("invalid date component")]
+    Number(#[from] ParseIntError),
+    #[error("calendar component out of range")]
+    Range(#[from] time::error::ComponentRange),
+}
+
+#[derive(Debug, Error, EnumDiscriminants)]
+#[strum_discriminants(name(ErrorKind), derive(IntoStaticStr, VariantArray), vis(pub))]
+pub(crate) enum Repr {
+    #[error("invalid JSON")]
+    #[strum_discriminants(strum(serialize = "json"))]
+    Json(#[source] serde_json::Error),
+    #[error("has no models")]
+    #[strum_discriminants(strum(serialize = "empty"))]
     Empty,
-    #[error("model {model:?} must be an object")]
-    EntryNotObject { model: String },
-    #[error("catalog has {actual} models, below minimum {minimum}")]
+    #[error("has {actual} models, below minimum {minimum}")]
+    #[strum_discriminants(strum(serialize = "below_minimum"))]
     BelowMinimum { actual: usize, minimum: usize },
-    #[error("catalog has {actual} models, below {ratio} of reference count {reference}")]
+    #[error("has {actual} models, below {ratio} of reference count {reference}")]
+    #[strum_discriminants(strum(serialize = "shrunk"))]
     Shrunk {
         actual: usize,
         reference: usize,
         ratio: f64,
     },
-    #[error("minimum reference ratio must be finite and between zero and one")]
-    InvalidRatio,
+    #[error("must be a JSON object")]
+    #[strum_discriminants(strum(serialize = "not_an_object"))]
+    NotAnObject,
+    #[error("invalid value")]
+    #[strum_discriminants(strum(serialize = "invalid"))]
+    Invalid(#[source] serde_json::Error),
+    #[error("not a field this block reads")]
+    #[strum_discriminants(strum(serialize = "unknown_block_key"))]
+    UnknownBlockKey,
+    #[error("required")]
+    #[strum_discriminants(strum(serialize = "missing_provider"))]
+    MissingProvider,
+    #[error("{zone:?} is not an IANA time zone")]
+    #[strum_discriminants(strum(serialize = "unknown_timezone"))]
+    UnknownTimezone { zone: String },
+    #[error("alias {alias:?} is already a model name")]
+    #[strum_discriminants(strum(serialize = "alias_shadows_model"))]
+    AliasShadowsModel { alias: String },
+    #[error("alias {alias:?} already belongs to {taken_by}")]
+    #[strum_discriminants(strum(serialize = "alias_taken"))]
+    AliasTaken { alias: String, taken_by: String },
+    #[error("{first} and {second} price the same meter under the same conditions")]
+    #[strum_discriminants(strum(serialize = "duplicate_price"))]
+    DuplicatePrice { first: String, second: String },
+    #[error("a price inside {block} takes no band or tier suffix")]
+    #[strum_discriminants(strum(serialize = "suffix_inside_block"))]
+    SuffixInsideBlock { block: &'static str },
+    #[error("a tier needs exactly one of range and max_results_range")]
+    #[strum_discriminants(strum(serialize = "tier_range"))]
+    TierRange,
+    #[error("the range ends below its start")]
+    #[strum_discriminants(strum(serialize = "tier_reversed"))]
+    TierReversed,
+    #[error("starts at {start}, but the tiers before it end at {expected}")]
+    #[strum_discriminants(strum(serialize = "tier_gap"))]
+    TierGap { start: u64, expected: u64 },
+    #[error("tiered_pricing mixes token ranges and result ranges")]
+    #[strum_discriminants(strum(serialize = "mixed_tier_basis"))]
+    MixedTierBasis,
+    #[error("a tier by result count only prices input_cost_per_query")]
+    #[strum_discriminants(strum(serialize = "not_priced_by_results"))]
+    NotPricedByResults,
+    #[error("tiered_pricing and prompt token bands are set in two places")]
+    #[strum_discriminants(strum(serialize = "bands_from_two_sources"))]
+    BandsFromTwoSources,
+    #[error("has no off-peak rate")]
+    #[strum_discriminants(strum(serialize = "not_an_off_peak_rate"))]
+    NotAnOffPeakRate,
+    #[error("off_peak_pricing has no prices")]
+    #[strum_discriminants(strum(serialize = "no_off_peak_prices"))]
+    NoOffPeakPrices,
+    #[error("off_peak_pricing needs hours_utc or windows")]
+    #[strum_discriminants(strum(serialize = "no_off_peak_hours"))]
+    NoOffPeakHours,
+    #[error("must list at least one entry")]
+    #[strum_discriminants(strum(serialize = "empty_list"))]
+    EmptyList,
+    #[error("expected a calendar date YYYY-MM-DD")]
+    #[strum_discriminants(strum(serialize = "invalid_date"))]
+    InvalidDate {
+        #[source]
+        cause: Option<DateCause>,
+    },
+    #[error("not a field litellm reads")]
+    #[strum_discriminants(strum(serialize = "unread_field"))]
+    UnreadField,
+    #[error("has no rules")]
+    #[strum_discriminants(strum(serialize = "no_fallback_rules"))]
+    NoFallbackRules,
+}
+
+/// What a problem takes out of the served map
+#[derive(Clone, Copy, Debug, PartialEq, Eq, VariantArray)]
+pub enum Rejects {
+    Document,
+    Row,
+    Nothing,
+}
+
+impl Error {
+    pub fn kind(&self) -> ErrorKind {
+        ErrorKind::from(self.repr())
+    }
+
+    pub fn rejects(&self) -> Rejects {
+        match self.repr() {
+            Repr::Json(_) | Repr::Empty | Repr::BelowMinimum { .. } | Repr::Shrunk { .. } => {
+                Rejects::Document
+            }
+            Repr::NotAnObject
+            | Repr::Invalid(_)
+            | Repr::UnknownBlockKey
+            | Repr::UnknownTimezone { .. }
+            | Repr::AliasShadowsModel { .. }
+            | Repr::AliasTaken { .. }
+            | Repr::DuplicatePrice { .. }
+            | Repr::SuffixInsideBlock { .. }
+            | Repr::TierRange
+            | Repr::TierReversed
+            | Repr::TierGap { .. }
+            | Repr::MixedTierBasis
+            | Repr::NotPricedByResults
+            | Repr::BandsFromTwoSources
+            | Repr::NotAnOffPeakRate
+            | Repr::NoOffPeakPrices
+            | Repr::NoOffPeakHours
+            | Repr::EmptyList
+            | Repr::InvalidDate { .. } => Rejects::Row,
+            Repr::MissingProvider | Repr::UnreadField | Repr::NoFallbackRules => Rejects::Nothing,
+        }
+    }
+}
+
+#[derive(Debug)]
+pub struct Loaded<T> {
+    pub value: T,
+    pub errors: Vec<Error>,
+}
+
+impl From<Repr> for Error {
+    fn from(cause: Repr) -> Self {
+        Self(Box::new(Context::Bare(cause)))
+    }
+}
+
+impl From<DateCause> for Error {
+    fn from(cause: DateCause) -> Self {
+        Repr::InvalidDate { cause: Some(cause) }.into()
+    }
+}
+
+impl Error {
+    fn repr(&self) -> &Repr {
+        match &*self.0 {
+            Context::Bare(cause) | Context::At { cause, .. } => cause,
+        }
+    }
+
+    pub(crate) fn new(path: impl Into<String>, cause: Repr) -> Self {
+        Self(Box::new(Context::At {
+            path: path.into(),
+            cause,
+        }))
+    }
+
+    pub(crate) fn from_key(key: &str, cause: Repr) -> Self {
+        Self::new(pointer_for_key(key), cause)
+    }
+
+    pub(crate) fn from_path(path: &str, cause: Repr) -> Self {
+        Self::new(path_to_pointer(path), cause)
+    }
+
+    pub fn path(&self) -> &str {
+        match &*self.0 {
+            Context::Bare(_) => "",
+            Context::At { path, .. } => path,
+        }
+    }
+
+    pub fn key(&self) -> Option<String> {
+        pointer_root(self.path())
+    }
+
+    pub fn field(&self) -> Option<String> {
+        let segments: Vec<_> = self
+            .path()
+            .split('/')
+            .skip(2)
+            .map(unescape_pointer_segment)
+            .collect();
+        (!segments.is_empty()).then(|| segments.join("/"))
+    }
+
+    pub(crate) fn prepend(self, key: &str) -> Self {
+        let (path, cause) = match *self.0 {
+            Context::Bare(cause) => (String::new(), cause),
+            Context::At { path, cause } => (path, cause),
+        };
+        Self::new(format!("/{}{path}", escape_pointer_segment(key)), cause)
+    }
+}
+
+pub(crate) fn pointer_root(path: &str) -> Option<String> {
+    path.strip_prefix('/')
+        .and_then(|path| path.split('/').next())
+        .map(unescape_pointer_segment)
+}
+
+fn escape_pointer_segment(segment: &str) -> String {
+    segment.replace('~', "~0").replace('/', "~1")
+}
+
+pub(crate) fn pointer_for_key(key: &str) -> String {
+    format!("/{}", escape_pointer_segment(key))
+}
+
+pub(crate) fn pointer_child(parent: &str, key: &str) -> String {
+    format!("{parent}/{}", escape_pointer_segment(key))
+}
+
+fn unescape_pointer_segment(segment: &str) -> String {
+    segment.replace("~1", "/").replace("~0", "~")
+}
+
+fn path_to_pointer(key: &str) -> String {
+    let mut pointer = String::new();
+    let mut segment = String::new();
+    let mut chars = key.chars();
+    let mut push_segment = |segment: &mut String| {
+        if !segment.is_empty() {
+            pointer.push('/');
+            pointer.push_str(&escape_pointer_segment(segment));
+            segment.clear();
+        }
+    };
+    while let Some(character) = chars.next() {
+        match character {
+            '.' => push_segment(&mut segment),
+            '[' => {
+                push_segment(&mut segment);
+                let mut index = String::new();
+                for character in chars.by_ref() {
+                    if character == ']' {
+                        break;
+                    }
+                    index.push(character);
+                }
+                push_segment(&mut index);
+            }
+            character => segment.push(character),
+        }
+    }
+    push_segment(&mut segment);
+    pointer
+}
+
+fn display_path(path: &str) -> String {
+    if path.is_empty() {
+        return "catalog".to_owned();
+    }
+    path.split('/')
+        .skip(1)
+        .map(unescape_pointer_segment)
+        .enumerate()
+        .fold(String::new(), |mut display, (index, segment)| {
+            if index == 0 {
+                display.push_str(&segment);
+            } else if segment.bytes().all(|byte| byte.is_ascii_digit()) {
+                display.push('[');
+                display.push_str(&segment);
+                display.push(']');
+            } else {
+                display.push('.');
+                display.push_str(&segment);
+            }
+            display
+        })
+}
+
+#[cfg(test)]
+mod tests {
+    use litellm_error::{Report, find};
+    use rstest::rstest;
+    use serde_json::Value;
+
+    use super::*;
+    use crate::{Catalog, wire::IsoDate};
+
+    #[rstest]
+    #[case::json(Repr::Json(serde_json::from_str::<Value>("{").unwrap_err()), Rejects::Document)]
+    #[case::empty(Repr::Empty, Rejects::Document)]
+    #[case::below_minimum(Repr::BelowMinimum { actual: 1, minimum: 2 }, Rejects::Document)]
+    #[case::shrunk(Repr::Shrunk { actual: 1, reference: 3, ratio: 0.5 }, Rejects::Document)]
+    #[case::not_an_object(Repr::NotAnObject, Rejects::Row)]
+    #[case::invalid(Repr::Invalid(serde_json::from_str::<Value>("x").unwrap_err()), Rejects::Row)]
+    #[case::unknown_block_key(Repr::UnknownBlockKey, Rejects::Row)]
+    #[case::missing_provider(Repr::MissingProvider, Rejects::Nothing)]
+    #[case::unknown_timezone(Repr::UnknownTimezone { zone: "invalid".into() }, Rejects::Row)]
+    #[case::alias_shadows_model(Repr::AliasShadowsModel { alias: "model".into() }, Rejects::Row)]
+    #[case::alias_taken(Repr::AliasTaken { alias: "alias".into(), taken_by: "owner".into() }, Rejects::Row)]
+    #[case::duplicate_price(Repr::DuplicatePrice { first: "a".into(), second: "b".into() }, Rejects::Row)]
+    #[case::suffix_inside_block(Repr::SuffixInsideBlock { block: "tiered_pricing" }, Rejects::Row)]
+    #[case::tier_range(Repr::TierRange, Rejects::Row)]
+    #[case::tier_reversed(Repr::TierReversed, Rejects::Row)]
+    #[case::tier_gap(Repr::TierGap { start: 1, expected: 0 }, Rejects::Row)]
+    #[case::mixed_tier_basis(Repr::MixedTierBasis, Rejects::Row)]
+    #[case::not_priced_by_results(Repr::NotPricedByResults, Rejects::Row)]
+    #[case::bands_from_two_sources(Repr::BandsFromTwoSources, Rejects::Row)]
+    #[case::not_an_off_peak_rate(Repr::NotAnOffPeakRate, Rejects::Row)]
+    #[case::no_off_peak_prices(Repr::NoOffPeakPrices, Rejects::Row)]
+    #[case::no_off_peak_hours(Repr::NoOffPeakHours, Rejects::Row)]
+    #[case::empty_list(Repr::EmptyList, Rejects::Row)]
+    #[case::invalid_date(Repr::InvalidDate { cause: None }, Rejects::Row)]
+    #[case::unread_field(Repr::UnreadField, Rejects::Nothing)]
+    #[case::no_fallback_rules(Repr::NoFallbackRules, Rejects::Nothing)]
+    fn policy_classifies_each_error(#[case] error: Repr, #[case] expected: Rejects) {
+        assert_eq!(Error::from(error).rejects(), expected);
+    }
+
+    #[rstest]
+    #[case::root("", "x", "/x")]
+    #[case::empty_model("/", "x", "//x")]
+    #[case::empty_parent_segment("/m/", "x", "/m//x")]
+    #[case::escaped("/m~1n", "a~/b", "/m~1n/a~0~1b")]
+    fn pointers_preserve_segments(#[case] parent: &str, #[case] key: &str, #[case] expected: &str) {
+        assert_eq!(pointer_child(parent, key), expected);
+    }
+
+    #[rstest]
+    #[case::malformed(br#"{"#)]
+    #[case::bad_price(br#"{"m":{"litellm_provider":"test","input_cost_per_token":"free"}}"#)]
+    fn json_causes_are_typed_and_rendered_once(#[case] body: &[u8]) {
+        let error = match Catalog::parse(body) {
+            Ok(loaded) => loaded.errors.into_iter().next().expect("invalid price"),
+            Err(error) => error,
+        };
+        let cause = find::<serde_json::Error>(&error)
+            .expect("typed JSON cause")
+            .to_string();
+        let report = Report::new(&error).to_string();
+        assert_eq!(report.matches(&cause).count(), 1, "{report}");
+        assert!(!error.to_string().contains(&cause));
+    }
+
+    #[rstest]
+    #[case::year("xxxx-01-01")]
+    #[case::month("2026-xx-01")]
+    #[case::day("2026-01-xx")]
+    fn date_number_errors_keep_the_cause(#[case] value: &str) {
+        let error = value.parse::<IsoDate>().unwrap_err();
+        let cause = find::<ParseIntError>(&error)
+            .expect("typed integer cause")
+            .to_string();
+        assert_eq!(error.rejects(), Rejects::Row);
+        assert_eq!(Report::new(&error).to_string().matches(&cause).count(), 1);
+        assert!(!error.to_string().contains(&cause));
+    }
+
+    #[rstest]
+    #[case::month("2026-13-01")]
+    #[case::day("2026-02-30")]
+    fn date_range_errors_keep_the_cause(#[case] value: &str) {
+        let error = value.parse::<IsoDate>().unwrap_err();
+        let cause = find::<time::error::ComponentRange>(&error)
+            .expect("typed calendar cause")
+            .to_string();
+        assert_eq!(error.rejects(), Rejects::Row);
+        assert_eq!(Report::new(&error).to_string().matches(&cause).count(), 1);
+        assert!(!error.to_string().contains(&cause));
+    }
 }

litellm-rust/crates/model-catalog/src/fallback.rs

@@ -1,10 +1,9 @@
-use serde::{Deserialize, Serialize};
 use serde_json::Value;
 use std::collections::BTreeMap;
 
 /// One regex rule generalizing unknown model ids to known families.
-#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
-#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
+#[macro_rules_attribute::apply(crate::wire_type)]
+#[derive(Eq, Hash)]
 pub struct FallbackRule {
     pub name: String,
     pub pattern: String,
@@ -15,8 +14,8 @@ pub struct FallbackRule {
 }
 
 /// Regex rules that generalize unknown model ids to known families; not a model entry.
-#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
-#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
+#[macro_rules_attribute::apply(crate::wire_type)]
+#[derive(Eq, Hash)]
 #[serde(deny_unknown_fields)]
 pub struct FallbackGeneralizations {
     pub rules: Vec<FallbackRule>,

litellm-rust/crates/model-catalog/src/layers.rs

@@ -0,0 +1,32 @@
+use std::sync::Arc;
+
+use indexmap::IndexMap;
+use serde_json::Value;
+
+use crate::catalog::Catalog;
+use crate::register::Registrations;
+use crate::wire::Fields;
+
+/// One field of a fetched row over the layer below, RFC 7386. An absent field inherits, JSON `null` is `Remove`
+#[derive(Clone, Debug, PartialEq)]
+pub enum Patch {
+    Set(Value),
+    Remove,
+}
+
+/// The four layers served rows are composed from, lowest first
+#[derive(Clone, Debug)]
+#[expect(
+    dead_code,
+    reason = "02 and 07 add the functions that read these layers"
+)]
+pub struct Layers {
+    /// 0, the wheel's embedded map
+    bundled: Arc<Catalog>,
+    /// 1, the last installed fetch, rejected rows left out
+    fetched: Option<Arc<Catalog>>,
+    /// 2, `register(persist=False)`, cleared by every install and refilled by the router replay
+    transient: IndexMap<String, Fields>,
+    /// 3, `register(persist=True)`, applied over every install
+    persisted: Registrations,
+}

litellm-rust/crates/model-catalog/src/lib.rs

@@ -1,18 +1,65 @@
-mod capabilities;
+//! The model catalog: per-model pricing, context limits and capabilities.
+//!
+//! This is the Rust form of `model_prices_and_context_window.json`, the map Python reads as
+//! `litellm.model_cost` and through `litellm.get_model_info`. It answers "what does this model
+//! cost and what can it do"; it knows nothing about configured deployments. The router's view of
+//! deployments (groups, aliases, wildcard patterns, team mappings) is the router registry, defined
+//! in `litellm-router-core`, which reads this catalog through its dynamic config.
+//!
+//! # Structure
+//! - `Catalog`: the parsed map. `Catalog::parse` takes the JSON body and returns `Loaded<Catalog>`,
+//!   keeping every entry in file order with its raw fields and collecting row and alias errors.
+//! - `ModelEntry`: one model. `ModelEntry::fields` is the raw JSON object, so unknown fields are
+//!   never lost; `ModelEntry::model` is the optional typed `Model` with attributes, prices,
+//!   adjustments and capabilities. Invalid rows retain their raw fields without a typed model.
+//! - Aliases: an entry may list `aliases`; each alias resolves to its canonical key. Problems
+//!   found while indexing them (bad lists, collisions) are kept in `Loaded::errors` instead of
+//!   failing the document.
+//! - `FallbackGeneralizations` / `FallbackRule`: regex rules that map unknown model ids onto known
+//!   families. They are data in the file, not entries.
+//! - Validation: `Catalog::check_size` checks a catalog against `IntegrityLimits` (a remote copy
+//!   that lost most of its models is refused), and `Catalog::row_errors` validates one row.
+//!
+//! # Lookups
+//! `Catalog::lookup` resolves a key exactly first (entry or alias), then case-insensitively, and
+//! returns a `ModelMatch` with both the key that matched and the canonical key of the entry. Provider
+//! prefixes are not stripped here; callers decide which keys to try.
+//!
+//! # Readers
+//! Cost tracking, the router (through `RouterDynamicConfig`) and the gateway. Nothing in this crate
+//! performs I/O; loading and refreshing the file is the caller's job.
+
+macro_rules_attribute::attribute_alias! {
+    #[apply(wire_type)] =
+        #[derive(Clone, Debug, PartialEq, serde::Serialize, serde::Deserialize)]
+        #[cfg_attr(feature = "schema", derive(schemars::JsonSchema))];
+}
+
+mod bundled;
 mod catalog;
 mod error;
 mod fallback;
+mod layers;
+mod lower;
+mod model;
 mod model_info;
-mod pricing;
-mod validation;
+mod register;
+mod resolve;
+mod store;
+mod vocabulary;
+pub mod wire;
 
-pub use capabilities::*;
+pub use bundled::*;
 pub use catalog::*;
 pub use error::*;
 pub use fallback::*;
+pub use layers::*;
+pub use model::*;
 pub use model_info::*;
-pub use pricing::*;
-pub use validation::*;
+pub use register::*;
+pub use resolve::*;
+pub use store::*;
+pub use vocabulary::*;
 
 #[cfg(feature = "schema")]
 mod schema;

litellm-rust/crates/model-catalog/src/lower.rs

@@ -0,0 +1,855 @@
+use std::collections::BTreeMap;
+use std::collections::btree_map::Entry;
+use std::num::NonZeroU64;
+
+use litellm_cost::money::UnitPrice;
+use litellm_cost::price::{
+    Adjustments, Band, BandEdge, Billing, CacheReadFallback, CacheReadMode, CacheTtl,
+    GuardrailUnit, InputUnit, Meter, OffPeak, OffPeakRates, OutputUnit, Price, Prices, Residency,
+    SearchContextSize, TextUnit, TierScope, TokenKind, Uplift, WallClock, WebSearchUnit, When,
+};
+use litellm_cost::schedule::{Schedule, Window};
+
+use crate::error::{Error, Repr, pointer_child, pointer_for_key};
+use crate::model::Model;
+use crate::vocabulary::Mode;
+use crate::wire::{EntryWire, FlatPrice, OffPeakBlock, TierRow, WebSearchBillingUnit};
+
+impl TryFrom<&EntryWire> for Prices {
+    type Error = Vec<Error>;
+
+    fn try_from(wire: &EntryWire) -> Result<Self, Self::Error> {
+        let mut problems = Vec::new();
+        let sourced = flat_prices(&wire.prices)
+            .chain(tier_prices(wire, &mut problems))
+            .chain(search_context_prices(wire))
+            .chain(guardrail_prices(wire));
+        let rates = dedupe(sourced, &mut problems);
+        let off_peak = wire
+            .off_peak
+            .as_ref()
+            .and_then(|block| off_peak(block, &mut problems));
+        if problems.is_empty() {
+            let billing = billing(wire, &rates);
+            Ok(Prices::new(rates, off_peak, billing))
+        } else {
+            Err(problems)
+        }
+    }
+}
+
+fn billing(wire: &EntryWire, rates: &BTreeMap<Price, UnitPrice>) -> Billing {
+    let provider = wire.attributes.litellm_provider.as_deref();
+    let prices_meter = |priced: fn(&Meter) -> bool| rates.keys().any(|price| priced(&price.meter));
+    let prices_tokens = prices_meter(|meter| {
+        matches!(
+            meter,
+            Meter::Input(InputUnit::Token(_)) | Meter::Output(OutputUnit::Token(_))
+        )
+    });
+    let prices_characters = prices_meter(|meter| {
+        matches!(
+            meter,
+            Meter::Input(InputUnit::Character) | Meter::Output(OutputUnit::Character)
+        )
+    });
+    let prices_cache_read = prices_meter(|meter| {
+        matches!(
+            meter,
+            Meter::CacheRead {
+                kind: TokenKind::Text,
+                ..
+            }
+        )
+    });
+    Billing {
+        text_unit: if prices_characters && !prices_tokens {
+            TextUnit::Character
+        } else {
+            TextUnit::Token
+        },
+        band_edge: if provider == Some("xai") {
+            BandEdge::Inclusive
+        } else {
+            BandEdge::Exclusive
+        },
+        cache_read_fallback: if provider == Some("fireworks_ai") && !prices_cache_read {
+            CacheReadFallback::HalfInputRate
+        } else {
+            CacheReadFallback::InputRate
+        },
+        wall_clock: match wire.attributes.mode {
+            None | Some(Mode::Chat | Mode::Completion | Mode::Embedding | Mode::Responses) => {
+                WallClock::ResponseTime
+            }
+            Some(Mode::AudioTranscription) => WallClock::AudioFile,
+            Some(_) => WallClock::Never,
+        },
+        web_search: match wire.attributes.web_search_billing_unit {
+            Some(WebSearchBillingUnit::PerQuery) => WebSearchUnit::PerQuery,
+            Some(WebSearchBillingUnit::PerPrompt) | None => WebSearchUnit::PerPrompt,
+        },
+        tier_scope: if provider == Some("dashscope") && wire.tier_rows.is_some() {
+            TierScope::WholeCall
+        } else {
+            TierScope::PerMeter
+        },
+    }
+}
+
+impl From<&EntryWire> for Adjustments {
+    fn from(wire: &EntryWire) -> Self {
+        Adjustments::new(
+            [
+                (Uplift::NonGlobalEndpoint, wire.regional_endpoint_uplift),
+                (Uplift::Residency(Residency::Eu), wire.residency_uplift_eu),
+                (Uplift::Residency(Residency::Us), wire.residency_uplift_us),
+            ]
+            .into_iter()
+            .filter_map(|(uplift, factor)| Some((uplift, factor?))),
+        )
+    }
+}
+
+impl TryFrom<EntryWire> for Model {
+    type Error = Vec<Error>;
+
+    fn try_from(wire: EntryWire) -> Result<Self, Self::Error> {
+        let prices = Prices::try_from(&wire);
+        let adjustments = Adjustments::from(&wire);
+        let EntryWire {
+            attributes,
+            capabilities,
+            unknown,
+            problems,
+            ..
+        } = wire;
+        let missing_provider = attributes.litellm_provider.is_none().then_some(Error::new(
+            pointer_for_key("litellm_provider"),
+            Repr::MissingProvider,
+        ));
+        match prices {
+            Ok(prices) if problems.is_empty() && missing_provider.is_none() => Ok(Model {
+                attributes,
+                prices,
+                adjustments,
+                capabilities,
+                unrecognized: unknown.into_iter().map(|(key, _)| key).collect(),
+            }),
+            prices => Err(problems
+                .into_iter()
+                .chain(missing_provider)
+                .chain(prices.err().into_iter().flatten())
+                .collect()),
+        }
+    }
+}
+
+struct Sourced {
+    price: Price,
+    rate: UnitPrice,
+    source: String,
+    path: String,
+}
+
+fn flat_prices(prices: &[FlatPrice]) -> impl Iterator<Item = Sourced> + '_ {
+    prices.iter().map(|flat| {
+        let (channel, when) = flat.flat.channel_and_when();
+        Sourced {
+            price: Price {
+                meter: flat.flat.meter.clone(),
+                channel,
+                when,
+            },
+            rate: flat.price,
+            source: flat.key.clone(),
+            path: flat.path.clone(),
+        }
+    })
+}
+
+fn dedupe(
+    sourced: impl Iterator<Item = Sourced>,
+    problems: &mut Vec<Error>,
+) -> BTreeMap<Price, UnitPrice> {
+    let mut seen: BTreeMap<Price, (UnitPrice, String, String)> = BTreeMap::new();
+    for Sourced {
+        price,
+        rate,
+        source,
+        path,
+    } in sourced
+    {
+        match seen.entry(price) {
+            Entry::Occupied(first) if first.get().0 == rate => {}
+            Entry::Occupied(first) => problems.push(Error::new(
+                path,
+                Repr::DuplicatePrice {
+                    first: first.get().1.clone(),
+                    second: source.clone(),
+                },
+            )),
+            Entry::Vacant(slot) => {
+                slot.insert((rate, source, path));
+            }
+        }
+    }
+    seen.into_iter()
+        .map(|(price, (rate, _, _))| (price, rate))
+        .collect()
+}
+
+#[derive(Clone, Copy, PartialEq, Eq)]
+enum TierBasis {
+    PromptTokens,
+    Results,
+}
+
+fn tier_basis(row: &TierRow) -> Result<(TierBasis, [u64; 2]), Error> {
+    match (row.range, row.max_results_range) {
+        (Some(range), None) => Ok((TierBasis::PromptTokens, range)),
+        (None, Some(range)) => Ok((TierBasis::Results, range)),
+        _ => Err(Error::new(row.path.clone(), Repr::TierRange)),
+    }
+}
+
+fn tier_prices(wire: &EntryWire, problems: &mut Vec<Error>) -> Vec<Sourced> {
+    let Some(rows) = wire.tier_rows.as_deref().filter(|rows| !rows.is_empty()) else {
+        return Vec::new();
+    };
+    let based: Vec<_> = rows
+        .iter()
+        .filter_map(|row| match tier_basis(row) {
+            Ok((basis, range)) => Some((basis, range, row)),
+            Err(problem) => {
+                problems.push(problem);
+                None
+            }
+        })
+        .collect();
+    let Some(&(basis, ..)) = based.first() else {
+        return Vec::new();
+    };
+    if based.iter().any(|(other, ..)| *other != basis) {
+        problems.push(Error::from_key("tiered_pricing", Repr::MixedTierBasis));
+        return Vec::new();
+    }
+    if basis == TierBasis::PromptTokens {
+        problems.extend(
+            wire.prices
+                .iter()
+                .filter(|flat| flat.flat.band != Band::Base)
+                .map(|flat| Error::new(flat.path.clone(), Repr::BandsFromTwoSources)),
+        );
+    }
+    let mut ordered = based;
+    ordered.sort_by_key(|(_, [start, _], _)| *start);
+    problems.extend(ladder_problems(basis, &ordered));
+    ordered
+        .iter()
+        .flat_map(|(basis, [start, _], row)| {
+            row.prices.iter().map(move |flat| (*basis, *start, flat))
+        })
+        .filter_map(
+            |(basis, start, flat)| match tier_price(basis, start, flat) {
+                Ok(sourced) => Some(sourced),
+                Err(problem) => {
+                    problems.push(problem);
+                    None
+                }
+            },
+        )
+        .collect()
+}
+
+fn ladder_problems(basis: TierBasis, ordered: &[(TierBasis, [u64; 2], &TierRow)]) -> Vec<Error> {
+    let step = match basis {
+        TierBasis::PromptTokens => 0,
+        TierBasis::Results => 1,
+    };
+    let reversed = ordered
+        .iter()
+        .filter(|(_, [start, end], _)| end < start)
+        .map(|(.., row)| Error::new(row.path.clone(), Repr::TierReversed));
+    let first = ordered
+        .first()
+        .filter(|(_, [start, _], _)| *start > step)
+        .map(|(_, [start, _], row)| {
+            Error::new(
+                row.path.clone(),
+                Repr::TierGap {
+                    start: *start,
+                    expected: 0,
+                },
+            )
+        });
+    let gaps = ordered.windows(2).filter_map(|pair| {
+        let (_, [_, end], _) = pair[0];
+        let (_, [start, _], row) = pair[1];
+        (end.checked_add(step) != Some(start)).then(|| {
+            Error::new(
+                row.path.clone(),
+                Repr::TierGap {
+                    start,
+                    expected: end,
+                },
+            )
+        })
+    });
+    reversed.chain(first).chain(gaps).collect()
+}
+
+fn tier_price(basis: TierBasis, start: u64, flat: &FlatPrice) -> Result<Sourced, Error> {
+    if flat.flat.has_suffix() {
+        return Err(Error::new(
+            flat.path.clone(),
+            Repr::SuffixInsideBlock {
+                block: "tiered_pricing",
+            },
+        ));
+    }
+    let (meter, band) = match (basis, &flat.flat.meter) {
+        (TierBasis::PromptTokens, meter) => (meter.clone(), Band::over_prompt_tokens(start)),
+        (TierBasis::Results, Meter::Input(InputUnit::Query { over_results: None })) => (
+            Meter::Input(InputUnit::Query {
+                over_results: NonZeroU64::new(start.saturating_sub(1)),
+            }),
+            Band::Base,
+        ),
+        (TierBasis::Results, _) => {
+            return Err(Error::new(flat.path.clone(), Repr::NotPricedByResults));
+        }
+    };
+    Ok(Sourced {
+        price: Price::online(
+            meter,
+            When {
+                band,
+                ..When::default()
+            },
+        ),
+        rate: flat.price,
+        source: flat.key.clone(),
+        path: flat.path.clone(),
+    })
+}
+
+fn search_context_prices(wire: &EntryWire) -> impl Iterator<Item = Sourced> + '_ {
+    wire.search_context.iter().flat_map(|costs| {
+        [
+            (SearchContextSize::Low, "low", costs.search_context_size_low),
+            (
+                SearchContextSize::Medium,
+                "medium",
+                costs.search_context_size_medium,
+            ),
+            (
+                SearchContextSize::High,
+                "high",
+                costs.search_context_size_high,
+            ),
+        ]
+        .into_iter()
+        .filter_map(|(size, name, rate)| {
+            Some(Sourced {
+                price: Price::online(Meter::WebSearch(size), When::default()),
+                rate: rate?,
+                source: format!("search_context_cost_per_query.search_context_size_{name}"),
+                path: pointer_child(
+                    &pointer_for_key("search_context_cost_per_query"),
+                    &format!("search_context_size_{name}"),
+                ),
+            })
+        })
+    })
+}
+
+fn guardrail_prices(wire: &EntryWire) -> impl Iterator<Item = Sourced> + '_ {
+    wire.guardrail.iter().flatten().map(|(unit, rate)| Sourced {
+        price: Price::online(Meter::Guardrail(GuardrailUnit::new(unit)), When::default()),
+        rate: *rate,
+        source: format!("guardrail_cost_per_unit.{unit}"),
+        path: pointer_child(&pointer_for_key("guardrail_cost_per_unit"), unit),
+    })
+}
+
+fn off_peak(block: &OffPeakBlock, problems: &mut Vec<Error>) -> Option<OffPeak> {
+    let rates = off_peak_rates(&block.prices, problems);
+    let schedule = schedule(block, problems)?;
+    let off_peak = OffPeak::new(schedule, rates);
+    if off_peak.is_none() {
+        problems.push(Error::from_key("off_peak_pricing", Repr::NoOffPeakPrices));
+    }
+    off_peak
+}
+
+fn off_peak_rates(prices: &[FlatPrice], problems: &mut Vec<Error>) -> OffPeakRates {
+    let mut rates = OffPeakRates::default();
+    for flat in prices {
+        if flat.flat.has_suffix() {
+            problems.push(Error::new(
+                flat.path.clone(),
+                Repr::SuffixInsideBlock {
+                    block: "off_peak_pricing",
+                },
+            ));
+            continue;
+        }
+        let slot = match flat.flat.meter {
+            Meter::Input(InputUnit::Token(TokenKind::Text)) => &mut rates.input,
+            Meter::Output(OutputUnit::Token(TokenKind::Text)) => &mut rates.output,
+            Meter::Output(OutputUnit::ReasoningToken) => &mut rates.reasoning,
+            Meter::CacheRead {
+                kind: TokenKind::Text,
+                mode: CacheReadMode::Implicit,
+            } => &mut rates.cache_read,
+            Meter::CacheRead {
+                kind: TokenKind::Text,
+                mode: CacheReadMode::Explicit,
+            } => &mut rates.explicit_cache_read,
+            Meter::CacheWrite {
+                kind: TokenKind::Text,
+                ttl: CacheTtl::Default,
+            } => &mut rates.cache_write,
+            _ => {
+                problems.push(Error::new(flat.path.clone(), Repr::NotAnOffPeakRate));
+                continue;
+            }
+        };
+        *slot = Some(flat.price);
+    }
+    rates
+}
+
+fn schedule(block: &OffPeakBlock, problems: &mut Vec<Error>) -> Option<Schedule> {
+    let empty = |key: String| Error::from_path(&key, Repr::EmptyList);
+    let every_day = block
+        .hours_utc
+        .iter()
+        .map(|hours| ("off_peak_pricing.hours_utc".to_owned(), hours, None));
+    let windows = block
+        .windows
+        .iter()
+        .flatten()
+        .enumerate()
+        .map(|(index, window)| {
+            (
+                format!("off_peak_pricing.windows[{index}].hours_utc"),
+                &window.hours_utc,
+                Some((index, window.weekdays.as_ref())),
+            )
+        });
+    let windows: Vec<Window> = every_day
+        .chain(windows)
+        .map(|(key, hours, weekdays)| {
+            if hours.windows().is_empty() {
+                problems.push(empty(key));
+            }
+            if let Some((index, Some(days))) = weekdays
+                && days.is_empty()
+            {
+                problems.push(empty(format!("off_peak_pricing.windows[{index}].weekdays")));
+            }
+            Window {
+                hours: hours.windows().to_vec(),
+                weekdays: weekdays.and_then(|(_, days)| days.cloned()),
+            }
+        })
+        .collect();
+    if block.windows.as_ref().is_some_and(Vec::is_empty) {
+        problems.push(empty("off_peak_pricing.windows".to_owned()));
+    }
+    if block.hours_utc.is_none() && block.windows.is_none() {
+        problems.push(Error::from_key("off_peak_pricing", Repr::NoOffPeakHours));
+        return None;
+    }
+    Some(Schedule {
+        windows,
+        weekday_timezone: block.weekday_timezone.clone(),
+    })
+}
+
+#[cfg(test)]
+mod tests {
+    use litellm_cost::money::Multiplier;
+    use litellm_cost::price::{Channel, ServiceTier};
+    use litellm_cost::schedule::{UtcWindow, Weekday};
+    use rstest::rstest;
+    use serde_json::{Value, json};
+
+    use super::*;
+
+    const TEXT_INPUT: Meter = Meter::Input(InputUnit::Token(TokenKind::Text));
+
+    fn wire(entry: Value) -> EntryWire {
+        let Value::Object(fields) = entry else {
+            panic!("entry must be an object")
+        };
+        let wire = EntryWire::read(&fields);
+        assert!(wire.problems.is_empty(), "{:?}", wire.problems);
+        wire
+    }
+
+    fn prices(entry: Value) -> Result<Prices, Vec<Error>> {
+        Prices::try_from(&wire(entry))
+    }
+
+    fn online(meter: Meter, tier: ServiceTier, band: Band) -> Price {
+        Price::online(meter, When { tier, band })
+    }
+
+    #[rstest]
+    #[case::flat_key(json!({"input_cost_per_token": 1e-6}), online(TEXT_INPUT, ServiceTier::Standard, Band::Base))]
+    #[case::band_and_tier(
+        json!({"input_cost_per_token_above_272k_tokens_priority": 1e-6}),
+        online(TEXT_INPUT, ServiceTier::Priority, Band::over_prompt_tokens(272_000))
+    )]
+    #[case::batch(
+        json!({"input_cost_per_token_batches": 1e-6}),
+        Price { meter: TEXT_INPUT, channel: Channel::Batch, when: When::default() }
+    )]
+    #[case::first_token_tier_is_the_base_band(
+        json!({"tiered_pricing": [{"range": [0.0, 256000.0], "input_cost_per_token": 1e-6}]}),
+        online(TEXT_INPUT, ServiceTier::Standard, Band::Base)
+    )]
+    #[case::result_tier_stays_in_its_meter(
+        json!({"tiered_pricing": [
+            {"max_results_range": [0, 25], "input_cost_per_query": 2e-6},
+            {"max_results_range": [26, 100], "input_cost_per_query": 1e-6}
+        ]}),
+        online(
+            Meter::Input(InputUnit::Query { over_results: NonZeroU64::new(25) }),
+            ServiceTier::Standard,
+            Band::Base
+        )
+    )]
+    #[case::search_context(
+        json!({"search_context_cost_per_query": {"search_context_size_low": 1e-6}}),
+        online(Meter::WebSearch(SearchContextSize::Low), ServiceTier::Standard, Band::Base)
+    )]
+    #[case::character_band_is_a_prompt_token_band(
+        json!({"input_cost_per_character_above_128k_tokens": 1e-6}),
+        online(Meter::Input(InputUnit::Character), ServiceTier::Standard, Band::over_prompt_tokens(128_000))
+    )]
+    #[case::guardrail(
+        json!({"guardrail_cost_per_unit": {"contentPolicyUnits": 1e-6}}),
+        online(Meter::Guardrail(GuardrailUnit::new("contentPolicyUnits")), ServiceTier::Standard, Band::Base)
+    )]
+    fn every_spelling_lowers_to_one_price(#[case] entry: Value, #[case] expected: Price) {
+        assert_eq!(prices(entry).unwrap().get(&expected), UnitPrice::new(1e-6));
+    }
+
+    #[rstest]
+    fn two_spellings_that_agree_are_one_price() {
+        let prices = prices(json!({
+            "input_cost_per_token": 1e-6,
+            "tiered_pricing": [
+                {"range": [0, 10], "input_cost_per_token": 1e-6},
+                {"range": [10, 20], "input_cost_per_token": 2e-6}
+            ]
+        }))
+        .unwrap();
+        assert_eq!(
+            prices.get(&Price::online(TEXT_INPUT, When::default())),
+            UnitPrice::new(1e-6)
+        );
+    }
+
+    #[rstest]
+    fn token_tiers_lower_to_prompt_bands_at_each_start() {
+        let prices = prices(json!({"tiered_pricing": [
+            {"range": [0, 256000], "input_cost_per_token": 1e-6},
+            {"range": [256000, 1000000], "input_cost_per_token": 2e-6}
+        ]}))
+        .unwrap();
+        assert_eq!(
+            prices.band(ServiceTier::Standard, 256_001),
+            Band::over_prompt_tokens(256_000)
+        );
+        assert_eq!(
+            prices
+                .rate(
+                    &TEXT_INPUT,
+                    ServiceTier::Standard,
+                    Band::over_prompt_tokens(256_000)
+                )
+                .map(UnitPrice::get),
+            Some(2e-6)
+        );
+    }
+
+    #[rstest]
+    #[case::two_spellings_of_one_price(
+        json!({
+            "input_cost_per_token": 1e-6,
+            "tiered_pricing": [{"range": [0, 10], "input_cost_per_token": 2e-6}]
+        }),
+        "/tiered_pricing/0/input_cost_per_token",
+        "duplicate_price"
+    )]
+    #[case::zero_band_is_the_base_price(
+        json!({"input_cost_per_token": 1e-6, "input_cost_per_token_above_0k_tokens": 2e-6}),
+        "/input_cost_per_token_above_0k_tokens",
+        "duplicate_price"
+    )]
+    #[case::tier_without_range(
+        json!({"tiered_pricing": [{"input_cost_per_token": 1e-6}]}),
+        "/tiered_pricing/0",
+        "tier_range"
+    )]
+    #[case::tier_with_suffix(
+        json!({"tiered_pricing": [{"range": [0, 10], "input_cost_per_token_priority": 1e-6}]}),
+        "/tiered_pricing/0/input_cost_per_token_priority",
+        "suffix_inside_block"
+    )]
+    #[case::reversed_range(
+        json!({"tiered_pricing": [{"range": [0, 10], "input_cost_per_token": 1e-6}, {"range": [10, 5], "input_cost_per_token": 1e-6}]}),
+        "/tiered_pricing/1",
+        "tier_reversed"
+    )]
+    #[case::gap_between_tiers(
+        json!({"tiered_pricing": [
+            {"range": [0, 10], "input_cost_per_token": 1e-6},
+            {"range": [20, 30], "input_cost_per_token": 2e-6}
+        ]}),
+        "/tiered_pricing/1",
+        "tier_gap"
+    )]
+    #[case::ladder_not_from_zero(
+        json!({"tiered_pricing": [{"range": [5, 10], "input_cost_per_token": 1e-6}]}),
+        "/tiered_pricing/0",
+        "tier_gap"
+    )]
+    #[case::result_ladder_overlaps(
+        json!({"tiered_pricing": [
+            {"max_results_range": [1, 10], "input_cost_per_query": 1e-6},
+            {"max_results_range": [10, 20], "input_cost_per_query": 2e-6}
+        ]}),
+        "/tiered_pricing/1",
+        "tier_gap"
+    )]
+    #[case::mixed_basis(
+        json!({"tiered_pricing": [
+            {"range": [0, 10], "input_cost_per_token": 1e-6},
+            {"max_results_range": [11, 20], "input_cost_per_query": 2e-6}
+        ]}),
+        "/tiered_pricing",
+        "mixed_tier_basis"
+    )]
+    #[case::result_tier_prices_a_token(
+        json!({"tiered_pricing": [{"max_results_range": [0, 10], "input_cost_per_token": 1e-6}]}),
+        "/tiered_pricing/0/input_cost_per_token",
+        "not_priced_by_results"
+    )]
+    #[case::two_band_sources(
+        json!({
+            "output_cost_per_token_above_200k_tokens": 1e-6,
+            "tiered_pricing": [{"range": [0, 10], "input_cost_per_token": 1e-6}]
+        }),
+        "/output_cost_per_token_above_200k_tokens",
+        "bands_from_two_sources"
+    )]
+    #[case::off_peak_without_hours(
+        json!({"off_peak_pricing": {"input_cost_per_token": 1e-6}}),
+        "/off_peak_pricing",
+        "no_off_peak_hours"
+    )]
+    #[case::off_peak_without_prices(
+        json!({"off_peak_pricing": {"hours_utc": "00:00-01:00"}}),
+        "/off_peak_pricing",
+        "no_off_peak_prices"
+    )]
+    #[case::off_peak_empty_windows(
+        json!({"off_peak_pricing": {"windows": [], "input_cost_per_token": 1e-6}}),
+        "/off_peak_pricing/windows",
+        "empty_list"
+    )]
+    #[case::off_peak_empty_hours(
+        json!({"off_peak_pricing": {"hours_utc": [], "input_cost_per_token": 1e-6}}),
+        "/off_peak_pricing/hours_utc",
+        "empty_list"
+    )]
+    #[case::off_peak_empty_weekdays(
+        json!({"off_peak_pricing": {
+            "windows": [{"hours_utc": "00:00-01:00", "weekdays": []}],
+            "input_cost_per_token": 1e-6
+        }}),
+        "/off_peak_pricing/windows/0/weekdays",
+        "empty_list"
+    )]
+    #[case::off_peak_image_price(
+        json!({"off_peak_pricing": {"hours_utc": "00:00-01:00", "output_cost_per_image": 1}}),
+        "/off_peak_pricing/output_cost_per_image",
+        "not_an_off_peak_rate"
+    )]
+    #[case::off_peak_band(
+        json!({"off_peak_pricing": {"hours_utc": "00:00-01:00", "input_cost_per_token_above_200k_tokens": 1}}),
+        "/off_peak_pricing/input_cost_per_token_above_200k_tokens",
+        "suffix_inside_block"
+    )]
+    fn rules_across_keys_name_where_each_price_came_from(
+        #[case] entry: Value,
+        #[case] expected_path: &str,
+        #[case] expected_variant: &str,
+    ) {
+        let problems = prices(entry).unwrap_err();
+        assert!(
+            problems
+                .iter()
+                .any(|problem| problem.path() == expected_path
+                    && <&str>::from(problem.kind()) == expected_variant),
+            "{problems:?}"
+        );
+    }
+
+    #[rstest]
+    fn off_peak_hours_and_windows_form_one_schedule_with_its_rates() {
+        let prices = prices(json!({"off_peak_pricing": {
+            "hours_utc": ["00:00-01:00"],
+            "windows": [{"hours_utc": "10:00-12:00", "weekdays": [6, " sun "]}],
+            "weekday_timezone": "Asia/Shanghai",
+            "output_cost_per_token": 1e-6,
+            "explicit_cache_read_input_token_cost": 2e-6
+        }}))
+        .unwrap();
+        let off_peak = prices.off_peak().unwrap();
+        let window = |text: &str| text.parse::<UtcWindow>().unwrap();
+
+        assert_eq!(
+            off_peak.schedule(),
+            &Schedule {
+                windows: vec![
+                    Window {
+                        hours: vec![window("00:00-01:00")],
+                        weekdays: None,
+                    },
+                    Window {
+                        hours: vec![window("10:00-12:00")],
+                        weekdays: Some(vec![Weekday::Saturday, Weekday::Sunday]),
+                    },
+                ],
+                weekday_timezone: Some("Asia/Shanghai".into()),
+            }
+        );
+        assert_eq!(
+            off_peak.rates(),
+            &OffPeakRates {
+                output: UnitPrice::new(1e-6),
+                explicit_cache_read: UnitPrice::new(2e-6),
+                ..OffPeakRates::default()
+            }
+        );
+        assert_eq!(prices.iter().count(), 0);
+    }
+
+    #[rstest]
+    fn uplifts_are_adjustments_not_prices() {
+        let wire = wire(json!({
+            "regional_endpoint_uplift_multiplier": 1.1,
+            "regional_processing_uplift_multiplier_eu": "1.2"
+        }));
+        let adjustments = Adjustments::from(&wire);
+
+        assert_eq!(Prices::try_from(&wire).unwrap().iter().count(), 0);
+        assert_eq!(
+            adjustments
+                .get(&Uplift::NonGlobalEndpoint)
+                .map(Multiplier::get),
+            Some(1.1)
+        );
+        assert_eq!(
+            adjustments
+                .get(&Uplift::Residency(Residency::Eu))
+                .map(Multiplier::get),
+            Some(1.2)
+        );
+        assert_eq!(adjustments.get(&Uplift::Residency(Residency::Us)), None);
+    }
+
+    #[rstest]
+    fn a_model_reports_field_provider_and_price_problems_together() {
+        let Value::Object(fields) = json!({
+            "mode": "chatt",
+            "tiered_pricing": [{"input_cost_per_token": 1e-6}]
+        }) else {
+            unreachable!()
+        };
+        let problems: Vec<String> = Model::from_fields(&fields)
+            .unwrap_err()
+            .iter()
+            .map(|problem| litellm_error::Report::new(problem).to_string())
+            .collect();
+
+        assert_eq!(problems.len(), 3, "{problems:?}");
+        assert!(problems[0].starts_with("mode: invalid value: unknown variant `chatt`"));
+        assert_eq!(problems[1], "litellm_provider: required");
+        assert_eq!(
+            problems[2],
+            "tiered_pricing[0]: a tier needs exactly one of range and max_results_range"
+        );
+    }
+
+    #[rstest]
+    fn a_null_value_is_absent() {
+        let Value::Object(fields) = json!({
+            "litellm_provider": "openai",
+            "input_cost_per_token": null,
+            "mode": null
+        }) else {
+            unreachable!()
+        };
+        let model = Model::from_fields(&fields).unwrap();
+
+        assert_eq!(model.prices.iter().count(), 0);
+        assert_eq!(model.attributes.mode, None);
+        assert!(model.unrecognized.is_empty());
+    }
+
+    #[rstest]
+    #[case::tokens_win_over_characters(
+        json!({"litellm_provider": "vertex_ai-language-models", "input_cost_per_character": 1e-6, "input_cost_per_token": 1e-6}),
+        Billing::default()
+    )]
+    #[case::characters_only(
+        json!({"litellm_provider": "vertex_ai-language-models", "input_cost_per_character": 1e-6}),
+        Billing { text_unit: TextUnit::Character, ..Billing::default() }
+    )]
+    #[case::xai_band_is_inclusive(
+        json!({"litellm_provider": "xai", "input_cost_per_token": 1e-6}),
+        Billing { band_edge: BandEdge::Inclusive, ..Billing::default() }
+    )]
+    #[case::fireworks_without_cache_read_borrows_half(
+        json!({"litellm_provider": "fireworks_ai", "input_cost_per_token": 1e-6}),
+        Billing { cache_read_fallback: CacheReadFallback::HalfInputRate, ..Billing::default() }
+    )]
+    #[case::fireworks_with_cache_read_keeps_it(
+        json!({"litellm_provider": "fireworks_ai", "input_cost_per_token": 1e-6, "cache_read_input_token_cost": 1e-7}),
+        Billing::default()
+    )]
+    #[case::transcription_prices_the_audio_file(
+        json!({"litellm_provider": "openai", "mode": "audio_transcription", "input_cost_per_second": 1e-4}),
+        Billing { wall_clock: WallClock::AudioFile, ..Billing::default() }
+    )]
+    #[case::provisioned_chat_prices_response_time(
+        json!({"litellm_provider": "bedrock", "mode": "chat", "cost_per_second": 1e-2}),
+        Billing::default()
+    )]
+    #[case::image_generation_is_never_wall_clock(
+        json!({"litellm_provider": "openai", "mode": "image_generation"}),
+        Billing { wall_clock: WallClock::Never, ..Billing::default() }
+    )]
+    #[case::gemini_per_query_search(
+        json!({"litellm_provider": "gemini", "web_search_billing_unit": "per_query"}),
+        Billing { web_search: WebSearchUnit::PerQuery, ..Billing::default() }
+    )]
+    #[case::dashscope_tiers_bill_the_whole_call(
+        json!({"litellm_provider": "dashscope", "tiered_pricing": [{"range": [0, 32000], "input_cost_per_token": 1e-6}]}),
+        Billing { tier_scope: TierScope::WholeCall, ..Billing::default() }
+    )]
+    fn billing_is_lowered_from_the_row_and_its_provider(
+        #[case] entry: Value,
+        #[case] expected: Billing,
+    ) {
+        assert_eq!(prices(entry).unwrap().billing(), expected);
+    }
+}

litellm-rust/crates/model-catalog/src/model.rs

@@ -0,0 +1,22 @@
+use std::collections::BTreeMap;
+
+use litellm_cost::price::{Adjustments, Prices};
+
+use crate::error::Error;
+use crate::wire::{Attributes, EntryWire, Fields};
+
+/// One catalog entry read into types.
+#[derive(Clone, Debug, PartialEq)]
+pub struct Model {
+    pub attributes: Attributes,
+    pub prices: Prices,
+    pub adjustments: Adjustments,
+    pub capabilities: BTreeMap<String, bool>,
+    pub unrecognized: Vec<String>,
+}
+
+impl Model {
+    pub fn from_fields(fields: &Fields) -> Result<Self, Vec<Error>> {
+        Self::try_from(EntryWire::read(fields))
+    }
+}

litellm-rust/crates/model-catalog/src/model_info.rs

@@ -1,582 +1,194 @@
-use crate::capabilities::{AudioFormat, InputModality, Mode, OutputModality, VertexAiAudioApi};
-use crate::pricing::{OffPeakPricing, SearchContextCostPerQuery, TieredRate, WebSearchBillingUnit};
-use litellm_llms_types::formats::chat_completions::ReasoningEffort;
-use serde::{Deserialize, Serialize};
+use indexmap::IndexMap;
 use serde_json::Value;
-use std::collections::BTreeMap;
 
-/// Typed mirror of one catalog model entry.
-#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
-#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
-pub struct ModelInfo {
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub annotation_cost_per_page: Option<f64>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub annotation_cost_per_page_batches: Option<f64>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub audio_transcription_config: Option<String>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub bedrock_converse_supports_strict_tools: Option<bool>,
-    /// Highest reasoning effort the Bedrock output_config accepts for this model.
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub bedrock_output_config_effort_ceiling: Option<ReasoningEffort>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub cache_creation_input_audio_token_cost: Option<f64>,
-    /// USD per token written to the provider's prompt cache.
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub cache_creation_input_token_cost: Option<f64>,
-    /// Rate applied once the prompt exceeds the token threshold in the field name.
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub cache_creation_input_token_cost_above_32k_tokens: Option<f64>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub cache_creation_input_token_cost_above_100k_tokens: Option<f64>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub cache_creation_input_token_cost_above_100k_tokens_batches: Option<f64>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub cache_creation_input_token_cost_above_128k_tokens: Option<f64>,
-    /// Rate applied once the prompt exceeds the token threshold in the field name.
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub cache_creation_input_token_cost_above_1hr: Option<f64>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub cache_creation_input_token_cost_above_1hr_above_100k_tokens: Option<f64>,
-    /// Rate applied once the prompt exceeds the token threshold in the field name.
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub cache_creation_input_token_cost_above_1hr_above_200k_tokens: Option<f64>,
-    /// Rate applied once the prompt exceeds the token threshold in the field name.
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub cache_creation_input_token_cost_above_200k_tokens: Option<f64>,
-    /// Rate applied once the prompt exceeds the token threshold in the field name.
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub cache_creation_input_token_cost_above_200k_tokens_batches: Option<f64>,
-    /// Rate applied once the prompt exceeds the token threshold in the field name.
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub cache_creation_input_token_cost_above_256k_tokens: Option<f64>,
-    /// Rate applied once the prompt exceeds the token threshold in the field name.
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub cache_creation_input_token_cost_above_272k_tokens: Option<f64>,
-    /// Rate applied once the prompt exceeds the token threshold in the field name.
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub cache_creation_input_token_cost_above_272k_tokens_batches: Option<f64>,
-    /// Flex service-tier rate for the same-named base field.
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub cache_creation_input_token_cost_above_272k_tokens_flex: Option<f64>,
-    /// Priority service-tier rate for the same-named base field.
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub cache_creation_input_token_cost_above_272k_tokens_priority: Option<f64>,
-    /// Ultrafast service-tier rate for the same-named base field.
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub cache_creation_input_token_cost_above_272k_tokens_ultrafast: Option<f64>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub cache_creation_input_token_cost_batches: Option<f64>,
-    /// Flex service-tier rate for the same-named base field.
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub cache_creation_input_token_cost_flex: Option<f64>,
-    /// Priority service-tier rate for the same-named base field.
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub cache_creation_input_token_cost_priority: Option<f64>,
-    /// Ultrafast service-tier rate for the same-named base field.
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub cache_creation_input_token_cost_ultrafast: Option<f64>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub cache_read_input_audio_token_cost: Option<f64>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub cache_read_input_image_token_cost: Option<f64>,
-    /// USD per prompt token served from the provider's prompt cache.
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub cache_read_input_token_cost: Option<f64>,
-    /// Rate applied once the prompt exceeds the token threshold in the field name.
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub cache_read_input_token_cost_above_32k_tokens: Option<f64>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub cache_read_input_token_cost_above_100k_tokens: Option<f64>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub cache_read_input_token_cost_above_100k_tokens_batches: Option<f64>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub cache_read_input_token_cost_above_128k_tokens: Option<f64>,
-    /// Rate applied once the prompt exceeds the token threshold in the field name.
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub cache_read_input_token_cost_above_200k_tokens: Option<f64>,
-    /// Rate applied once the prompt exceeds the token threshold in the field name.
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub cache_read_input_token_cost_above_200k_tokens_batches: Option<f64>,
-    /// Priority service-tier rate for the same-named base field.
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub cache_read_input_token_cost_above_200k_tokens_priority: Option<f64>,
-    /// Rate applied once the prompt exceeds the token threshold in the field name.
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub cache_read_input_token_cost_above_256k_tokens: Option<f64>,
-    /// Rate applied once the prompt exceeds the token threshold in the field name.
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub cache_read_input_token_cost_above_272k_tokens: Option<f64>,
-    /// Rate applied once the prompt exceeds the token threshold in the field name.
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub cache_read_input_token_cost_above_272k_tokens_batches: Option<f64>,
-    /// Flex service-tier rate for the same-named base field.
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub cache_read_input_token_cost_above_272k_tokens_flex: Option<f64>,
-    /// Priority service-tier rate for the same-named base field.
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub cache_read_input_token_cost_above_272k_tokens_priority: Option<f64>,
-    /// Ultrafast service-tier rate for the same-named base field.
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub cache_read_input_token_cost_above_272k_tokens_ultrafast: Option<f64>,
-    /// Rate applied once the prompt exceeds the token threshold in the field name.
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub cache_read_input_token_cost_above_512k_tokens: Option<f64>,
-    /// Balanced service-tier rate for the same-named base field.
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub cache_read_input_token_cost_balanced: Option<f64>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub cache_read_input_token_cost_batches: Option<f64>,
-    /// Flex service-tier rate for the same-named base field.
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub cache_read_input_token_cost_flex: Option<f64>,
-    /// Priority service-tier rate for the same-named base field.
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub cache_read_input_token_cost_priority: Option<f64>,
-    /// Ultrafast service-tier rate for the same-named base field.
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub cache_read_input_token_cost_ultrafast: Option<f64>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub citation_cost_per_token: Option<f64>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub code_interpreter_cost_per_session: Option<f64>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub comment: Option<String>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub computer_use_input_cost_per_1k_tokens: Option<f64>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub computer_use_output_cost_per_1k_tokens: Option<f64>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub cost_per_second: Option<f64>,
-    /// Reasoning effort the provider applies when the request omits reasoning_effort. Gates whether a non-default temperature or the top_p/logprobs sampling params are accepted, which hold only when the effort resolves to 'none'.
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub default_reasoning_effort: Option<ReasoningEffort>,
-    /// Date the provider deprecates the model, YYYY-MM-DD.
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub deprecation_date: Option<String>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub file_search_cost_per_1k_calls: Option<f64>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub file_search_cost_per_gb_per_day: Option<f64>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub gemini_audio_only_live: Option<bool>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub gemini_native_audio: Option<bool>,
-    /// USD per Grounding with Google Maps request; billed per query or per prompt per web_search_billing_unit.
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub google_maps_grounding_cost_per_query: Option<f64>,
-    /// USD cost per billable guardrail unit, keyed by the provider's usage counter name (e.g. Bedrock's contentPolicyUnits).
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub guardrail_cost_per_unit: Option<BTreeMap<String, f64>>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub input_cost_per_audio_per_second: Option<f64>,
-    /// Rate applied once the prompt exceeds the token threshold in the field name.
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub input_cost_per_audio_per_second_above_128k_tokens: Option<f64>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub input_cost_per_audio_token: Option<f64>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub input_cost_per_audio_token_batches: Option<f64>,
-    /// Priority service-tier rate for the same-named base field.
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub input_cost_per_audio_token_priority: Option<f64>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub input_cost_per_character: Option<f64>,
-    /// Rate applied once the prompt exceeds the token threshold in the field name.
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub input_cost_per_character_above_128k_tokens: Option<f64>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub input_cost_per_image: Option<f64>,
-    /// Rate applied once the prompt exceeds the token threshold in the field name.
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub input_cost_per_image_above_128k_tokens: Option<f64>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub input_cost_per_image_token: Option<f64>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub input_cost_per_image_token_batches: Option<f64>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub input_cost_per_pixel: Option<f64>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub input_cost_per_query: Option<f64>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub input_cost_per_request: Option<f64>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub input_cost_per_second: Option<f64>,
-    /// USD per prompt token.
-    #[serde(skip_serializing_if = "Option::is_none")]
+/// The fields main's `ModelInfoBase(...)` copies (`litellm/utils.py` at c7c895db45), one typed field each,
+/// read from the raw row without lowering it, so building one never fails
+#[derive(Clone, Debug, Default, PartialEq)]
+pub struct ModelInfoFields {
+    pub key: String,
+    pub max_tokens: Option<i64>,
+    pub max_input_tokens: Option<i64>,
+    pub max_output_tokens: Option<i64>,
     pub input_cost_per_token: Option<f64>,
-    /// Rate applied once the prompt exceeds the token threshold in the field name.
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub input_cost_per_token_above_32k_tokens: Option<f64>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub input_cost_per_token_above_100k_tokens: Option<f64>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub input_cost_per_token_above_100k_tokens_batches: Option<f64>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub input_cost_per_token_above_128k_tokens: Option<f64>,
-    /// Rate applied once the prompt exceeds the token threshold in the field name.
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub input_cost_per_token_above_200k_tokens: Option<f64>,
-    /// Rate applied once the prompt exceeds the token threshold in the field name.
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub input_cost_per_token_above_200k_tokens_batches: Option<f64>,
-    /// Priority service-tier rate for the same-named base field.
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub input_cost_per_token_above_200k_tokens_priority: Option<f64>,
-    /// Rate applied once the prompt exceeds the token threshold in the field name.
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub input_cost_per_token_above_256k_tokens: Option<f64>,
-    /// Rate applied once the prompt exceeds the token threshold in the field name.
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub input_cost_per_token_above_272k_tokens: Option<f64>,
-    /// Rate applied once the prompt exceeds the token threshold in the field name.
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub input_cost_per_token_above_272k_tokens_batches: Option<f64>,
-    /// Flex service-tier rate for the same-named base field.
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub input_cost_per_token_above_272k_tokens_flex: Option<f64>,
-    /// Priority service-tier rate for the same-named base field.
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub input_cost_per_token_above_272k_tokens_priority: Option<f64>,
-    /// Ultrafast service-tier rate for the same-named base field.
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub input_cost_per_token_above_272k_tokens_ultrafast: Option<f64>,
-    /// Rate applied once the prompt exceeds the token threshold in the field name.
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub input_cost_per_token_above_512k_tokens: Option<f64>,
-    /// Balanced service-tier rate for the same-named base field.
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub input_cost_per_token_balanced: Option<f64>,
-    /// USD per prompt token via the provider's batch API.
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub input_cost_per_token_batches: Option<f64>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub input_cost_per_token_cache_hit: Option<f64>,
-    /// Flex service-tier rate for the same-named base field.
-    #[serde(skip_serializing_if = "Option::is_none")]
     pub input_cost_per_token_flex: Option<f64>,
-    /// Priority service-tier rate for the same-named base field.
-    #[serde(skip_serializing_if = "Option::is_none")]
     pub input_cost_per_token_priority: Option<f64>,
-    /// Ultrafast service-tier rate for the same-named base field.
-    #[serde(skip_serializing_if = "Option::is_none")]
+    pub input_cost_per_token_balanced: Option<f64>,
     pub input_cost_per_token_ultrafast: Option<f64>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub input_cost_per_video_per_second: Option<f64>,
-    /// Rate applied once the prompt exceeds the token threshold in the field name.
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub input_cost_per_video_per_second_above_128k_tokens: Option<f64>,
-    /// Rate applied once the prompt exceeds the token threshold in the field name.
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub input_cost_per_video_per_second_above_15s_interval: Option<f64>,
-    /// Rate applied once the prompt exceeds the token threshold in the field name.
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub input_cost_per_video_per_second_above_8s_interval: Option<f64>,
-    #[serde(skip_serializing_if = "Option::is_none")]
+    pub cache_creation_input_token_cost: Option<f64>,
+    pub cache_creation_input_token_cost_above_200k_tokens: Option<f64>,
+    pub cache_creation_input_token_cost_above_272k_tokens: Option<f64>,
+    pub cache_creation_input_token_cost_above_272k_tokens_priority: Option<f64>,
+    pub cache_creation_input_token_cost_above_272k_tokens_flex: Option<f64>,
+    pub cache_creation_input_token_cost_above_272k_tokens_ultrafast: Option<f64>,
+    pub cache_creation_input_token_cost_flex: Option<f64>,
+    pub cache_creation_input_token_cost_priority: Option<f64>,
+    pub cache_creation_input_token_cost_ultrafast: Option<f64>,
+    pub cache_read_input_token_cost: Option<f64>,
+    pub cache_read_input_audio_token_cost: Option<f64>,
+    pub cache_read_input_audio_token_cost_flex: Option<f64>,
+    pub cache_read_input_audio_token_cost_priority: Option<f64>,
+    pub cache_read_input_audio_token_cost_batches: Option<f64>,
+    pub explicit_cache_read_input_token_cost: Option<f64>,
+    pub prompt_cache_min_tokens: Option<i64>,
+    pub cache_read_input_token_cost_above_200k_tokens: Option<f64>,
+    pub cache_read_input_token_cost_above_200k_tokens_priority: Option<f64>,
+    pub cache_read_input_token_cost_above_272k_tokens: Option<f64>,
+    pub cache_read_input_token_cost_above_272k_tokens_priority: Option<f64>,
+    pub cache_read_input_token_cost_above_272k_tokens_flex: Option<f64>,
+    pub cache_read_input_token_cost_above_272k_tokens_ultrafast: Option<f64>,
+    pub cache_read_input_token_cost_above_512k_tokens: Option<f64>,
+    pub cache_read_input_token_cost_flex: Option<f64>,
+    pub cache_read_input_token_cost_priority: Option<f64>,
+    pub cache_read_input_token_cost_balanced: Option<f64>,
+    pub cache_read_input_token_cost_ultrafast: Option<f64>,
+    pub cache_read_input_token_cost_batches: Option<f64>,
+    pub cache_read_input_token_cost_above_200k_tokens_batches: Option<f64>,
+    pub cache_read_input_token_cost_above_272k_tokens_batches: Option<f64>,
+    pub cache_creation_input_token_cost_batches: Option<f64>,
+    pub cache_creation_input_token_cost_above_200k_tokens_batches: Option<f64>,
+    pub cache_creation_input_token_cost_above_272k_tokens_batches: Option<f64>,
+    pub cache_creation_input_token_cost_above_1hr: Option<f64>,
+    pub cache_creation_input_token_cost_above_12hr: Option<f64>,
+    pub cache_creation_input_token_cost_above_12hr_flex: Option<f64>,
+    pub cache_creation_input_token_cost_above_12hr_priority: Option<f64>,
+    pub cache_creation_input_token_cost_above_12hr_above_272k_tokens: Option<f64>,
+    pub cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex: Option<f64>,
+    pub cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority: Option<f64>,
+    pub cache_creation_input_token_cost_above_12hr_batches: Option<f64>,
+    pub cache_creation_input_token_cost_above_12hr_ultrafast: Option<f64>,
+    pub cache_creation_input_token_cost_above_12hr_above_272k_tokens_batches: Option<f64>,
+    pub cache_creation_input_token_cost_above_12hr_above_272k_tokens_ultrafast: Option<f64>,
+    pub off_peak_pricing: Option<Value>,
+    pub input_cost_per_character: Option<f64>,
+    pub input_cost_per_token_above_128k_tokens: Option<f64>,
+    pub input_cost_per_token_above_200k_tokens: Option<f64>,
+    pub input_cost_per_token_above_200k_tokens_priority: Option<f64>,
+    pub input_cost_per_token_above_272k_tokens: Option<f64>,
+    pub input_cost_per_token_above_272k_tokens_priority: Option<f64>,
+    pub input_cost_per_token_above_272k_tokens_flex: Option<f64>,
+    pub input_cost_per_token_above_272k_tokens_ultrafast: Option<f64>,
+    pub input_cost_per_token_above_512k_tokens: Option<f64>,
+    pub input_cost_per_query: Option<f64>,
+    pub cost_per_second: Option<f64>,
+    pub input_cost_per_second: Option<f64>,
+    pub input_cost_per_text_message: Option<f64>,
+    pub input_cost_per_audio_token: Option<f64>,
+    pub input_cost_per_audio_token_flex: Option<f64>,
+    pub input_cost_per_audio_token_priority: Option<f64>,
+    pub input_cost_per_image_token: Option<f64>,
     pub input_cost_per_video_token: Option<f64>,
-    #[serde(skip_serializing_if = "Option::is_none")]
+    pub input_cost_per_audio_token_batches: Option<f64>,
+    pub input_cost_per_image_token_batches: Option<f64>,
+    pub input_cost_per_image: Option<f64>,
+    pub input_cost_per_audio_per_second: Option<f64>,
+    pub input_cost_per_video_per_second: Option<f64>,
+    pub input_cost_per_token_batches: Option<f64>,
     pub input_cost_per_video_token_batches: Option<f64>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub input_dbu_cost_per_token: Option<f64>,
-    /// LiteLLM provider slug; one of https://docs.litellm.ai/docs/providers.
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub litellm_provider: Option<String>,
-    /// Maximum prompt/context tokens the model accepts.
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub max_input_tokens: Option<u64>,
-    /// Maximum tokens the model can generate in one response.
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub max_output_tokens: Option<u64>,
-    /// Legacy field: max output tokens if the provider specifies it, else max input tokens.
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub max_tokens: Option<u64>,
-    /// Free-form notes about the entry (e.g. pricing derivation).
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub metadata: Option<BTreeMap<String, Value>>,
-    /// Primary API surface / task type of the model.
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub mode: Option<Mode>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub ocr_cost_per_credit: Option<f64>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub ocr_cost_per_page: Option<f64>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub ocr_cost_per_page_batches: Option<f64>,
-    /// Rates that replace the same-named base fields while the request falls inside the stated UTC windows.
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub off_peak_pricing: Option<OffPeakPricing>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub output_cost_per_audio_token: Option<f64>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub output_cost_per_character: Option<f64>,
-    /// Rate applied once the prompt exceeds the token threshold in the field name.
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub output_cost_per_character_above_128k_tokens: Option<f64>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub output_cost_per_image: Option<f64>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub output_cost_per_image_1024: Option<f64>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub output_cost_per_image_1536: Option<f64>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub output_cost_per_image_512: Option<f64>,
-    #[serde(
-        rename = "output_cost_per_image_0.5K",
-        skip_serializing_if = "Option::is_none"
-    )]
-    pub output_cost_per_image_0_5k: Option<f64>,
-    #[serde(
-        rename = "output_cost_per_image_1K",
-        skip_serializing_if = "Option::is_none"
-    )]
-    pub output_cost_per_image_1k: Option<f64>,
-    #[serde(
-        rename = "output_cost_per_image_2K",
-        skip_serializing_if = "Option::is_none"
-    )]
-    pub output_cost_per_image_2k: Option<f64>,
-    #[serde(
-        rename = "output_cost_per_image_4K",
-        skip_serializing_if = "Option::is_none"
-    )]
-    pub output_cost_per_image_4k: Option<f64>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub output_cost_per_image_token: Option<f64>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub output_cost_per_pixel: Option<f64>,
-    /// USD per reasoning/thinking token, when billed separately.
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub output_cost_per_reasoning_token: Option<f64>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub output_cost_per_second: Option<f64>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub output_cost_per_second_1080p: Option<f64>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub output_cost_per_second_2k: Option<f64>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub output_cost_per_second_480p: Option<f64>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub output_cost_per_second_4k: Option<f64>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub output_cost_per_second_720p: Option<f64>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub output_cost_per_second_768p: Option<f64>,
-    /// USD per generated token.
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub output_cost_per_token: Option<f64>,
-    /// Rate applied once the prompt exceeds the token threshold in the field name.
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub output_cost_per_token_above_32k_tokens: Option<f64>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub output_cost_per_token_above_100k_tokens: Option<f64>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub output_cost_per_token_above_100k_tokens_batches: Option<f64>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub output_cost_per_token_above_128k_tokens: Option<f64>,
-    /// Rate applied once the prompt exceeds the token threshold in the field name.
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub output_cost_per_token_above_200k_tokens: Option<f64>,
-    /// Rate applied once the prompt exceeds the token threshold in the field name.
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub output_cost_per_token_above_200k_tokens_batches: Option<f64>,
-    /// Priority service-tier rate for the same-named base field.
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub output_cost_per_token_above_200k_tokens_priority: Option<f64>,
-    /// Rate applied once the prompt exceeds the token threshold in the field name.
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub output_cost_per_token_above_256k_tokens: Option<f64>,
-    /// Rate applied once the prompt exceeds the token threshold in the field name.
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub output_cost_per_token_above_272k_tokens: Option<f64>,
-    /// Rate applied once the prompt exceeds the token threshold in the field name.
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub output_cost_per_token_above_272k_tokens_batches: Option<f64>,
-    /// Flex service-tier rate for the same-named base field.
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub output_cost_per_token_above_272k_tokens_flex: Option<f64>,
-    /// Priority service-tier rate for the same-named base field.
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub output_cost_per_token_above_272k_tokens_priority: Option<f64>,
-    /// Ultrafast service-tier rate for the same-named base field.
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub output_cost_per_token_above_272k_tokens_ultrafast: Option<f64>,
-    /// Rate applied once the prompt exceeds the token threshold in the field name.
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub output_cost_per_token_above_512k_tokens: Option<f64>,
-    /// Balanced service-tier rate for the same-named base field.
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub output_cost_per_token_balanced: Option<f64>,
-    /// USD per generated token via the provider's batch API.
-    #[serde(skip_serializing_if = "Option::is_none")]
+    pub input_cost_per_token_above_200k_tokens_batches: Option<f64>,
+    pub input_cost_per_token_above_272k_tokens_batches: Option<f64>,
     pub output_cost_per_token_batches: Option<f64>,
-    /// Flex service-tier rate for the same-named base field.
-    #[serde(skip_serializing_if = "Option::is_none")]
+    pub output_cost_per_token_above_200k_tokens_batches: Option<f64>,
+    pub output_cost_per_token_above_272k_tokens_batches: Option<f64>,
+    pub output_cost_per_token: Option<f64>,
     pub output_cost_per_token_flex: Option<f64>,
-    /// Priority service-tier rate for the same-named base field.
-    #[serde(skip_serializing_if = "Option::is_none")]
     pub output_cost_per_token_priority: Option<f64>,
-    /// Ultrafast service-tier rate for the same-named base field.
-    #[serde(skip_serializing_if = "Option::is_none")]
+    pub output_cost_per_token_balanced: Option<f64>,
     pub output_cost_per_token_ultrafast: Option<f64>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub output_cost_per_video_per_second: Option<f64>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub output_cost_per_video_token: Option<f64>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub output_dbu_cost_per_token: Option<f64>,
-    /// Embedding dimension for embedding models.
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub output_vector_size: Option<u64>,
-    /// Smallest prefix the provider will actually cache; absent means the provider default applies.
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub prompt_cache_min_tokens: Option<u64>,
-    /// Provider-internal routing hints (e.g. bedrock_invocation_schema).
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub provider_specific_entry: Option<BTreeMap<String, Value>>,
-    /// Exact reasoning_effort levels this deployment accepts; wins over supports_* flags.
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub reasoning_effort_levels: Option<Vec<ReasoningEffort>>,
-    /// Multiplier applied to all token costs when served from a non-global Vertex AI endpoint (e.g. 1.10 = +10%).
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub regional_endpoint_uplift_multiplier: Option<f64>,
-    /// Multiplier applied to all token costs for EU data residency (e.g. 1.10 = +10%).
-    #[serde(skip_serializing_if = "Option::is_none")]
     pub regional_processing_uplift_multiplier_eu: Option<f64>,
-    /// Multiplier applied to all token costs for US data residency (e.g. 1.10 = +10%).
-    #[serde(skip_serializing_if = "Option::is_none")]
     pub regional_processing_uplift_multiplier_us: Option<f64>,
-    /// Provider default requests-per-minute limit.
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub rpm: Option<u64>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub rules: Option<Vec<Value>>,
-    /// USD cost per web search query, keyed by search context size.
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub search_context_cost_per_query: Option<SearchContextCostPerQuery>,
-    /// URL of the provider pricing/model page this entry was taken from.
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub source: Option<String>,
-    /// Audio container formats the model can return.
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub supported_audio_formats: Option<Vec<AudioFormat>>,
-    /// OpenAI-style API routes this model can be called through, e.g. /v1/chat/completions.
-    #[serde(skip_serializing_if = "Option::is_none")]
+    pub regional_endpoint_uplift_multiplier: Option<f64>,
+    pub output_cost_per_audio_token: Option<f64>,
+    pub output_cost_per_character: Option<f64>,
+    pub output_cost_per_reasoning_token: Option<f64>,
+    pub output_cost_per_reasoning_token_flex: Option<f64>,
+    pub output_cost_per_reasoning_token_priority: Option<f64>,
+    pub output_cost_per_token_above_128k_tokens: Option<f64>,
+    pub output_cost_per_character_above_128k_tokens: Option<f64>,
+    pub output_cost_per_token_above_200k_tokens: Option<f64>,
+    pub output_cost_per_token_above_200k_tokens_priority: Option<f64>,
+    pub output_cost_per_token_above_272k_tokens: Option<f64>,
+    pub output_cost_per_token_above_272k_tokens_priority: Option<f64>,
+    pub output_cost_per_token_above_272k_tokens_flex: Option<f64>,
+    pub output_cost_per_token_above_272k_tokens_ultrafast: Option<f64>,
+    pub output_cost_per_token_above_512k_tokens: Option<f64>,
+    pub output_cost_per_second: Option<f64>,
+    pub output_cost_per_second_1080p: Option<f64>,
+    pub output_cost_per_second_480p: Option<f64>,
+    pub output_cost_per_second_720p: Option<f64>,
+    pub output_cost_per_second_768p: Option<f64>,
+    pub output_cost_per_second_2k: Option<f64>,
+    pub output_cost_per_second_4k: Option<f64>,
+    pub output_cost_per_video_per_second: Option<f64>,
+    pub output_cost_per_image: Option<f64>,
+    pub output_cost_per_pixel: Option<f64>,
+    pub output_cost_per_image_token: Option<f64>,
+    pub output_cost_per_image_token_batches: Option<f64>,
+    pub output_cost_per_video_token: Option<f64>,
+    pub output_vector_size: Option<i64>,
+    pub citation_cost_per_token: Option<f64>,
+    pub tiered_pricing: Option<Value>,
+    pub litellm_provider: Option<String>,
+    pub mode: Option<Value>,
     pub supported_endpoints: Option<Vec<String>>,
-    /// Input modalities the model accepts.
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub supported_modalities: Option<Vec<InputModality>>,
-    /// Output modalities the model can produce.
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub supported_output_modalities: Option<Vec<OutputModality>>,
-    /// Cloud regions the model is available in ('global' or region ids).
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub supported_regions: Option<Vec<String>>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub supports_adaptive_thinking: Option<bool>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub supports_anthropic_compaction: Option<bool>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub supports_anthropic_thinking_payload: Option<bool>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub supports_assistant_prefill: Option<bool>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub supports_audio_input: Option<bool>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub supports_audio_output: Option<bool>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub supports_bedrock_runtime_chat_completions_response_format: Option<bool>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub supports_bedrock_runtime_chat_completions_tools_with_reasoning: Option<bool>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub supports_computer_use: Option<bool>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub supports_embedding_image_input: Option<bool>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub supports_fast_mode: Option<bool>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub supports_forced_tool_use: Option<bool>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub supports_function_calling: Option<bool>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub supports_image_input: Option<bool>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub supports_image_size: Option<bool>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub supports_legacy_thinking: Option<bool>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub supports_low_reasoning_effort: Option<bool>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub supports_max_reasoning_effort: Option<bool>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub supports_mid_conversation_system: Option<bool>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub supports_minimal_reasoning_effort: Option<bool>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub supports_multimodal: Option<bool>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub supports_native_streaming: Option<bool>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub supports_native_structured_output: Option<bool>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub supports_none_reasoning_effort: Option<bool>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub supports_nova_canvas_image_edit: Option<bool>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub supports_output_config: Option<bool>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub supports_parallel_function_calling: Option<bool>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub supports_parallel_tool_use_config: Option<bool>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub supports_pdf_input: Option<bool>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub supports_prompt_cache_breakpoint: Option<bool>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub supports_prompt_caching: Option<bool>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub supports_reasoning: Option<bool>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub supports_response_schema: Option<bool>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub supports_sampling_params: Option<bool>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub supports_speed: Option<bool>,
-    #[serde(skip_serializing_if = "Option::is_none")]
     pub supports_system_messages: Option<bool>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub supports_thinking_cache_preservation: Option<bool>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub supports_tool_choice: Option<bool>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub supports_tool_search: Option<bool>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub supports_url_context: Option<bool>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub supports_video_input: Option<bool>,
-    #[serde(skip_serializing_if = "Option::is_none")]
+    pub supports_response_schema: Option<bool>,
     pub supports_vision: Option<bool>,
-    #[serde(skip_serializing_if = "Option::is_none")]
+    pub supports_function_calling: Option<bool>,
+    pub supports_parallel_function_calling: Option<bool>,
+    pub supports_tool_choice: Option<bool>,
+    pub supports_assistant_prefill: Option<bool>,
+    pub supports_prompt_caching: Option<bool>,
+    pub supports_prompt_cache_breakpoint: Option<bool>,
+    pub supports_thinking_cache_preservation: Option<bool>,
+    pub supports_audio_input: Option<bool>,
+    pub supports_video_input: Option<bool>,
+    pub supports_box_questions: Option<bool>,
+    pub supports_audio_output: Option<bool>,
+    pub supports_pdf_input: Option<bool>,
+    pub supports_embedding_image_input: Option<bool>,
+    pub supports_native_streaming: Option<bool>,
+    pub supports_native_structured_output: Option<bool>,
+    pub supports_bedrock_native_structured_output: Option<bool>,
     pub supports_web_search: Option<bool>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub supports_xhigh_reasoning_effort: Option<bool>,
-    #[serde(skip_serializing_if = "Option::is_none")]
+    pub supports_url_context: Option<bool>,
+    pub supports_reasoning: Option<bool>,
+    pub supports_adaptive_thinking: Option<bool>,
+    pub supports_legacy_thinking: Option<bool>,
     pub thinking_always_on: Option<bool>,
-    /// Context-length or result-count tiered rates; each tier's costs apply within its range.
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub tiered_pricing: Option<Vec<TieredRate>>,
-    /// Provider default tokens-per-minute limit.
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub tpm: Option<u64>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub use_openai_responses_path: Option<bool>,
-    #[serde(skip_serializing_if = "Option::is_none")]
+    pub supports_tool_search: Option<bool>,
+    pub supports_mid_conversation_system: Option<bool>,
+    pub supports_anthropic_thinking_payload: Option<bool>,
+    pub supports_anthropic_compaction: Option<bool>,
+    pub supports_none_reasoning_effort: Option<bool>,
+    pub supports_minimal_reasoning_effort: Option<bool>,
+    pub supports_low_reasoning_effort: Option<bool>,
+    pub supports_xhigh_reasoning_effort: Option<bool>,
+    pub supports_max_reasoning_effort: Option<bool>,
+    pub reasoning_effort_levels: Option<Value>,
+    pub default_reasoning_effort: Option<Value>,
+    pub bedrock_output_config_effort_ceiling: Option<Value>,
+    pub bedrock_converse_supports_strict_tools: Option<bool>,
+    pub supports_regex_lookaround: Option<bool>,
+    pub supports_computer_use: Option<bool>,
+    pub search_context_cost_per_query: Option<Value>,
+    pub web_search_billing_unit: Option<Value>,
+    pub search_free_requests: Option<Value>,
+    pub google_maps_grounding_cost_per_query: Option<f64>,
+    pub tpm: Option<i64>,
+    pub rpm: Option<i64>,
+    pub ocr_cost_per_page: Option<f64>,
+    pub ocr_cost_per_page_batches: Option<f64>,
+    pub ocr_cost_per_credit: Option<f64>,
+    pub annotation_cost_per_page: Option<f64>,
+    pub annotation_cost_per_page_batches: Option<f64>,
+    pub provider_specific_entry: Option<Value>,
     pub uses_embed_content: Option<bool>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub vector_store_cost_per_gb_per_day: Option<f64>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub vertex_ai_audio_api: Option<VertexAiAudioApi>,
-    /// Whether web search is billed per query or per prompt.
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub web_search_billing_unit: Option<WebSearchBillingUnit>,
+    pub supports_image_size: Option<bool>,
+    pub supported_audio_formats: Option<Value>,
+    pub token_count_framing: Option<i64>,
+    pub token_count_model: Option<Value>,
+    pub token_count_tokenizer: Option<Value>,
+    pub vertex_ai_audio_api: Option<Value>,
+    /// Every key matching `ABOVE_THRESHOLD_COST_KEY_PATTERN`, copied as written
+    pub above_threshold_costs: IndexMap<String, Value>,
 }

litellm-rust/crates/model-catalog/src/pricing.rs

@@ -1,97 +0,0 @@
-use serde::{Deserialize, Serialize};
-
-/// Whether web search is billed per query or per prompt.
-#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)]
-#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
-#[serde(rename_all = "snake_case")]
-pub enum WebSearchBillingUnit {
-    PerQuery,
-    PerPrompt,
-}
-
-/// UTC "HH:MM-HH:MM" window, or a list of them; a window may wrap past midnight.
-#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
-#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
-#[serde(untagged)]
-pub enum UtcHours {
-    Single(String),
-    Multiple(Vec<String>),
-}
-
-/// ISO-8601 weekday number (1 = Monday .. 7 = Sunday) or English day name.
-#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
-#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
-#[serde(untagged)]
-pub enum Weekday {
-    Number(u8),
-    Name(String),
-}
-
-/// One off-peak window entry inside `windows`.
-#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
-#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
-#[serde(deny_unknown_fields)]
-pub struct OffPeakWindow {
-    pub hours_utc: UtcHours,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub weekdays: Option<Vec<Weekday>>,
-}
-
-/// Rates that replace the same-named base fields inside the stated UTC windows.
-#[derive(Clone, Debug, Default, Deserialize, PartialEq, Serialize)]
-#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
-#[serde(deny_unknown_fields)]
-pub struct OffPeakPricing {
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub hours_utc: Option<UtcHours>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub windows: Option<Vec<OffPeakWindow>>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub weekday_timezone: Option<String>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub input_cost_per_token: Option<f64>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub output_cost_per_token: Option<f64>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub output_cost_per_reasoning_token: Option<f64>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub cache_read_input_token_cost: Option<f64>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub cache_creation_input_token_cost: Option<f64>,
-}
-
-/// USD cost per web search query, keyed by search context size.
-#[derive(Clone, Debug, Default, Deserialize, PartialEq, Serialize)]
-#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
-#[serde(deny_unknown_fields)]
-pub struct SearchContextCostPerQuery {
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub search_context_size_low: Option<f64>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub search_context_size_medium: Option<f64>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub search_context_size_high: Option<f64>,
-}
-
-/// One tier of a context-length or result-count tiered rate.
-#[derive(Clone, Debug, Default, Deserialize, PartialEq, Serialize)]
-#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
-#[serde(deny_unknown_fields)]
-pub struct TieredRate {
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub range: Option<[f64; 2]>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub max_results_range: Option<[f64; 2]>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub input_cost_per_token: Option<f64>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub output_cost_per_token: Option<f64>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub output_cost_per_reasoning_token: Option<f64>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub cache_read_input_token_cost: Option<f64>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub cache_creation_input_token_cost: Option<f64>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub input_cost_per_query: Option<f64>,
-}